def get_inference_results(projectID: int = Form(...),
modelID: int = Form(...),
inferenceDataFile: UploadFile = File(...)):
newDataPath = '/'
pickleFilePath = '/'
path = '/'
inferenceDataResultsPath = '/'
isAuto = False
try:
result = Project21Database.find_one(settings.DB_COLLECTION_PROJECT,
{"projectID": projectID})
if result is not None:
result = serialiseDict(result)
isAuto = result["isAuto"]
except Exception as e:
print("An Error Occured: ", e)
print("Could not find the project in the Project Collection")
try:
result = Project21Database.find_one(settings.DB_COLLECTION_MODEL, {
"modelID": modelID,
"belongsToProjectID": projectID
})
if result is not None:
result = serialiseDict(result)
if result["pickleFilePath"] is not None:
pickleFilePath = result["pickleFilePath"]
if result["pickleFolderPath"] is not None:
projectRunPath = os.path.join(result["pickleFolderPath"],
os.pardir)
path = os.path.join(projectRunPath, "inference_data")
if (not os.path.exists(path)):
os.makedirs(path)
newDataPath = os.path.join(path, 'inference_data.csv')
with open(newDataPath, "wb") as buffer:
shutil.copyfileobj(inferenceDataFile.file, buffer)
inference = Inference()
inferenceDataResultsPath = inference.inference(
pickleFilePath, newDataPath, path, isAuto)
Project21Database.insert_one(
settings.DB_COLLECTION_INFERENCE, {
"newData": newDataPath,
"results": inferenceDataResultsPath,
"belongsToUserID": currentIDs.get_current_user_id(),
"belongsToProjectID": projectID,
"belongsToModelID": modelID
})
if os.path.exists(inferenceDataResultsPath):
print({"Metrics Generation": "Successful"})
return FileResponse(inferenceDataResultsPath,
media_type="text/csv",
filename="inference.csv")
except Exception as e:
print("An error occured: ", e)
print("Unable to find model from model Collection")
return JSONResponse({"Metrics Generation": "Failed"})
def generate_project_manual_config_file(projectID, preprocessJSONFormData,
Project21Database):
"""
"""
# user_yaml=yaml.load(open(settings.CONFIG_PREPROCESS_YAML_FILE),Loader=SafeLoader)
random_id = generate_random_id()
preprocessJSONFormData["id"] = random_id
preprocessJSONFormData["raw_data_address"] = get_raw_data_path(
projectID, Project21Database)
location = "/"
random_id = generate_random_id()
try:
result_project = Project21Database.find_one(
settings.DB_COLLECTION_PROJECT, {"projectID": projectID})
result_project = serialiseDict(result_project)
if result_project is not None:
location = os.path.join(result_project["projectFolderPath"],
'run' + str(random_id))
except:
print("Unable to Update User's Project's Config File")
if (not os.path.exists(location)):
os.makedirs(location)
preprocessJSONFormData["location"] = location
with open(os.path.join(location, "preprocess_config.yaml"), "w") as f:
yaml.dump(preprocessJSONFormData, f)
f.close()
return os.path.join(location, 'preprocess_config.yaml'
), random_id, result_project["projectType"], location
def generate_project_timeseries_rf_config_file(projectID,currentIDs,timeseriesFormData,Project21Database):
user_yaml=yaml.load(open(settings.CONFIG_TIMESERIES_MANUAL_FILE),Loader=SafeLoader)
random_id=generate_random_id()
user_yaml["id"]=random_id
user_yaml["raw_data_address"]=get_raw_data_path(projectID,Project21Database)
user_yaml["target_column_name"]=timeseriesFormData["target"]
user_yaml["date_index"]=timeseriesFormData["dateColumn"]
user_yaml["frequency"]=timeseriesFormData["frequency"]
try:
result_project=Project21Database.find_one(settings.DB_COLLECTION_PROJECT,{"projectID":projectID})
result_project=serialiseDict(result_project)
if result_project is not None:
user_yaml["location"]=os.path.join(result_project["projectFolderPath"],'run'+str(random_id))
user_yaml["experimentname"]=result_project["projectName"]
else:
user_yaml["location"]='/'
user_yaml["experimentname"]='default'
except Exception as e:
print("Unable to Update User's Project's Config File. An Error Occured: ",e)
if(not os.path.exists(user_yaml["location"])):
os.makedirs(user_yaml["location"])
with open(os.path.join(user_yaml["location"],"preprocess_config.yaml"), "w") as f:
yaml.dump(user_yaml,f)
f.close()
return os.path.join(user_yaml["location"],'preprocess_config.yaml'), user_yaml["location"],random_id, result_project["projectType"]
def get_one_data(dataID: int):
try:
data = serialiseDict(
Project21Database.find_one(settings.DB_COLLECTION_DATA,
{"dataID": dataID}))
except:
return ErrorResponseModel("An Error Occured", 404,
"Data could not be found")
return data
def get_one_model(modelID: int):
try:
model = serialiseDict(
Project21Database.find_one(settings.DB_COLLECTION_MODEL,
{"modelID": modelID}))
except:
return ErrorResponseModel("An Error Occured", 404,
"Model could not be found")
return model
def generate_project_auto_config_file(projectID,currentIDs,formData,Project21Database):
"""
Returns the auto config file generated for the project
...
Parameters
----------
projectID: int
currentIDs: obj
formData: obj
Project21Databse: obj
Returns
-------
tuple: path, randomID, problemType
"""
user_yaml=yaml.load(open(settings.CONFIG_AUTO_YAML_FILE),Loader=SafeLoader)
random_id=generate_random_id()
user_yaml["id"]=random_id
user_yaml["raw_data_address"]=get_raw_data_path(projectID,Project21Database)
user_yaml["target_col_name"]=formData["target"]
user_yaml["na_value"]=formData["nulltype"]
user_yaml["n"]=formData["modelnumber"]
user_yaml["problem_type"]=get_project_type(projectID,Project21Database)
if(user_yaml["problem_type"]=='clustering'):
user_yaml["clusteringType"]=formData["clusteringType"]
user_yaml["numClusters"]=formData["numClusters"]
# try:
# result_model=Project21Database.find_one(settings.DB_COLLECTION_MODEL,{"modelID":currentIDs.get_current_model_id()})
# result_model=serialiseDict(result_model)
# if result_model is not None:
# user_yaml["problem_type"]=result_model["modelType"]
# else:
# user_yaml["problem_type"]='default'
# except:
# print("Unable to Update User's Project's AutoConfig File")
try:
result_project=Project21Database.find_one(settings.DB_COLLECTION_PROJECT,{"projectID":projectID})
result_project=serialiseDict(result_project)
if result_project is not None:
user_yaml["location"]=os.path.join(result_project["projectFolderPath"],'run'+str(random_id))
user_yaml["experimentname"]=result_project["projectName"]
else:
user_yaml["location"]='/'
user_yaml["experimentname"]='default'
except:
print("Unable to Update User's Project's Config File")
if(not os.path.exists(user_yaml["location"])):
os.makedirs(user_yaml["location"])
with open(os.path.join(user_yaml["location"],"autoConfig.yaml"), "w") as f:
yaml.dump(user_yaml,f)
f.close()
return os.path.join(user_yaml["location"],'autoConfig.yaml'), random_id , user_yaml["problem_type"]
def get_metrics(projectID, modelID, Project21Database):
try:
result = Project21Database.find_one(settings.DB_COLLECTION_METRICS, {
"belongsToProjectID": projectID,
"belongsToModelID": modelID
})
if result is not None:
result = serialiseDict(result)
if result["addressOfMetricsFile"] is not None:
return str(result["addressOfMetricsFile"])
except Exception as e:
print("An Error Occured: ", e)
return 'Metrics not found'
def get_clean_data_path(dataID, Project21Database):
try:
result = Project21Database.find_one(settings.DB_COLLECTION_DATA,
{"dataID": dataID})
result = serialiseDict(result)
if result is not None:
return result["cleanDataPath"]
else:
return '/'
except:
print(
"An error occured while retreiving cleanDataPath from Data Collection"
)
def get_pickle_file_path(modelID,Project21Database):
try:
result=Project21Database.find_one(settings.DB_COLLECTION_MODEL,{"modelID":modelID})
if result is not None:
result=serialiseDict(result)
return result["pickleFilePath"]
# if result["pickleFilePath"] is not None:
# return result["pickleFilePath"]
else:
print("result is none")
return '/'
except Exception as e:
print("An error occured while retreiving pickleFilePath from the Model Collection")
print("Error: ",e)
return '/'
def get_raw_data_path(projectID, Project21Database):
try:
result = Project21Database.find_one(settings.DB_COLLECTION_PROJECT,
{"projectID": projectID})
result = serialiseDict(result)
if result is not None:
return result["rawDataPath"] #path string returned
else:
return ''
except Exception as e:
print("An Error Occured: ", e)
print(
"An Error Occured While Retreiving rawDataPath from the Project Collection"
)
return ''
def get_project_type(projectID, Project21Database):
try:
result = Project21Database.find_one(settings.DB_COLLECTION_PROJECT,
{"projectID": projectID})
result = serialiseDict(result)
if result is not None:
return result["projectType"]
else:
return ''
except Exception as e:
print("An Error Occured: ", e)
print(
"An Error Occured while retreiving projectType from the Project Collection"
)
return ''
def get_all_project_details(userID: int):
listOfProjects = []
try:
results = Project21Database.find(settings.DB_COLLECTION_PROJECT,
{"belongsToUserID": userID})
for result in results:
result = serialiseDict(result)
if result["target"] is not None:
projectTemplate = {
"projectID": result["projectID"],
"projectName": result["projectName"],
"target": result["target"],
"modelType": result["projectType"],
"listOfDataIDs": result["listOfDataIDs"],
"isAuto": result["isAuto"]
}
listOfProjects.append(projectTemplate)
except Exception as e:
print("An Error Occured: ", e)
print("Unable to get all projects")
return JSONResponse({"GetAllProjects": "Failed"})
return listOfProjects
def get_plots(
projectID: int): #check if it already exists - change location address
try:
result = Project21Database.find_one(settings.DB_COLLECTION_PROJECT,
{"projectID": projectID})
if result is not None:
result = serialiseDict(result)
if (result["projectType"] == 'clustering'):
return FileResponse(result["clusterPlotLocation"],
media_type="text/html",
filename="plot.html")
if (result["projectType"] == 'timeseries'):
return FileResponse(result["plotLocation"],
media_type="text/html",
filename="plot.html")
if result["configFileLocation"] is not None:
plotFilePath = plot(
result["configFileLocation"]
) #plot function requires the auto config file
try:
Project21Database.update_one(
settings.DB_COLLECTION_PROJECT,
{"projectID": projectID},
{"$set": {
"plotsPath": plotFilePath
}})
except Exception as e:
print(
"An Error occured while storing the plot path into the project collection"
)
return FileResponse(plotFilePath,
media_type='text/html',
filename='plot.html')
except Exception as e:
print("An Error Occured: ", e)
return JSONResponse({"Plots": "Not generated"})
def get_one_metrics(belongsToModelID:int):
try:
metrics=serialiseDict(Project21Database.find_one(settings.DB_COLLECTION_METRICS,{"belongsToModelID":belongsToModelID}))
except:
return ErrorResponseModel("An Error Occured",404,"Metrics could not be found")
return metrics
def get_one_project(projectID:int):
try:
project=serialiseDict(Project21Database.find_one(settings.DB_COLLECTION_PROJECT,{"projectID":projectID}))
except:
return ErrorResponseModel("An Error Occured",404,"Project could not be found")
return project
def start_auto_preprocessing_and_training(autoFormData: AutoFormData):
autoFormData = dict(autoFormData)
projectAutoConfigFileLocation, dataID, problem_type = generate_project_auto_config_file(
autoFormData["projectID"], currentIDs, autoFormData, Project21Database)
resultsCache.set_auto_mode_status(False)
if (problem_type == 'regression'):
automatic_model_training = AutoReg()
Operation = automatic_model_training.auto(
projectAutoConfigFileLocation)
elif (problem_type == 'classification'):
automatic_model_training = Auto()
Operation = automatic_model_training.auto(
projectAutoConfigFileLocation)
elif (problem_type == 'clustering'):
automatic_model_training = Autoclu()
Operation = automatic_model_training.auto(
projectAutoConfigFileLocation)
if Operation["Successful"]:
try:
Project21Database.insert_one(
settings.DB_COLLECTION_DATA, {
"dataID": dataID,
"cleanDataPath": Operation["cleanDataPath"],
"target": autoFormData["target"],
"belongsToUserID": currentIDs.get_current_user_id(),
"belongsToProjectID": autoFormData["projectID"]
})
currentIDs.set_current_data_id(dataID)
Project21Database.insert_one(
settings.DB_COLLECTION_MODEL, {
"modelID": dataID,
"modelName": "Default Name",
"modelType": problem_type,
"pickleFolderPath": Operation["pickleFolderPath"],
"pickleFilePath": Operation["pickleFilePath"],
"belongsToUserID": autoFormData["userID"],
"belongsToProjectID": autoFormData["projectID"],
"belongsToDataID": dataID
})
Project21Database.insert_one(
settings.DB_COLLECTION_METRICS, {
"belongsToUserID": autoFormData["userID"],
"belongsToProjectID": autoFormData["projectID"],
"belongsToModelID": dataID,
"addressOfMetricsFile": Operation["metricsLocation"]
})
result = Project21Database.find_one(
settings.DB_COLLECTION_PROJECT,
{"projectID": autoFormData["projectID"]})
result = serialiseDict(result)
if result is not None:
if result["listOfDataIDs"] is not None:
newListOfDataIDs = result["listOfDataIDs"]
newListOfDataIDs.append(dataID)
Project21Database.update_one(
settings.DB_COLLECTION_PROJECT,
{"projectID": result["projectID"]}, {
"$set": {
"listOfDataIDs": newListOfDataIDs,
"configFileLocation":
projectAutoConfigFileLocation,
"isAuto": autoFormData["isauto"],
"target": autoFormData["target"]
}
})
else:
Project21Database.update_one(
settings.DB_COLLECTION_PROJECT,
{"projectID": result["projectID"]}, {
"$set": {
"listOfDataIDs": [dataID],
"configFileLocation":
projectAutoConfigFileLocation,
"isAuto": autoFormData["isauto"],
"target": autoFormData["target"]
}
})
if (problem_type == 'clustering'):
Project21Database.update_one(
settings.DB_COLLECTION_PROJECT,
{"projectID": result["projectID"]}, {
"$set": {
"clusterPlotLocation":
Operation["clusterPlotLocation"]
}
})
except Exception as e:
print("An Error occured: ", e)
return JSONResponse({
"Auto": "Success",
"Database Insertion": "Failure",
"Project Collection Updation": "Unsuccessful"
})
resultsCache.set_clean_data_path(Operation["cleanDataPath"])
resultsCache.set_metrics_path(Operation["metricsLocation"])
resultsCache.set_pickle_file_path(Operation["pickleFilePath"])
resultsCache.set_pickle_folder_path(Operation["pickleFolderPath"])
resultsCache.set_auto_mode_status(True)
return JSONResponse({
"Successful": "True",
"userID": currentIDs.get_current_user_id(),
"projectID": autoFormData["projectID"],
"dataID": dataID,
"modelID": dataID
})
else:
return JSONResponse({"Successful": "False"})
def usersEntity(entity) -> list:
return [serialiseDict(item) for item in entity]
def create_project(projectName: str = Form(...),
mtype: str = Form(...),
train: UploadFile = File(...)):
inserted_projectID = 0
Operation = generate_project_folder(projectName, train)
if Operation["Success"]:
try:
inserted_projectID = create_project_id(Project21Database)
# inserted_modelID=create_model_id(Project21Database)
currentIDs.set_current_project_id(inserted_projectID)
# currentIDs.set_current_model_id(inserted_modelID)
resultsCache.set_project_folder_path(
Operation["ProjectFolderPath"])
Project21Database.insert_one(
settings.DB_COLLECTION_PROJECT, {
"projectID": inserted_projectID,
"projectName": projectName,
"rawDataPath": Operation["RawDataPath"],
"projectFolderPath": Operation["ProjectFolderPath"],
"belongsToUserID": currentIDs.get_current_user_id(),
"listOfDataIDs": [],
"configFileLocation": None,
"plotsPath": None,
"projectType": mtype,
"target": None
})
# Project21Database.insert_one(settings.DB_COLLECTION_MODEL,{
# "modelID": inserted_modelID,
# "modelName": "Default Model",
# "modelType": mtype,
# "belongsToUserID": currentIDs.get_current_user_id(),
# "belongsToProjectID": inserted_projectID
# })
try:
result = Project21Database.find_one(
settings.DB_COLLECTION_USER,
{"userID": currentIDs.get_current_user_id()})
if result is not None:
result = serialiseDict(result)
if result["listOfProjects"] is not None:
newListOfProjects = result["listOfProjects"]
newListOfProjects.append(inserted_projectID)
Project21Database.update_one(
settings.DB_COLLECTION_USER,
{"userID": result["userID"]},
{"$set": {
"listOfProjects": newListOfProjects
}})
else:
Project21Database.update_one(
settings.DB_COLLECTION_USER,
{"userID": result["userID"]},
{"$set": {
"listOfProjects": [inserted_projectID]
}})
except Exception as e:
print("An Error occured: ", e)
return JSONResponse({
"File Received": "Success",
"Project Folder": "Success",
"Database Update": "Partially Successful"
})
except Exception as e:
print("An Error occured: ", e)
return JSONResponse({
"File Received": "Success",
"Project Folder": "Success",
"Database Update": "Failure"
})
return JSONResponse({
"userID": currentIDs.get_current_user_id(),
"projectID": inserted_projectID
})
else:
return JSONResponse(Operation["Error"])
def timeseries_training(timeseriesFormData: TimeseriesFormData):
print(timeseriesFormData)
timeseriesFormData = dict(timeseriesFormData)
projectConfigFileLocation, projectFolderPath, dataID, projectType = generate_project_timeseries_config_file(
timeseriesFormData["projectID"], currentIDs, timeseriesFormData,
Project21Database)
timeseriesPreprocessObj = TimeseriesPreprocess()
cleanDataPath = timeseriesPreprocessObj.preprocess(
projectConfigFileLocation, projectFolderPath)
try:
Project21Database.insert_one(
settings.DB_COLLECTION_DATA, {
"dataID": dataID,
"cleanDataPath": cleanDataPath,
"target": timeseriesFormData["target"],
"belongsToUserID": timeseriesFormData["userID"],
"belongsToProjectID": timeseriesFormData["projectID"]
})
except Exception as e:
print("Could not insert into Data Collection. An Error Occured: ", e)
timeseriesObj = timeseries()
Operation = timeseriesObj.createarima(projectConfigFileLocation)
if Operation["Successful"]:
try:
Project21Database.insert_one(
settings.DB_COLLECTION_MODEL, {
"modelID": dataID,
"modelName": "Default Name",
"modelType": "timeseries",
"pickleFolderPath": Operation["pickleFolderPath"],
"pickleFilePath": Operation["pickleFilePath"],
"belongsToUserID": timeseriesFormData["userID"],
"belongsToProjectID": timeseriesFormData["projectID"],
"belongsToDataID": dataID
})
except Exception as e:
print(
"Could not insert into Model Collection. An Error Occurred: ",
e)
try:
Project21Database.insert_one(
settings.DB_COLLECTION_METRICS, {
"belongsToUserID": timeseriesFormData["userID"],
"belongsToProjectID": timeseriesFormData["projectID"],
"belongsToModelID": dataID,
"addressOfMetricsFile": Operation["metricsLocation"]
})
except Exception as e:
print(
"Could not insert into Metrics Collection. An Error Occured: ",
e)
try:
result = Project21Database.find_one(
settings.DB_COLLECTION_PROJECT,
{"projectID": timeseriesFormData["projectID"]})
result = serialiseDict(result)
if result is not None:
if result["listOfDataIDs"] is not None:
newListOfDataIDs = result["listOfDataIDs"]
newListOfDataIDs.append(dataID)
Project21Database.update_one(
settings.DB_COLLECTION_PROJECT,
{"projectID": result["projectID"]}, {
"$set": {
"listOfDataIDs": newListOfDataIDs,
"configFileLocation":
projectConfigFileLocation,
"isAuto": False,
"target": timeseriesFormData["target"]
}
})
else:
Project21Database.update_one(
settings.DB_COLLECTION_PROJECT,
{"projectID": result["projectID"]}, {
"$set": {
"listOfDataIDs": [dataID],
"configFileLocation":
projectConfigFileLocation,
"isAuto": False,
"target": timeseriesFormData["target"]
}
})
if (projectType == 'timeseries'):
Project21Database.update_one(
settings.DB_COLLECTION_PROJECT,
{"projectID": result["projectID"]},
{"$set": {
"plotLocation": Operation["plotLocation"]
}})
except Exception as e:
print("An Error Occured: ", e)
return JSONResponse({
"Successful": "True",
"userID": currentIDs.get_current_user_id(),
"projectID": timeseriesFormData["projectID"],
"dataID": dataID,
"modelID": dataID
})
# @app.websocket("/ws")
# async def training_status(websocket: WebSocket):
# print("Connecting to the Frontend...")
# await websocket.accept()
# # while (not resultsCache.get_auto_mode_status()):
# try:
# data={
# "Successful":"False",
# "Status": "Model Running"
# }
# if (resultsCache.get_auto_mode_status()):
# data={
# "Successful":"True",
# "Status": "Model Successfully Created",
# "userID": currentIDs.get_current_user_id(),
# "projectID": currentIDs.get_current_project_id(),
# "dataID":currentIDs.get_current_data_id(),
# "modelID": currentIDs.get_current_model_id()
# }
# await websocket.send_json(data)
# data2= await websocket.receive_text() #Can be used to receive data from frontend
# print(data2)
# await websocket.send_json(data) #Can be used to return data to the frontend
# except Exception as e:
# print("Error: ",e)
# # break
# print("Websocket connection closing...")
