def get_inference_results(projectID: int = Form(...), modelID: int = Form(...), inferenceDataFile: UploadFile = File(...)): newDataPath = '/' pickleFilePath = '/' path = '/' inferenceDataResultsPath = '/' isAuto = False try: result = Project21Database.find_one(settings.DB_COLLECTION_PROJECT, {"projectID": projectID}) if result is not None: result = serialiseDict(result) isAuto = result["isAuto"] except Exception as e: print("An Error Occured: ", e) print("Could not find the project in the Project Collection") try: result = Project21Database.find_one(settings.DB_COLLECTION_MODEL, { "modelID": modelID, "belongsToProjectID": projectID }) if result is not None: result = serialiseDict(result) if result["pickleFilePath"] is not None: pickleFilePath = result["pickleFilePath"] if result["pickleFolderPath"] is not None: projectRunPath = os.path.join(result["pickleFolderPath"], os.pardir) path = os.path.join(projectRunPath, "inference_data") if (not os.path.exists(path)): os.makedirs(path) newDataPath = os.path.join(path, 'inference_data.csv') with open(newDataPath, "wb") as buffer: shutil.copyfileobj(inferenceDataFile.file, buffer) inference = Inference() inferenceDataResultsPath = inference.inference( pickleFilePath, newDataPath, path, isAuto) Project21Database.insert_one( settings.DB_COLLECTION_INFERENCE, { "newData": newDataPath, "results": inferenceDataResultsPath, "belongsToUserID": currentIDs.get_current_user_id(), "belongsToProjectID": projectID, "belongsToModelID": modelID }) if os.path.exists(inferenceDataResultsPath): print({"Metrics Generation": "Successful"}) return FileResponse(inferenceDataResultsPath, media_type="text/csv", filename="inference.csv") except Exception as e: print("An error occured: ", e) print("Unable to find model from model Collection") return JSONResponse({"Metrics Generation": "Failed"})
def generate_project_manual_config_file(projectID, preprocessJSONFormData, Project21Database): """ """ # user_yaml=yaml.load(open(settings.CONFIG_PREPROCESS_YAML_FILE),Loader=SafeLoader) random_id = generate_random_id() preprocessJSONFormData["id"] = random_id preprocessJSONFormData["raw_data_address"] = get_raw_data_path( projectID, Project21Database) location = "/" random_id = generate_random_id() try: result_project = Project21Database.find_one( settings.DB_COLLECTION_PROJECT, {"projectID": projectID}) result_project = serialiseDict(result_project) if result_project is not None: location = os.path.join(result_project["projectFolderPath"], 'run' + str(random_id)) except: print("Unable to Update User's Project's Config File") if (not os.path.exists(location)): os.makedirs(location) preprocessJSONFormData["location"] = location with open(os.path.join(location, "preprocess_config.yaml"), "w") as f: yaml.dump(preprocessJSONFormData, f) f.close() return os.path.join(location, 'preprocess_config.yaml' ), random_id, result_project["projectType"], location
def generate_project_timeseries_rf_config_file(projectID,currentIDs,timeseriesFormData,Project21Database): user_yaml=yaml.load(open(settings.CONFIG_TIMESERIES_MANUAL_FILE),Loader=SafeLoader) random_id=generate_random_id() user_yaml["id"]=random_id user_yaml["raw_data_address"]=get_raw_data_path(projectID,Project21Database) user_yaml["target_column_name"]=timeseriesFormData["target"] user_yaml["date_index"]=timeseriesFormData["dateColumn"] user_yaml["frequency"]=timeseriesFormData["frequency"] try: result_project=Project21Database.find_one(settings.DB_COLLECTION_PROJECT,{"projectID":projectID}) result_project=serialiseDict(result_project) if result_project is not None: user_yaml["location"]=os.path.join(result_project["projectFolderPath"],'run'+str(random_id)) user_yaml["experimentname"]=result_project["projectName"] else: user_yaml["location"]='/' user_yaml["experimentname"]='default' except Exception as e: print("Unable to Update User's Project's Config File. An Error Occured: ",e) if(not os.path.exists(user_yaml["location"])): os.makedirs(user_yaml["location"]) with open(os.path.join(user_yaml["location"],"preprocess_config.yaml"), "w") as f: yaml.dump(user_yaml,f) f.close() return os.path.join(user_yaml["location"],'preprocess_config.yaml'), user_yaml["location"],random_id, result_project["projectType"]
def get_one_data(dataID: int): try: data = serialiseDict( Project21Database.find_one(settings.DB_COLLECTION_DATA, {"dataID": dataID})) except: return ErrorResponseModel("An Error Occured", 404, "Data could not be found") return data
def get_one_model(modelID: int): try: model = serialiseDict( Project21Database.find_one(settings.DB_COLLECTION_MODEL, {"modelID": modelID})) except: return ErrorResponseModel("An Error Occured", 404, "Model could not be found") return model
def generate_project_auto_config_file(projectID,currentIDs,formData,Project21Database): """ Returns the auto config file generated for the project ... Parameters ---------- projectID: int currentIDs: obj formData: obj Project21Databse: obj Returns ------- tuple: path, randomID, problemType """ user_yaml=yaml.load(open(settings.CONFIG_AUTO_YAML_FILE),Loader=SafeLoader) random_id=generate_random_id() user_yaml["id"]=random_id user_yaml["raw_data_address"]=get_raw_data_path(projectID,Project21Database) user_yaml["target_col_name"]=formData["target"] user_yaml["na_value"]=formData["nulltype"] user_yaml["n"]=formData["modelnumber"] user_yaml["problem_type"]=get_project_type(projectID,Project21Database) if(user_yaml["problem_type"]=='clustering'): user_yaml["clusteringType"]=formData["clusteringType"] user_yaml["numClusters"]=formData["numClusters"] # try: # result_model=Project21Database.find_one(settings.DB_COLLECTION_MODEL,{"modelID":currentIDs.get_current_model_id()}) # result_model=serialiseDict(result_model) # if result_model is not None: # user_yaml["problem_type"]=result_model["modelType"] # else: # user_yaml["problem_type"]='default' # except: # print("Unable to Update User's Project's AutoConfig File") try: result_project=Project21Database.find_one(settings.DB_COLLECTION_PROJECT,{"projectID":projectID}) result_project=serialiseDict(result_project) if result_project is not None: user_yaml["location"]=os.path.join(result_project["projectFolderPath"],'run'+str(random_id)) user_yaml["experimentname"]=result_project["projectName"] else: user_yaml["location"]='/' user_yaml["experimentname"]='default' except: print("Unable to Update User's Project's Config File") if(not os.path.exists(user_yaml["location"])): os.makedirs(user_yaml["location"]) with open(os.path.join(user_yaml["location"],"autoConfig.yaml"), "w") as f: yaml.dump(user_yaml,f) f.close() return os.path.join(user_yaml["location"],'autoConfig.yaml'), random_id , user_yaml["problem_type"]
def get_metrics(projectID, modelID, Project21Database): try: result = Project21Database.find_one(settings.DB_COLLECTION_METRICS, { "belongsToProjectID": projectID, "belongsToModelID": modelID }) if result is not None: result = serialiseDict(result) if result["addressOfMetricsFile"] is not None: return str(result["addressOfMetricsFile"]) except Exception as e: print("An Error Occured: ", e) return 'Metrics not found'
def get_clean_data_path(dataID, Project21Database): try: result = Project21Database.find_one(settings.DB_COLLECTION_DATA, {"dataID": dataID}) result = serialiseDict(result) if result is not None: return result["cleanDataPath"] else: return '/' except: print( "An error occured while retreiving cleanDataPath from Data Collection" )
def get_pickle_file_path(modelID,Project21Database): try: result=Project21Database.find_one(settings.DB_COLLECTION_MODEL,{"modelID":modelID}) if result is not None: result=serialiseDict(result) return result["pickleFilePath"] # if result["pickleFilePath"] is not None: # return result["pickleFilePath"] else: print("result is none") return '/' except Exception as e: print("An error occured while retreiving pickleFilePath from the Model Collection") print("Error: ",e) return '/'
def get_raw_data_path(projectID, Project21Database): try: result = Project21Database.find_one(settings.DB_COLLECTION_PROJECT, {"projectID": projectID}) result = serialiseDict(result) if result is not None: return result["rawDataPath"] #path string returned else: return '' except Exception as e: print("An Error Occured: ", e) print( "An Error Occured While Retreiving rawDataPath from the Project Collection" ) return ''
def get_project_type(projectID, Project21Database): try: result = Project21Database.find_one(settings.DB_COLLECTION_PROJECT, {"projectID": projectID}) result = serialiseDict(result) if result is not None: return result["projectType"] else: return '' except Exception as e: print("An Error Occured: ", e) print( "An Error Occured while retreiving projectType from the Project Collection" ) return ''
def get_all_project_details(userID: int): listOfProjects = [] try: results = Project21Database.find(settings.DB_COLLECTION_PROJECT, {"belongsToUserID": userID}) for result in results: result = serialiseDict(result) if result["target"] is not None: projectTemplate = { "projectID": result["projectID"], "projectName": result["projectName"], "target": result["target"], "modelType": result["projectType"], "listOfDataIDs": result["listOfDataIDs"], "isAuto": result["isAuto"] } listOfProjects.append(projectTemplate) except Exception as e: print("An Error Occured: ", e) print("Unable to get all projects") return JSONResponse({"GetAllProjects": "Failed"}) return listOfProjects
def get_plots( projectID: int): #check if it already exists - change location address try: result = Project21Database.find_one(settings.DB_COLLECTION_PROJECT, {"projectID": projectID}) if result is not None: result = serialiseDict(result) if (result["projectType"] == 'clustering'): return FileResponse(result["clusterPlotLocation"], media_type="text/html", filename="plot.html") if (result["projectType"] == 'timeseries'): return FileResponse(result["plotLocation"], media_type="text/html", filename="plot.html") if result["configFileLocation"] is not None: plotFilePath = plot( result["configFileLocation"] ) #plot function requires the auto config file try: Project21Database.update_one( settings.DB_COLLECTION_PROJECT, {"projectID": projectID}, {"$set": { "plotsPath": plotFilePath }}) except Exception as e: print( "An Error occured while storing the plot path into the project collection" ) return FileResponse(plotFilePath, media_type='text/html', filename='plot.html') except Exception as e: print("An Error Occured: ", e) return JSONResponse({"Plots": "Not generated"})
def get_one_metrics(belongsToModelID:int): try: metrics=serialiseDict(Project21Database.find_one(settings.DB_COLLECTION_METRICS,{"belongsToModelID":belongsToModelID})) except: return ErrorResponseModel("An Error Occured",404,"Metrics could not be found") return metrics
def get_one_project(projectID:int): try: project=serialiseDict(Project21Database.find_one(settings.DB_COLLECTION_PROJECT,{"projectID":projectID})) except: return ErrorResponseModel("An Error Occured",404,"Project could not be found") return project
def start_auto_preprocessing_and_training(autoFormData: AutoFormData): autoFormData = dict(autoFormData) projectAutoConfigFileLocation, dataID, problem_type = generate_project_auto_config_file( autoFormData["projectID"], currentIDs, autoFormData, Project21Database) resultsCache.set_auto_mode_status(False) if (problem_type == 'regression'): automatic_model_training = AutoReg() Operation = automatic_model_training.auto( projectAutoConfigFileLocation) elif (problem_type == 'classification'): automatic_model_training = Auto() Operation = automatic_model_training.auto( projectAutoConfigFileLocation) elif (problem_type == 'clustering'): automatic_model_training = Autoclu() Operation = automatic_model_training.auto( projectAutoConfigFileLocation) if Operation["Successful"]: try: Project21Database.insert_one( settings.DB_COLLECTION_DATA, { "dataID": dataID, "cleanDataPath": Operation["cleanDataPath"], "target": autoFormData["target"], "belongsToUserID": currentIDs.get_current_user_id(), "belongsToProjectID": autoFormData["projectID"] }) currentIDs.set_current_data_id(dataID) Project21Database.insert_one( settings.DB_COLLECTION_MODEL, { "modelID": dataID, "modelName": "Default Name", "modelType": problem_type, "pickleFolderPath": Operation["pickleFolderPath"], "pickleFilePath": Operation["pickleFilePath"], "belongsToUserID": autoFormData["userID"], "belongsToProjectID": autoFormData["projectID"], "belongsToDataID": dataID }) Project21Database.insert_one( settings.DB_COLLECTION_METRICS, { "belongsToUserID": autoFormData["userID"], "belongsToProjectID": autoFormData["projectID"], "belongsToModelID": dataID, "addressOfMetricsFile": Operation["metricsLocation"] }) result = Project21Database.find_one( settings.DB_COLLECTION_PROJECT, {"projectID": autoFormData["projectID"]}) result = serialiseDict(result) if result is not None: if result["listOfDataIDs"] is not None: newListOfDataIDs = result["listOfDataIDs"] newListOfDataIDs.append(dataID) Project21Database.update_one( settings.DB_COLLECTION_PROJECT, {"projectID": result["projectID"]}, { "$set": { "listOfDataIDs": newListOfDataIDs, "configFileLocation": projectAutoConfigFileLocation, "isAuto": autoFormData["isauto"], "target": autoFormData["target"] } }) else: Project21Database.update_one( settings.DB_COLLECTION_PROJECT, {"projectID": result["projectID"]}, { "$set": { "listOfDataIDs": [dataID], "configFileLocation": projectAutoConfigFileLocation, "isAuto": autoFormData["isauto"], "target": autoFormData["target"] } }) if (problem_type == 'clustering'): Project21Database.update_one( settings.DB_COLLECTION_PROJECT, {"projectID": result["projectID"]}, { "$set": { "clusterPlotLocation": Operation["clusterPlotLocation"] } }) except Exception as e: print("An Error occured: ", e) return JSONResponse({ "Auto": "Success", "Database Insertion": "Failure", "Project Collection Updation": "Unsuccessful" }) resultsCache.set_clean_data_path(Operation["cleanDataPath"]) resultsCache.set_metrics_path(Operation["metricsLocation"]) resultsCache.set_pickle_file_path(Operation["pickleFilePath"]) resultsCache.set_pickle_folder_path(Operation["pickleFolderPath"]) resultsCache.set_auto_mode_status(True) return JSONResponse({ "Successful": "True", "userID": currentIDs.get_current_user_id(), "projectID": autoFormData["projectID"], "dataID": dataID, "modelID": dataID }) else: return JSONResponse({"Successful": "False"})
def usersEntity(entity) -> list: return [serialiseDict(item) for item in entity]
def create_project(projectName: str = Form(...), mtype: str = Form(...), train: UploadFile = File(...)): inserted_projectID = 0 Operation = generate_project_folder(projectName, train) if Operation["Success"]: try: inserted_projectID = create_project_id(Project21Database) # inserted_modelID=create_model_id(Project21Database) currentIDs.set_current_project_id(inserted_projectID) # currentIDs.set_current_model_id(inserted_modelID) resultsCache.set_project_folder_path( Operation["ProjectFolderPath"]) Project21Database.insert_one( settings.DB_COLLECTION_PROJECT, { "projectID": inserted_projectID, "projectName": projectName, "rawDataPath": Operation["RawDataPath"], "projectFolderPath": Operation["ProjectFolderPath"], "belongsToUserID": currentIDs.get_current_user_id(), "listOfDataIDs": [], "configFileLocation": None, "plotsPath": None, "projectType": mtype, "target": None }) # Project21Database.insert_one(settings.DB_COLLECTION_MODEL,{ # "modelID": inserted_modelID, # "modelName": "Default Model", # "modelType": mtype, # "belongsToUserID": currentIDs.get_current_user_id(), # "belongsToProjectID": inserted_projectID # }) try: result = Project21Database.find_one( settings.DB_COLLECTION_USER, {"userID": currentIDs.get_current_user_id()}) if result is not None: result = serialiseDict(result) if result["listOfProjects"] is not None: newListOfProjects = result["listOfProjects"] newListOfProjects.append(inserted_projectID) Project21Database.update_one( settings.DB_COLLECTION_USER, {"userID": result["userID"]}, {"$set": { "listOfProjects": newListOfProjects }}) else: Project21Database.update_one( settings.DB_COLLECTION_USER, {"userID": result["userID"]}, {"$set": { "listOfProjects": [inserted_projectID] }}) except Exception as e: print("An Error occured: ", e) return JSONResponse({ "File Received": "Success", "Project Folder": "Success", "Database Update": "Partially Successful" }) except Exception as e: print("An Error occured: ", e) return JSONResponse({ "File Received": "Success", "Project Folder": "Success", "Database Update": "Failure" }) return JSONResponse({ "userID": currentIDs.get_current_user_id(), "projectID": inserted_projectID }) else: return JSONResponse(Operation["Error"])
def timeseries_training(timeseriesFormData: TimeseriesFormData): print(timeseriesFormData) timeseriesFormData = dict(timeseriesFormData) projectConfigFileLocation, projectFolderPath, dataID, projectType = generate_project_timeseries_config_file( timeseriesFormData["projectID"], currentIDs, timeseriesFormData, Project21Database) timeseriesPreprocessObj = TimeseriesPreprocess() cleanDataPath = timeseriesPreprocessObj.preprocess( projectConfigFileLocation, projectFolderPath) try: Project21Database.insert_one( settings.DB_COLLECTION_DATA, { "dataID": dataID, "cleanDataPath": cleanDataPath, "target": timeseriesFormData["target"], "belongsToUserID": timeseriesFormData["userID"], "belongsToProjectID": timeseriesFormData["projectID"] }) except Exception as e: print("Could not insert into Data Collection. An Error Occured: ", e) timeseriesObj = timeseries() Operation = timeseriesObj.createarima(projectConfigFileLocation) if Operation["Successful"]: try: Project21Database.insert_one( settings.DB_COLLECTION_MODEL, { "modelID": dataID, "modelName": "Default Name", "modelType": "timeseries", "pickleFolderPath": Operation["pickleFolderPath"], "pickleFilePath": Operation["pickleFilePath"], "belongsToUserID": timeseriesFormData["userID"], "belongsToProjectID": timeseriesFormData["projectID"], "belongsToDataID": dataID }) except Exception as e: print( "Could not insert into Model Collection. An Error Occurred: ", e) try: Project21Database.insert_one( settings.DB_COLLECTION_METRICS, { "belongsToUserID": timeseriesFormData["userID"], "belongsToProjectID": timeseriesFormData["projectID"], "belongsToModelID": dataID, "addressOfMetricsFile": Operation["metricsLocation"] }) except Exception as e: print( "Could not insert into Metrics Collection. An Error Occured: ", e) try: result = Project21Database.find_one( settings.DB_COLLECTION_PROJECT, {"projectID": timeseriesFormData["projectID"]}) result = serialiseDict(result) if result is not None: if result["listOfDataIDs"] is not None: newListOfDataIDs = result["listOfDataIDs"] newListOfDataIDs.append(dataID) Project21Database.update_one( settings.DB_COLLECTION_PROJECT, {"projectID": result["projectID"]}, { "$set": { "listOfDataIDs": newListOfDataIDs, "configFileLocation": projectConfigFileLocation, "isAuto": False, "target": timeseriesFormData["target"] } }) else: Project21Database.update_one( settings.DB_COLLECTION_PROJECT, {"projectID": result["projectID"]}, { "$set": { "listOfDataIDs": [dataID], "configFileLocation": projectConfigFileLocation, "isAuto": False, "target": timeseriesFormData["target"] } }) if (projectType == 'timeseries'): Project21Database.update_one( settings.DB_COLLECTION_PROJECT, {"projectID": result["projectID"]}, {"$set": { "plotLocation": Operation["plotLocation"] }}) except Exception as e: print("An Error Occured: ", e) return JSONResponse({ "Successful": "True", "userID": currentIDs.get_current_user_id(), "projectID": timeseriesFormData["projectID"], "dataID": dataID, "modelID": dataID }) # @app.websocket("/ws") # async def training_status(websocket: WebSocket): # print("Connecting to the Frontend...") # await websocket.accept() # # while (not resultsCache.get_auto_mode_status()): # try: # data={ # "Successful":"False", # "Status": "Model Running" # } # if (resultsCache.get_auto_mode_status()): # data={ # "Successful":"True", # "Status": "Model Successfully Created", # "userID": currentIDs.get_current_user_id(), # "projectID": currentIDs.get_current_project_id(), # "dataID":currentIDs.get_current_data_id(), # "modelID": currentIDs.get_current_model_id() # } # await websocket.send_json(data) # data2= await websocket.receive_text() #Can be used to receive data from frontend # print(data2) # await websocket.send_json(data) #Can be used to return data to the frontend # except Exception as e: # print("Error: ",e) # # break # print("Websocket connection closing...")