def getFile(fileID): # declare as global to update the original variable. global REQUEST_ID # generate a new id for each request. REQUEST_ID = str(uuid.uuid4()) accessID = getAccessId() mfileID = fileID if isAccessIdAuthentic(accessID): fileName = dc.translateFileidToFilename(mfileID) userFilesDir = dc.getUserFilesDir(accessID) print("INFO: FILEDOWNLOAD: Requested File Name: " + str(fileName)) if fileName != False: try: md5Checksum = generateMD5(os.path.join(userFilesDir, fileName)) print( "INFO: FILEDOWNLOAD: File with the given FileID found on server." ) status = "File with the given FileID found on server." httpCode = 200 res = createRequestDetails(fileName, mfileID, md5Checksum, status, httpCode, REQUEST_ID) return send_from_directory(userFilesDir, filename=fileName, as_attachment=True) except Exception as e: print( "ERROR: FILEDOWNLOAD: Error(s) encountered while transferring file." ) print(str(e)) status = "Error(s) encountered while transferring file." httpCode = 500 res = createRequestDetails("N/A", mfileID, "N/A", status, httpCode, REQUEST_ID) else: print( "ERROR: FILEDOWNLOAD: File with the given FileID parameter not found" ) status = "File with the given FileID parameter not found" httpCode = 404 res = createRequestDetails("N/A", mfileID, "N/A", status, httpCode, REQUEST_ID) else: print("ERROR: FILEDOWNLOAD: Invalid Access-ID, request denied.") status = "[ERR] Invalid Access-ID, request denied." httpCode = 403 res = createRequestDetails("N/A", "N/A", "N/A", status, httpCode, REQUEST_ID) res = make_response(res, httpCode) return res ############################ END OF SCRIPT ###################################
def checkFileStatus(fileID): # declare as global to update the original variable. global REQUEST_ID # generate a new id for each request. REQUEST_ID = str(uuid.uuid4()) accessID = getAccessId() if isAccessIdAuthentic(accessID): mfileID = fileID userWorkspace = os.path.join(WORKSPACE_DIR, accessID) userFilesDir = os.path.join(userWorkspace, USER_FILES) fileName = dc.translateFileidToFilename(mfileID) if fileName != False: md5Checksum = generateMD5(os.path.join(userFilesDir, fileName)) status = "File with the given FileID found on server." httpCode = 200 res = createRequestDetails(fileName, mfileID, md5Checksum, status, httpCode, REQUEST_ID) return res else: print("File with the given FileID parameter not found") status = "File with the given FileID parameter not found" httpCode = 404 res = createRequestDetails("N/A", mfileID, "N/A", status, httpCode, REQUEST_ID) return res else: print("Invalid Access-ID, request denied.") status = "[ERR] Invalid Access-ID, request denied." httpCode = 403 res = createRequestDetails("N/A", "N/A", "N/A", status, httpCode, REQUEST_ID) res = make_response(res, httpCode) return res
def trainModel(accessID, workflowID, inputFileID): isInitSuccessful() REQUEST_ID = str(uuid.uuid4()) inputFileName = dc.translateFileidToFilename(inputFileID) inputFilePath = dc.getArchiveUploadPath((accessID)) if inputFileName != False: print("INFO: HyAPI_ML_INTERNAL: FileName: " + inputFileName) inputFile = os.path.join(inputFilePath, inputFileName) else: print( "ERROR: HyAPI_ML_INTERNAL: Input image with the given FileID parameter not found" ) return False # Get the path to the user's workspace and the specified workflow: # STEP-1: EXTRACT THE INPUT ZIP TO THE TEMP DIRECTORY. tempDirectory = MLDC.getUserTF_tmp_dir(accessID, workflowID) if extractZip(inputFile, tempDirectory): print("INFO: HyAPI_ML_INTERNAL: Inputs extracted to temp directory.") else: print( "ERROR: HyAPI_ML_INTERNAL: Error extracting input .zip file to workflow." ) return False #STEP-2: MOVE THE FILES TO THE RELEVANT DIRECTORIES IN THE WORKFLOW STRUCTURE. userWorkflow = MLDC.getUserTF_workflow(accessID, workflowID) workflow_Dataset = MLDC.getDatasetDirectory(accessID, workflowID) workflow_Annotations = MLDC.getAnnotationsDirectory(accessID, workflowID) workflow_Training = MLDC.getTrainingDirectory(accessID, workflowID) try: testData_tmp = os.path.join(tempDirectory, MLC.USR_INPUT_TEST_IMAGES_DIR) testData_final = os.path.join(workflow_Dataset, MLC.USR_INPUT_TEST_IMAGES_DIR) trainData_tmp = os.path.join(tempDirectory, MLC.USR_INPUT_TRAIN_IMAGES_DIR) trainData_final = os.path.join(workflow_Dataset, MLC.USR_INPUT_TRAIN_IMAGES_DIR) labelMap_tmp = os.path.join(tempDirectory, MLC.USR_INPUT_LABEL_MAP_FILE) labelMap_final = os.path.join(workflow_Annotations, MLC.USR_INPUT_LABEL_MAP_FILE) pipeline_tmp = os.path.join(tempDirectory, MLC.USR_INPUT_PIPELINE_FILE) pipeline_final = os.path.join(workflow_Training, MLC.USR_INPUT_PIPELINE_FILE) # (source,destination) if fh.moveFile(testData_tmp, testData_final) and fh.moveFile( trainData_tmp, trainData_final) and fh.moveFile( labelMap_tmp, labelMap_final) and fh.moveFile( pipeline_tmp, pipeline_final): print("INFO: HyAPI_ML_INTERNAL:Files moved successfully") else: print("ERROR: HyAPI_ML_INTERNAL: Error moving files.") return False except Exception as e: print(str(e)) return False #STEP-3-a: GENERATE UNIFIED CSV FILES FOR TEST AND TRAIN DATA. train_csv_path = os.path.join(workflow_Annotations, MLC.USR_OUTPUT_TRAIN_CSV) test_csv_path = os.path.join(workflow_Annotations, MLC.USR_OUTPUT_TEST_CSV) try: xml_to_csv.generateCsvFromXml(trainData_final, train_csv_path) xml_to_csv.generateCsvFromXml(testData_final, test_csv_path) print( "INFO: HyAPI_ML_INTERNAL: Dataset XMLs converted to CSV files successfully." ) except Exception as e: print(str(e)) return False #STEP-3-b: GENERATE TF-RECORDS FILES FOR TEST AND TRAIN DATA train_records_path = os.path.join(workflow_Annotations, MLC.USR_OUTPUT_TRAIN_RECORD) test_records_path = os.path.join(workflow_Annotations, MLC.USR_OUTPUT_TEST_RECORD) try: generate_tfrecord.generateTfRecords(train_records_path, trainData_final, train_csv_path) generate_tfrecord.generateTfRecords(test_records_path, testData_final, test_csv_path) print( "INFO: HyAPI_ML_INTERNAL: TF-Records file generated successfully.") except Exception as e: print(str(e)) return False #STEP-4: START THE TRAINING JOB try: containerID = DockerInterface.startModelTraining( REQUEST_ID, userWorkflow) if containerID != False: print( "INFO: HyAPI_ML_INTERNAL: Training process started, Container ID: " + str(containerID)) status = "Training started." if MLDC.updateWorkflow(accessID, workflowID, status, containerID): return True else: print( "ERROR: HyAPI_ML_INTERNAL: Error saving workflow details to db." ) return False else: print( "ERROR: HyAPI_ML_INTERNAL: Error(s) encountered while starting the training job." ) return False except Exception as e: print(str(e)) return False
def validateCompuMethods(): # declare as global to update the original variable. global REQUEST_ID # generate a new id for each request. REQUEST_ID = str(uuid.uuid4()) accessID = getAccessId() if isAccessIdAuthentic(accessID): #########Get the File IDs from the request parameters######### inputFileID = request.json userFilesDir = dc.getUserFilesDir(accessID) try: #Get the File IDs from the request parameters xl_FileID = inputFileID["xlFileID"] except Exception as e: print("ERROR: CM_Validation: " + str(e)) xl_FileID = "x" try: a2l_FileID = inputFileID["a2lFileID"] except Exception as e: print("ERROR: CM_Validation: " + str(e)) a2l_FileID = "x" #########File IDs to FileName translation######### fileName = dc.translateFileidToFilename(xl_FileID) if fileName != False: UPLOAD_XSLX_PATH = dc.getExcelUploadPath(accessID) xl_file = os.path.join(UPLOAD_XSLX_PATH, fileName) print(xl_file) else: print( "ERROR: CM_Validation: Excel file with the given FileID parameter not found" ) status = "Excel file with the given FileID parameter not found" httpCode = 404 res = createRequestDetails("404", xl_FileID, "404", status, httpCode, REQUEST_ID) return res fileName = dc.translateFileidToFilename(a2l_FileID) if fileName != False: UPLOAD_A2L_PATH = dc.getA2lUploadPath(accessID) a2l_file = os.path.join(UPLOAD_A2L_PATH, fileName) print(a2l_file) else: print( "ERROR: CM_Validation: a2l file with the given FileID parameter not found" ) status = "a2l file with the given FileID parameter not found" httpCode = 404 res = createRequestDetails("404", a2l_FileID, "404", status, httpCode, REQUEST_ID) return res OUT_DIR = CMConst.OUTPUT_DIR # Call the service. tool = CMConst.CLI_INTERFACE PARAMS = tool + " " + a2l_file + " " + xl_file + " " + OUT_DIR + " " + OUT_DIR #cmd = PYTHON_PATH + " " + tool + " " + PARAMS try: #subprocess.Popen(cmd) subprocess.call([tool, a2l_file, xl_file, OUT_DIR, OUT_DIR]) except Exception as e: print("ERROR: CM_Validation: " + str(e)) status = "Error(s) encountered in executing the service." httpCode = 500 res = createRequestDetails("404", "404", "404", status, httpCode, REQUEST_ID) return res fileName, md5chkSum = postProcessing(userFilesDir) if fileName != False: print("INFO: CM_Validation: Service executed successfully") status = "Service executed successfully." httpCode = 201 Generated_fileID = str(fileNameToHash(REQUEST_ID + getTimeStamp())) res = createRequestDetails(fileName, Generated_fileID, md5chkSum, status, httpCode, REQUEST_ID) if createFileDetails(Generated_fileID, fileName, accessID): res = make_response(res, httpCode) return res else: print("ERROR: CM_Validation: Error Saving results to server") status = "[ERR] Error saving file to disk." httpCode = 500 res["Status"] = status res["HttpResponse"] = httpCode res = make_response(res, httpCode) #abort(500, "Error saving file to disk") return res else: status = "Error(s) encountered in post processing the results." httpCode = 500 res = createRequestDetails("404", "404", "404", status, httpCode, REQUEST_ID) return res else: print("ERROR: CM_Validation: Invalid Access-ID, request denied.") status = "[ERR] Invalid Access-ID, request denied." httpCode = 403 res = createRequestDetails("xxx", "xxx", "xxx", status, httpCode, REQUEST_ID) res = make_response(res, httpCode) return res
def generateUmlDiagram(): # declare as global to update the original variable. global REQUEST_ID # generate a new id for each request. REQUEST_ID = str(uuid.uuid4()) accessID = getAccessId() if isAccessIdAuthentic(accessID): #Get the File IDs from the request parameters inputFileID = request.json #This is the folder in workspace where all the files generated by services shoudl be stored. userFilesDir = dc.getUserFilesDir(accessID) sourceFilePath = dc.getTextUploadPath(accessID) try: inputFile_ID = inputFileID["InputFileID"] except Exception as e: print("ERROR: GEN_UML: " + str(e)) inputFile_ID = "x" print("INFO_GEN_UML: " + inputFile_ID) # File IDs to FileName translation sourceFileName = dc.translateFileidToFilename(inputFile_ID) print("INFO_GEN_UML: FileName on Server: " + sourceFileName) if sourceFileName != False: sourcefile = os.path.join(sourceFilePath, sourceFileName) else: print( "ERROR: GEN_UML: Configuration(source code) with the given FileID parameter not found" ) status = "Configuration(source code) with the given FileID parameter not found" httpCode = 404 res = createRequestDetails("N/A", inputFile_ID, "N/A", status, httpCode, REQUEST_ID) return res # Call the service. tool = TOOL_PATH # the cmd script takes in two parameters Input .TXT file and the path to # the output directory which in this case is the "USER FILES DIRECTORY" #PARAMS = sourcefile + " " + OUTPUT_DIR #cmd = tool + " " + PARAMS try: subprocess.call( ['java', '-jar', tool, sourcefile, "-o", OUTPUT_DIR]) except Exception as e: print("ERROR: GEN_UML: " + str(e)) status = "Error(s) encountered in executing the service." httpCode = 500 res = createRequestDetails("N/A", inputFile_ID, "N/A", status, httpCode, REQUEST_ID) return res #processing on generate file #Plant uml names the generated file the same as input source file. fileName, md5chkSum = postProcessing(sourceFileName, userFilesDir) if fileName != False: print("INFO: GEN_UML: Service executed successfully.") status = "UML diagram generated successfully." Generated_fileID = str(fileNameToHash(REQUEST_ID + getTimeStamp())) httpCode = 201 res = createRequestDetails(fileName, Generated_fileID, md5chkSum, status, httpCode, REQUEST_ID) if createFileDetails(Generated_fileID, fileName, accessID): res = make_response(res, httpCode) return res else: print("ERROR: GEN_UML: Error Saving file to server") status = "[ERR] Error saving file to disk." httpCode = 500 res["Status"] = status res["HttpResponse"] = httpCode res = make_response(res, httpCode) #abort(500, "Error saving file to disk") return res else: print( "ERROR: GEN_UML: Error(s) encountered in post processing the results." ) status = "Error(s) encountered in post processing the results." httpCode = 500 res = createRequestDetails(fileName, inputFile_ID, "N/A", status, httpCode, REQUEST_ID) return res else: print("ERROR: GEN_UML: Invalid Access-ID, request denied.") status = "[ERR] Invalid Access-ID, request denied." httpCode = 403 res = createRequestDetails("N/A", "N/A", "N/A", status, httpCode, REQUEST_ID) res = make_response(res, httpCode) return res
def trainModel(workflowID, inputFileID): # declare as global to update the original variable. global REQUEST_ID # generate a new id for each request. REQUEST_ID = str(uuid.uuid4()) accessID = getAccessId() if fh.isAccessIdAuthentic(accessID): InputFileID = inputFileID inputFilePath = dc.getArchiveUploadPath((accessID)) ####################################################### if workflowID != "" and InputFileID != "": print("INFO: HyAPI_ML: " + InputFileID) # File IDs to FileName translation inputFileName = dc.translateFileidToFilename(InputFileID) if inputFileName != False: print("INFO: HyAPI_ML: FileName on Server: " + inputFileName) inputFile = os.path.join(inputFilePath, inputFileName) else: print( "ERROR: HyAPI_ML: Input image with the given FileID parameter not found" ) status = "Input image with the given FileID parameter not found" httpCode = MLC.HTTP_NOT_FOUND res = createReqDetails_training(workflowID, "NA", "NI", REQUEST_ID, status, httpCode) return res # Get the path to the user's workspace and the specified workflow: # STEP-1: EXTRACT THE INPUT ZIP TO THE TEMP DIRECTORY. tempDirectory = MLDC.getUserTF_tmp_dir(accessID, workflowID) if extractZip(inputFile, tempDirectory): print("INFO: HyAPI_ML: Inputs extracted to temp directory.") else: status = "Error extracting input .zip file to workflow." print("ERROR: HyAPI_ML: " + status) httpCode = MLC.HTTP_SERVER_ERROR res = createReqDetails_training(workflowID, "NA", "NI", REQUEST_ID, status, httpCode) return make_response(res, httpCode) #STEP-2: MOVE THE FILES TO THE RELEVANT DIRECTORIES IN THE WORKFLOW STRUCTURE. userWorkflow = MLDC.getUserTF_workflow(accessID, workflowID) workflow_Dataset = MLDC.getDatasetDirectory(accessID, workflowID) workflow_Annotations = MLDC.getAnnotationsDirectory( accessID, workflowID) workflow_Training = MLDC.getTrainingDirectory(accessID, workflowID) try: testData_tmp = os.path.join(tempDirectory, MLC.USR_INPUT_TEST_IMAGES_DIR) testData_final = os.path.join(workflow_Dataset, MLC.USR_INPUT_TEST_IMAGES_DIR) trainData_tmp = os.path.join(tempDirectory, MLC.USR_INPUT_TRAIN_IMAGES_DIR) trainData_final = os.path.join(workflow_Dataset, MLC.USR_INPUT_TRAIN_IMAGES_DIR) labelMap_tmp = os.path.join(tempDirectory, MLC.USR_INPUT_LABEL_MAP_FILE) labelMap_final = os.path.join(workflow_Annotations, MLC.USR_INPUT_LABEL_MAP_FILE) pipeline_tmp = os.path.join(tempDirectory, MLC.USR_INPUT_PIPELINE_FILE) pipeline_final = os.path.join(workflow_Training, MLC.USR_INPUT_PIPELINE_FILE) # (source,destination) if fh.moveFile(testData_tmp, testData_final) and fh.moveFile( trainData_tmp, trainData_final) and fh.moveFile( labelMap_tmp, labelMap_final) and fh.moveFile( pipeline_tmp, pipeline_final): print("INFO: HyAPI_ML:Files moved successfully") else: print("INFO: HyAPI_ML: Error moving files.") status = "Error(s) while populating the workflow with new files" httpCode = MLC.HTTP_SERVER_ERROR res = createReqDetails_training(workflowID, "NA", "NI", REQUEST_ID, status, httpCode) return make_response(res, httpCode) except Exception as e: print(str(e)) status = "Error(s) while populating the workflow with new files" httpCode = MLC.HTTP_SERVER_ERROR res = createReqDetails_training(workflowID, "NA", "NI", REQUEST_ID, status, httpCode) return make_response(res, httpCode) #STEP-3-a: GENERATE UNIFIED CSV FILES FOR TEST AND TRAIN DATA. train_csv_path = os.path.join(workflow_Annotations, MLC.USR_OUTPUT_TRAIN_CSV) test_csv_path = os.path.join(workflow_Annotations, MLC.USR_OUTPUT_TEST_CSV) try: xml_to_csv.generateCsvFromXml(trainData_final, train_csv_path) xml_to_csv.generateCsvFromXml(testData_final, test_csv_path) print("Dataset XMLs converted to CSV files successfully.") except Exception as e: print(str(e)) status = "Error(s) while generating annotaion CSV files from the dataset XMLs." httpCode = MLC.HTTP_SERVER_ERROR res = createReqDetails_training(workflowID, "NA", "NI", REQUEST_ID, status, httpCode) return make_response(res, httpCode) #STEP-3-b: GENERATE TF-RECORDS FILES FOR TEST AND TRAIN DATA train_records_path = os.path.join(workflow_Annotations, MLC.USR_OUTPUT_TRAIN_RECORD) test_records_path = os.path.join(workflow_Annotations, MLC.USR_OUTPUT_TEST_RECORD) try: generate_tfrecord.generateTfRecords(train_records_path, trainData_final, train_csv_path) generate_tfrecord.generateTfRecords(test_records_path, testData_final, test_csv_path) print("TF-Records file generated successfully.") except Exception as e: print(str(e)) status = "Error(s) occured while generating tf-records file." httpCode = MLC.HTTP_SERVER_ERROR res = createReqDetails_training(workflowID, "NA", "NI", REQUEST_ID, status, httpCode) return make_response(res, httpCode) #STEP-4: START THE TRAINING JOB try: containerID = DockerInterface.startModelTraining( REQUEST_ID, userWorkflow) if containerID != False: print("Training process started, Container ID: " + str(containerID)) status = "Training process started, Model will be available for export once this process finishes." httpCode = MLC.HTTP_CREATED res = createReqDetails_training(workflowID, containerID, "NI", REQUEST_ID, status, httpCode) if MLDC.updateWorkflow(accessID, workflowID, status, containerID): return make_response(res, httpCode) else: status = "Error saving workflow details to db." print("ERROR: HyAPI_ML: " + status) httpCode = MLC.HTTP_SERVER_ERROR res = createReqDetails_training( workflowID, containerID, "NI", REQUEST_ID, status, httpCode) return make_response(res, httpCode) else: status = "Error(s) encountered while starting the training job." print(status) httpCode = MLC.HTTP_SERVER_ERROR res = createReqDetails_training(workflowID, "NA", "NI", REQUEST_ID, status, httpCode) return make_response(res, httpCode) except Exception as e: print(str(e)) status = "Error(s) encountered while starting the training job." httpCode = MLC.HTTP_SERVER_ERROR res = createReqDetails_training(workflowID, "NA", "NI", REQUEST_ID, status, httpCode) return make_response(res, httpCode) #test = DockerInterface.checkContainerStatus(containerID) #print(str(test)) #If one of the required parameters in the request are missing. else: status = "WorkflowName & BaseModel values are required in input json. One or both are missing in the request." print("ERROR: HyAPI_ML: " + status) httpCode = MLC.HTTP_BAD_REQUEST res = createReqDetails_newWorkflow("N/A", "N/A", "N/A", REQUEST_ID, status, httpCode) return make_response(res, httpCode)
def detectStopSign(): # declare as global to update the original variable. global REQUEST_ID # generate a new id for each request. REQUEST_ID = str(uuid.uuid4()) accessID = getAccessId() if fh.isAccessIdAuthentic(accessID): #Get the File IDs from the request parameters inputFileID = request.json #This is the folder in workspace where all the files generated by services shoudl be stored. userFilesDir = dc.getUserFilesDir(accessID) inputFilePath = dc.getImageUploadPath((accessID)) try: inputFile_ID = inputFileID["InputFileID"] except Exception as e: print("ERROR: HyAPI_ML: " + str(e)) inputFile_ID = "x" print("INFO: HyAPI_ML: " + inputFile_ID) # File IDs to FileName translation inputFileName = dc.translateFileidToFilename(inputFile_ID) if inputFileName != False: print("INFO: HyAPI_ML FileName on Server: " + inputFileName) inputFile = os.path.join(inputFilePath, inputFileName) else: print( "ERROR: HyAPI_ML: Input image with the given FileID parameter not found" ) status = "Input image with the given FileID parameter not found" httpCode = MLC.HTTP_NOT_FOUND res = createRequestDetails(inputFile_ID, "N/A", status, httpCode, REQUEST_ID) return res #procStatus holds the filename procStatus = ml_od.detectStopSign(inputFile, FGB, LMB, userFilesDir) if procStatus != False: outputFileID = str(fh.generateHash(REQUEST_ID + fh.getTimeStamp())) status = "Object detection process successful" httpCode = MLC.HTTP_CREATED res = createRequestDetails(inputFileName, outputFileID, status, httpCode, REQUEST_ID) if dc.createFileDetails(outputFileID, procStatus, accessID): print("INFO: HyAPI_ML: Object detection process successful") res = make_response(res, httpCode) return res else: #Internal server error print("ERROR: HyAPI_ML: Error Saving file to server") status = "[ERR] Error saving file to disk." httpCode = MLC.HTTP_SERVER_ERROR res["Status"] = status res["HttpResponse"] = httpCode res = make_response(res, httpCode) #abort(500, "Error saving file to disk") return res else: print("ERROR: HyAPI_ML: Invalid Access-ID, request denied.") status = "[ERR] Invalid Access-ID, request denied." httpCode = MLC.HTTP_FORBIDDEN res = createRequestDetails("N/A", "N/A", status, httpCode, REQUEST_ID) res = make_response(res, httpCode) return res