def trainModel(accessID, workflowID, inputFileID): isInitSuccessful() REQUEST_ID = str(uuid.uuid4()) inputFileName = dc.translateFileidToFilename(inputFileID) inputFilePath = dc.getArchiveUploadPath((accessID)) if inputFileName != False: print("INFO: HyAPI_ML_INTERNAL: FileName: " + inputFileName) inputFile = os.path.join(inputFilePath, inputFileName) else: print( "ERROR: HyAPI_ML_INTERNAL: Input image with the given FileID parameter not found" ) return False # Get the path to the user's workspace and the specified workflow: # STEP-1: EXTRACT THE INPUT ZIP TO THE TEMP DIRECTORY. tempDirectory = MLDC.getUserTF_tmp_dir(accessID, workflowID) if extractZip(inputFile, tempDirectory): print("INFO: HyAPI_ML_INTERNAL: Inputs extracted to temp directory.") else: print( "ERROR: HyAPI_ML_INTERNAL: Error extracting input .zip file to workflow." ) return False #STEP-2: MOVE THE FILES TO THE RELEVANT DIRECTORIES IN THE WORKFLOW STRUCTURE. userWorkflow = MLDC.getUserTF_workflow(accessID, workflowID) workflow_Dataset = MLDC.getDatasetDirectory(accessID, workflowID) workflow_Annotations = MLDC.getAnnotationsDirectory(accessID, workflowID) workflow_Training = MLDC.getTrainingDirectory(accessID, workflowID) try: testData_tmp = os.path.join(tempDirectory, MLC.USR_INPUT_TEST_IMAGES_DIR) testData_final = os.path.join(workflow_Dataset, MLC.USR_INPUT_TEST_IMAGES_DIR) trainData_tmp = os.path.join(tempDirectory, MLC.USR_INPUT_TRAIN_IMAGES_DIR) trainData_final = os.path.join(workflow_Dataset, MLC.USR_INPUT_TRAIN_IMAGES_DIR) labelMap_tmp = os.path.join(tempDirectory, MLC.USR_INPUT_LABEL_MAP_FILE) labelMap_final = os.path.join(workflow_Annotations, MLC.USR_INPUT_LABEL_MAP_FILE) pipeline_tmp = os.path.join(tempDirectory, MLC.USR_INPUT_PIPELINE_FILE) pipeline_final = os.path.join(workflow_Training, MLC.USR_INPUT_PIPELINE_FILE) # (source,destination) if fh.moveFile(testData_tmp, testData_final) and fh.moveFile( trainData_tmp, trainData_final) and fh.moveFile( labelMap_tmp, labelMap_final) and fh.moveFile( pipeline_tmp, pipeline_final): print("INFO: HyAPI_ML_INTERNAL:Files moved successfully") else: print("ERROR: HyAPI_ML_INTERNAL: Error moving files.") return False except Exception as e: print(str(e)) return False #STEP-3-a: GENERATE UNIFIED CSV FILES FOR TEST AND TRAIN DATA. train_csv_path = os.path.join(workflow_Annotations, MLC.USR_OUTPUT_TRAIN_CSV) test_csv_path = os.path.join(workflow_Annotations, MLC.USR_OUTPUT_TEST_CSV) try: xml_to_csv.generateCsvFromXml(trainData_final, train_csv_path) xml_to_csv.generateCsvFromXml(testData_final, test_csv_path) print( "INFO: HyAPI_ML_INTERNAL: Dataset XMLs converted to CSV files successfully." ) except Exception as e: print(str(e)) return False #STEP-3-b: GENERATE TF-RECORDS FILES FOR TEST AND TRAIN DATA train_records_path = os.path.join(workflow_Annotations, MLC.USR_OUTPUT_TRAIN_RECORD) test_records_path = os.path.join(workflow_Annotations, MLC.USR_OUTPUT_TEST_RECORD) try: generate_tfrecord.generateTfRecords(train_records_path, trainData_final, train_csv_path) generate_tfrecord.generateTfRecords(test_records_path, testData_final, test_csv_path) print( "INFO: HyAPI_ML_INTERNAL: TF-Records file generated successfully.") except Exception as e: print(str(e)) return False #STEP-4: START THE TRAINING JOB try: containerID = DockerInterface.startModelTraining( REQUEST_ID, userWorkflow) if containerID != False: print( "INFO: HyAPI_ML_INTERNAL: Training process started, Container ID: " + str(containerID)) status = "Training started." if MLDC.updateWorkflow(accessID, workflowID, status, containerID): return True else: print( "ERROR: HyAPI_ML_INTERNAL: Error saving workflow details to db." ) return False else: print( "ERROR: HyAPI_ML_INTERNAL: Error(s) encountered while starting the training job." ) return False except Exception as e: print(str(e)) return False
def trainModel(workflowID, inputFileID): # declare as global to update the original variable. global REQUEST_ID # generate a new id for each request. REQUEST_ID = str(uuid.uuid4()) accessID = getAccessId() if fh.isAccessIdAuthentic(accessID): InputFileID = inputFileID inputFilePath = dc.getArchiveUploadPath((accessID)) ####################################################### if workflowID != "" and InputFileID != "": print("INFO: HyAPI_ML: " + InputFileID) # File IDs to FileName translation inputFileName = dc.translateFileidToFilename(InputFileID) if inputFileName != False: print("INFO: HyAPI_ML: FileName on Server: " + inputFileName) inputFile = os.path.join(inputFilePath, inputFileName) else: print( "ERROR: HyAPI_ML: Input image with the given FileID parameter not found" ) status = "Input image with the given FileID parameter not found" httpCode = MLC.HTTP_NOT_FOUND res = createReqDetails_training(workflowID, "NA", "NI", REQUEST_ID, status, httpCode) return res # Get the path to the user's workspace and the specified workflow: # STEP-1: EXTRACT THE INPUT ZIP TO THE TEMP DIRECTORY. tempDirectory = MLDC.getUserTF_tmp_dir(accessID, workflowID) if extractZip(inputFile, tempDirectory): print("INFO: HyAPI_ML: Inputs extracted to temp directory.") else: status = "Error extracting input .zip file to workflow." print("ERROR: HyAPI_ML: " + status) httpCode = MLC.HTTP_SERVER_ERROR res = createReqDetails_training(workflowID, "NA", "NI", REQUEST_ID, status, httpCode) return make_response(res, httpCode) #STEP-2: MOVE THE FILES TO THE RELEVANT DIRECTORIES IN THE WORKFLOW STRUCTURE. userWorkflow = MLDC.getUserTF_workflow(accessID, workflowID) workflow_Dataset = MLDC.getDatasetDirectory(accessID, workflowID) workflow_Annotations = MLDC.getAnnotationsDirectory( accessID, workflowID) workflow_Training = MLDC.getTrainingDirectory(accessID, workflowID) try: testData_tmp = os.path.join(tempDirectory, MLC.USR_INPUT_TEST_IMAGES_DIR) testData_final = os.path.join(workflow_Dataset, MLC.USR_INPUT_TEST_IMAGES_DIR) trainData_tmp = os.path.join(tempDirectory, MLC.USR_INPUT_TRAIN_IMAGES_DIR) trainData_final = os.path.join(workflow_Dataset, MLC.USR_INPUT_TRAIN_IMAGES_DIR) labelMap_tmp = os.path.join(tempDirectory, MLC.USR_INPUT_LABEL_MAP_FILE) labelMap_final = os.path.join(workflow_Annotations, MLC.USR_INPUT_LABEL_MAP_FILE) pipeline_tmp = os.path.join(tempDirectory, MLC.USR_INPUT_PIPELINE_FILE) pipeline_final = os.path.join(workflow_Training, MLC.USR_INPUT_PIPELINE_FILE) # (source,destination) if fh.moveFile(testData_tmp, testData_final) and fh.moveFile( trainData_tmp, trainData_final) and fh.moveFile( labelMap_tmp, labelMap_final) and fh.moveFile( pipeline_tmp, pipeline_final): print("INFO: HyAPI_ML:Files moved successfully") else: print("INFO: HyAPI_ML: Error moving files.") status = "Error(s) while populating the workflow with new files" httpCode = MLC.HTTP_SERVER_ERROR res = createReqDetails_training(workflowID, "NA", "NI", REQUEST_ID, status, httpCode) return make_response(res, httpCode) except Exception as e: print(str(e)) status = "Error(s) while populating the workflow with new files" httpCode = MLC.HTTP_SERVER_ERROR res = createReqDetails_training(workflowID, "NA", "NI", REQUEST_ID, status, httpCode) return make_response(res, httpCode) #STEP-3-a: GENERATE UNIFIED CSV FILES FOR TEST AND TRAIN DATA. train_csv_path = os.path.join(workflow_Annotations, MLC.USR_OUTPUT_TRAIN_CSV) test_csv_path = os.path.join(workflow_Annotations, MLC.USR_OUTPUT_TEST_CSV) try: xml_to_csv.generateCsvFromXml(trainData_final, train_csv_path) xml_to_csv.generateCsvFromXml(testData_final, test_csv_path) print("Dataset XMLs converted to CSV files successfully.") except Exception as e: print(str(e)) status = "Error(s) while generating annotaion CSV files from the dataset XMLs." httpCode = MLC.HTTP_SERVER_ERROR res = createReqDetails_training(workflowID, "NA", "NI", REQUEST_ID, status, httpCode) return make_response(res, httpCode) #STEP-3-b: GENERATE TF-RECORDS FILES FOR TEST AND TRAIN DATA train_records_path = os.path.join(workflow_Annotations, MLC.USR_OUTPUT_TRAIN_RECORD) test_records_path = os.path.join(workflow_Annotations, MLC.USR_OUTPUT_TEST_RECORD) try: generate_tfrecord.generateTfRecords(train_records_path, trainData_final, train_csv_path) generate_tfrecord.generateTfRecords(test_records_path, testData_final, test_csv_path) print("TF-Records file generated successfully.") except Exception as e: print(str(e)) status = "Error(s) occured while generating tf-records file." httpCode = MLC.HTTP_SERVER_ERROR res = createReqDetails_training(workflowID, "NA", "NI", REQUEST_ID, status, httpCode) return make_response(res, httpCode) #STEP-4: START THE TRAINING JOB try: containerID = DockerInterface.startModelTraining( REQUEST_ID, userWorkflow) if containerID != False: print("Training process started, Container ID: " + str(containerID)) status = "Training process started, Model will be available for export once this process finishes." httpCode = MLC.HTTP_CREATED res = createReqDetails_training(workflowID, containerID, "NI", REQUEST_ID, status, httpCode) if MLDC.updateWorkflow(accessID, workflowID, status, containerID): return make_response(res, httpCode) else: status = "Error saving workflow details to db." print("ERROR: HyAPI_ML: " + status) httpCode = MLC.HTTP_SERVER_ERROR res = createReqDetails_training( workflowID, containerID, "NI", REQUEST_ID, status, httpCode) return make_response(res, httpCode) else: status = "Error(s) encountered while starting the training job." print(status) httpCode = MLC.HTTP_SERVER_ERROR res = createReqDetails_training(workflowID, "NA", "NI", REQUEST_ID, status, httpCode) return make_response(res, httpCode) except Exception as e: print(str(e)) status = "Error(s) encountered while starting the training job." httpCode = MLC.HTTP_SERVER_ERROR res = createReqDetails_training(workflowID, "NA", "NI", REQUEST_ID, status, httpCode) return make_response(res, httpCode) #test = DockerInterface.checkContainerStatus(containerID) #print(str(test)) #If one of the required parameters in the request are missing. else: status = "WorkflowName & BaseModel values are required in input json. One or both are missing in the request." print("ERROR: HyAPI_ML: " + status) httpCode = MLC.HTTP_BAD_REQUEST res = createReqDetails_newWorkflow("N/A", "N/A", "N/A", REQUEST_ID, status, httpCode) return make_response(res, httpCode)