def trainModel(accessID, workflowID, inputFileID):
isInitSuccessful()
REQUEST_ID = str(uuid.uuid4())
inputFileName = dc.translateFileidToFilename(inputFileID)
inputFilePath = dc.getArchiveUploadPath((accessID))
if inputFileName != False:
print("INFO: HyAPI_ML_INTERNAL: FileName: " + inputFileName)
inputFile = os.path.join(inputFilePath, inputFileName)
else:
print(
"ERROR: HyAPI_ML_INTERNAL: Input image with the given FileID parameter not found"
)
return False
# Get the path to the user's workspace and the specified workflow:
# STEP-1: EXTRACT THE INPUT ZIP TO THE TEMP DIRECTORY.
tempDirectory = MLDC.getUserTF_tmp_dir(accessID, workflowID)
if extractZip(inputFile, tempDirectory):
print("INFO: HyAPI_ML_INTERNAL: Inputs extracted to temp directory.")
else:
print(
"ERROR: HyAPI_ML_INTERNAL: Error extracting input .zip file to workflow."
)
return False
#STEP-2: MOVE THE FILES TO THE RELEVANT DIRECTORIES IN THE WORKFLOW STRUCTURE.
userWorkflow = MLDC.getUserTF_workflow(accessID, workflowID)
workflow_Dataset = MLDC.getDatasetDirectory(accessID, workflowID)
workflow_Annotations = MLDC.getAnnotationsDirectory(accessID, workflowID)
workflow_Training = MLDC.getTrainingDirectory(accessID, workflowID)
try:
testData_tmp = os.path.join(tempDirectory,
MLC.USR_INPUT_TEST_IMAGES_DIR)
testData_final = os.path.join(workflow_Dataset,
MLC.USR_INPUT_TEST_IMAGES_DIR)
trainData_tmp = os.path.join(tempDirectory,
MLC.USR_INPUT_TRAIN_IMAGES_DIR)
trainData_final = os.path.join(workflow_Dataset,
MLC.USR_INPUT_TRAIN_IMAGES_DIR)
labelMap_tmp = os.path.join(tempDirectory,
MLC.USR_INPUT_LABEL_MAP_FILE)
labelMap_final = os.path.join(workflow_Annotations,
MLC.USR_INPUT_LABEL_MAP_FILE)
pipeline_tmp = os.path.join(tempDirectory, MLC.USR_INPUT_PIPELINE_FILE)
pipeline_final = os.path.join(workflow_Training,
MLC.USR_INPUT_PIPELINE_FILE)
# (source,destination)
if fh.moveFile(testData_tmp, testData_final) and fh.moveFile(
trainData_tmp, trainData_final) and fh.moveFile(
labelMap_tmp, labelMap_final) and fh.moveFile(
pipeline_tmp, pipeline_final):
print("INFO: HyAPI_ML_INTERNAL:Files moved successfully")
else:
print("ERROR: HyAPI_ML_INTERNAL: Error moving files.")
return False
except Exception as e:
print(str(e))
return False
#STEP-3-a: GENERATE UNIFIED CSV FILES FOR TEST AND TRAIN DATA.
train_csv_path = os.path.join(workflow_Annotations,
MLC.USR_OUTPUT_TRAIN_CSV)
test_csv_path = os.path.join(workflow_Annotations, MLC.USR_OUTPUT_TEST_CSV)
try:
xml_to_csv.generateCsvFromXml(trainData_final, train_csv_path)
xml_to_csv.generateCsvFromXml(testData_final, test_csv_path)
print(
"INFO: HyAPI_ML_INTERNAL: Dataset XMLs converted to CSV files successfully."
)
except Exception as e:
print(str(e))
return False
#STEP-3-b: GENERATE TF-RECORDS FILES FOR TEST AND TRAIN DATA
train_records_path = os.path.join(workflow_Annotations,
MLC.USR_OUTPUT_TRAIN_RECORD)
test_records_path = os.path.join(workflow_Annotations,
MLC.USR_OUTPUT_TEST_RECORD)
try:
generate_tfrecord.generateTfRecords(train_records_path,
trainData_final, train_csv_path)
generate_tfrecord.generateTfRecords(test_records_path, testData_final,
test_csv_path)
print(
"INFO: HyAPI_ML_INTERNAL: TF-Records file generated successfully.")
except Exception as e:
print(str(e))
return False
#STEP-4: START THE TRAINING JOB
try:
containerID = DockerInterface.startModelTraining(
REQUEST_ID, userWorkflow)
if containerID != False:
print(
"INFO: HyAPI_ML_INTERNAL: Training process started, Container ID: "
+ str(containerID))
status = "Training started."
if MLDC.updateWorkflow(accessID, workflowID, status, containerID):
return True
else:
print(
"ERROR: HyAPI_ML_INTERNAL: Error saving workflow details to db."
)
return False
else:
print(
"ERROR: HyAPI_ML_INTERNAL: Error(s) encountered while starting the training job."
)
return False
except Exception as e:
print(str(e))
return False
def trainModel(workflowID, inputFileID):
# declare as global to update the original variable.
global REQUEST_ID
# generate a new id for each request.
REQUEST_ID = str(uuid.uuid4())
accessID = getAccessId()
if fh.isAccessIdAuthentic(accessID):
InputFileID = inputFileID
inputFilePath = dc.getArchiveUploadPath((accessID))
#######################################################
if workflowID != "" and InputFileID != "":
print("INFO: HyAPI_ML: " + InputFileID)
# File IDs to FileName translation
inputFileName = dc.translateFileidToFilename(InputFileID)
if inputFileName != False:
print("INFO: HyAPI_ML: FileName on Server: " + inputFileName)
inputFile = os.path.join(inputFilePath, inputFileName)
else:
print(
"ERROR: HyAPI_ML: Input image with the given FileID parameter not found"
)
status = "Input image with the given FileID parameter not found"
httpCode = MLC.HTTP_NOT_FOUND
res = createReqDetails_training(workflowID, "NA", "NI",
REQUEST_ID, status, httpCode)
return res
# Get the path to the user's workspace and the specified workflow:
# STEP-1: EXTRACT THE INPUT ZIP TO THE TEMP DIRECTORY.
tempDirectory = MLDC.getUserTF_tmp_dir(accessID, workflowID)
if extractZip(inputFile, tempDirectory):
print("INFO: HyAPI_ML: Inputs extracted to temp directory.")
else:
status = "Error extracting input .zip file to workflow."
print("ERROR: HyAPI_ML: " + status)
httpCode = MLC.HTTP_SERVER_ERROR
res = createReqDetails_training(workflowID, "NA", "NI",
REQUEST_ID, status, httpCode)
return make_response(res, httpCode)
#STEP-2: MOVE THE FILES TO THE RELEVANT DIRECTORIES IN THE WORKFLOW STRUCTURE.
userWorkflow = MLDC.getUserTF_workflow(accessID, workflowID)
workflow_Dataset = MLDC.getDatasetDirectory(accessID, workflowID)
workflow_Annotations = MLDC.getAnnotationsDirectory(
accessID, workflowID)
workflow_Training = MLDC.getTrainingDirectory(accessID, workflowID)
try:
testData_tmp = os.path.join(tempDirectory,
MLC.USR_INPUT_TEST_IMAGES_DIR)
testData_final = os.path.join(workflow_Dataset,
MLC.USR_INPUT_TEST_IMAGES_DIR)
trainData_tmp = os.path.join(tempDirectory,
MLC.USR_INPUT_TRAIN_IMAGES_DIR)
trainData_final = os.path.join(workflow_Dataset,
MLC.USR_INPUT_TRAIN_IMAGES_DIR)
labelMap_tmp = os.path.join(tempDirectory,
MLC.USR_INPUT_LABEL_MAP_FILE)
labelMap_final = os.path.join(workflow_Annotations,
MLC.USR_INPUT_LABEL_MAP_FILE)
pipeline_tmp = os.path.join(tempDirectory,
MLC.USR_INPUT_PIPELINE_FILE)
pipeline_final = os.path.join(workflow_Training,
MLC.USR_INPUT_PIPELINE_FILE)
# (source,destination)
if fh.moveFile(testData_tmp, testData_final) and fh.moveFile(
trainData_tmp, trainData_final) and fh.moveFile(
labelMap_tmp, labelMap_final) and fh.moveFile(
pipeline_tmp, pipeline_final):
print("INFO: HyAPI_ML:Files moved successfully")
else:
print("INFO: HyAPI_ML: Error moving files.")
status = "Error(s) while populating the workflow with new files"
httpCode = MLC.HTTP_SERVER_ERROR
res = createReqDetails_training(workflowID, "NA", "NI",
REQUEST_ID, status,
httpCode)
return make_response(res, httpCode)
except Exception as e:
print(str(e))
status = "Error(s) while populating the workflow with new files"
httpCode = MLC.HTTP_SERVER_ERROR
res = createReqDetails_training(workflowID, "NA", "NI",
REQUEST_ID, status, httpCode)
return make_response(res, httpCode)
#STEP-3-a: GENERATE UNIFIED CSV FILES FOR TEST AND TRAIN DATA.
train_csv_path = os.path.join(workflow_Annotations,
MLC.USR_OUTPUT_TRAIN_CSV)
test_csv_path = os.path.join(workflow_Annotations,
MLC.USR_OUTPUT_TEST_CSV)
try:
xml_to_csv.generateCsvFromXml(trainData_final, train_csv_path)
xml_to_csv.generateCsvFromXml(testData_final, test_csv_path)
print("Dataset XMLs converted to CSV files successfully.")
except Exception as e:
print(str(e))
status = "Error(s) while generating annotaion CSV files from the dataset XMLs."
httpCode = MLC.HTTP_SERVER_ERROR
res = createReqDetails_training(workflowID, "NA", "NI",
REQUEST_ID, status, httpCode)
return make_response(res, httpCode)
#STEP-3-b: GENERATE TF-RECORDS FILES FOR TEST AND TRAIN DATA
train_records_path = os.path.join(workflow_Annotations,
MLC.USR_OUTPUT_TRAIN_RECORD)
test_records_path = os.path.join(workflow_Annotations,
MLC.USR_OUTPUT_TEST_RECORD)
try:
generate_tfrecord.generateTfRecords(train_records_path,
trainData_final,
train_csv_path)
generate_tfrecord.generateTfRecords(test_records_path,
testData_final,
test_csv_path)
print("TF-Records file generated successfully.")
except Exception as e:
print(str(e))
status = "Error(s) occured while generating tf-records file."
httpCode = MLC.HTTP_SERVER_ERROR
res = createReqDetails_training(workflowID, "NA", "NI",
REQUEST_ID, status, httpCode)
return make_response(res, httpCode)
#STEP-4: START THE TRAINING JOB
try:
containerID = DockerInterface.startModelTraining(
REQUEST_ID, userWorkflow)
if containerID != False:
print("Training process started, Container ID: " +
str(containerID))
status = "Training process started, Model will be available for export once this process finishes."
httpCode = MLC.HTTP_CREATED
res = createReqDetails_training(workflowID, containerID,
"NI", REQUEST_ID, status,
httpCode)
if MLDC.updateWorkflow(accessID, workflowID, status,
containerID):
return make_response(res, httpCode)
else:
status = "Error saving workflow details to db."
print("ERROR: HyAPI_ML: " + status)
httpCode = MLC.HTTP_SERVER_ERROR
res = createReqDetails_training(
workflowID, containerID, "NI", REQUEST_ID, status,
httpCode)
return make_response(res, httpCode)
else:
status = "Error(s) encountered while starting the training job."
print(status)
httpCode = MLC.HTTP_SERVER_ERROR
res = createReqDetails_training(workflowID, "NA", "NI",
REQUEST_ID, status,
httpCode)
return make_response(res, httpCode)
except Exception as e:
print(str(e))
status = "Error(s) encountered while starting the training job."
httpCode = MLC.HTTP_SERVER_ERROR
res = createReqDetails_training(workflowID, "NA", "NI",
REQUEST_ID, status, httpCode)
return make_response(res, httpCode)
#test = DockerInterface.checkContainerStatus(containerID)
#print(str(test))
#If one of the required parameters in the request are missing.
else:
status = "WorkflowName & BaseModel values are required in input json. One or both are missing in the request."
print("ERROR: HyAPI_ML: " + status)
httpCode = MLC.HTTP_BAD_REQUEST
res = createReqDetails_newWorkflow("N/A", "N/A", "N/A", REQUEST_ID,
status, httpCode)
return make_response(res, httpCode)