def getFile(fileID):
# declare as global to update the original variable.
global REQUEST_ID
# generate a new id for each request.
REQUEST_ID = str(uuid.uuid4())
accessID = getAccessId()
mfileID = fileID
if isAccessIdAuthentic(accessID):
fileName = dc.translateFileidToFilename(mfileID)
userFilesDir = dc.getUserFilesDir(accessID)
print("INFO: FILEDOWNLOAD: Requested File Name: " + str(fileName))
if fileName != False:
try:
md5Checksum = generateMD5(os.path.join(userFilesDir, fileName))
print(
"INFO: FILEDOWNLOAD: File with the given FileID found on server."
)
status = "File with the given FileID found on server."
httpCode = 200
res = createRequestDetails(fileName, mfileID, md5Checksum,
status, httpCode, REQUEST_ID)
return send_from_directory(userFilesDir,
filename=fileName,
as_attachment=True)
except Exception as e:
print(
"ERROR: FILEDOWNLOAD: Error(s) encountered while transferring file."
)
print(str(e))
status = "Error(s) encountered while transferring file."
httpCode = 500
res = createRequestDetails("N/A", mfileID, "N/A", status,
httpCode, REQUEST_ID)
else:
print(
"ERROR: FILEDOWNLOAD: File with the given FileID parameter not found"
)
status = "File with the given FileID parameter not found"
httpCode = 404
res = createRequestDetails("N/A", mfileID, "N/A", status, httpCode,
REQUEST_ID)
else:
print("ERROR: FILEDOWNLOAD: Invalid Access-ID, request denied.")
status = "[ERR] Invalid Access-ID, request denied."
httpCode = 403
res = createRequestDetails("N/A", "N/A", "N/A", status, httpCode,
REQUEST_ID)
res = make_response(res, httpCode)
return res
############################ END OF SCRIPT ###################################
def checkFileStatus(fileID):
# declare as global to update the original variable.
global REQUEST_ID
# generate a new id for each request.
REQUEST_ID = str(uuid.uuid4())
accessID = getAccessId()
if isAccessIdAuthentic(accessID):
mfileID = fileID
userWorkspace = os.path.join(WORKSPACE_DIR, accessID)
userFilesDir = os.path.join(userWorkspace, USER_FILES)
fileName = dc.translateFileidToFilename(mfileID)
if fileName != False:
md5Checksum = generateMD5(os.path.join(userFilesDir, fileName))
status = "File with the given FileID found on server."
httpCode = 200
res = createRequestDetails(fileName, mfileID, md5Checksum, status,
httpCode, REQUEST_ID)
return res
else:
print("File with the given FileID parameter not found")
status = "File with the given FileID parameter not found"
httpCode = 404
res = createRequestDetails("N/A", mfileID, "N/A", status, httpCode,
REQUEST_ID)
return res
else:
print("Invalid Access-ID, request denied.")
status = "[ERR] Invalid Access-ID, request denied."
httpCode = 403
res = createRequestDetails("N/A", "N/A", "N/A", status, httpCode,
REQUEST_ID)
res = make_response(res, httpCode)
return res
def trainModel(accessID, workflowID, inputFileID):
isInitSuccessful()
REQUEST_ID = str(uuid.uuid4())
inputFileName = dc.translateFileidToFilename(inputFileID)
inputFilePath = dc.getArchiveUploadPath((accessID))
if inputFileName != False:
print("INFO: HyAPI_ML_INTERNAL: FileName: " + inputFileName)
inputFile = os.path.join(inputFilePath, inputFileName)
else:
print(
"ERROR: HyAPI_ML_INTERNAL: Input image with the given FileID parameter not found"
)
return False
# Get the path to the user's workspace and the specified workflow:
# STEP-1: EXTRACT THE INPUT ZIP TO THE TEMP DIRECTORY.
tempDirectory = MLDC.getUserTF_tmp_dir(accessID, workflowID)
if extractZip(inputFile, tempDirectory):
print("INFO: HyAPI_ML_INTERNAL: Inputs extracted to temp directory.")
else:
print(
"ERROR: HyAPI_ML_INTERNAL: Error extracting input .zip file to workflow."
)
return False
#STEP-2: MOVE THE FILES TO THE RELEVANT DIRECTORIES IN THE WORKFLOW STRUCTURE.
userWorkflow = MLDC.getUserTF_workflow(accessID, workflowID)
workflow_Dataset = MLDC.getDatasetDirectory(accessID, workflowID)
workflow_Annotations = MLDC.getAnnotationsDirectory(accessID, workflowID)
workflow_Training = MLDC.getTrainingDirectory(accessID, workflowID)
try:
testData_tmp = os.path.join(tempDirectory,
MLC.USR_INPUT_TEST_IMAGES_DIR)
testData_final = os.path.join(workflow_Dataset,
MLC.USR_INPUT_TEST_IMAGES_DIR)
trainData_tmp = os.path.join(tempDirectory,
MLC.USR_INPUT_TRAIN_IMAGES_DIR)
trainData_final = os.path.join(workflow_Dataset,
MLC.USR_INPUT_TRAIN_IMAGES_DIR)
labelMap_tmp = os.path.join(tempDirectory,
MLC.USR_INPUT_LABEL_MAP_FILE)
labelMap_final = os.path.join(workflow_Annotations,
MLC.USR_INPUT_LABEL_MAP_FILE)
pipeline_tmp = os.path.join(tempDirectory, MLC.USR_INPUT_PIPELINE_FILE)
pipeline_final = os.path.join(workflow_Training,
MLC.USR_INPUT_PIPELINE_FILE)
# (source,destination)
if fh.moveFile(testData_tmp, testData_final) and fh.moveFile(
trainData_tmp, trainData_final) and fh.moveFile(
labelMap_tmp, labelMap_final) and fh.moveFile(
pipeline_tmp, pipeline_final):
print("INFO: HyAPI_ML_INTERNAL:Files moved successfully")
else:
print("ERROR: HyAPI_ML_INTERNAL: Error moving files.")
return False
except Exception as e:
print(str(e))
return False
#STEP-3-a: GENERATE UNIFIED CSV FILES FOR TEST AND TRAIN DATA.
train_csv_path = os.path.join(workflow_Annotations,
MLC.USR_OUTPUT_TRAIN_CSV)
test_csv_path = os.path.join(workflow_Annotations, MLC.USR_OUTPUT_TEST_CSV)
try:
xml_to_csv.generateCsvFromXml(trainData_final, train_csv_path)
xml_to_csv.generateCsvFromXml(testData_final, test_csv_path)
print(
"INFO: HyAPI_ML_INTERNAL: Dataset XMLs converted to CSV files successfully."
)
except Exception as e:
print(str(e))
return False
#STEP-3-b: GENERATE TF-RECORDS FILES FOR TEST AND TRAIN DATA
train_records_path = os.path.join(workflow_Annotations,
MLC.USR_OUTPUT_TRAIN_RECORD)
test_records_path = os.path.join(workflow_Annotations,
MLC.USR_OUTPUT_TEST_RECORD)
try:
generate_tfrecord.generateTfRecords(train_records_path,
trainData_final, train_csv_path)
generate_tfrecord.generateTfRecords(test_records_path, testData_final,
test_csv_path)
print(
"INFO: HyAPI_ML_INTERNAL: TF-Records file generated successfully.")
except Exception as e:
print(str(e))
return False
#STEP-4: START THE TRAINING JOB
try:
containerID = DockerInterface.startModelTraining(
REQUEST_ID, userWorkflow)
if containerID != False:
print(
"INFO: HyAPI_ML_INTERNAL: Training process started, Container ID: "
+ str(containerID))
status = "Training started."
if MLDC.updateWorkflow(accessID, workflowID, status, containerID):
return True
else:
print(
"ERROR: HyAPI_ML_INTERNAL: Error saving workflow details to db."
)
return False
else:
print(
"ERROR: HyAPI_ML_INTERNAL: Error(s) encountered while starting the training job."
)
return False
except Exception as e:
print(str(e))
return False
def validateCompuMethods():
# declare as global to update the original variable.
global REQUEST_ID
# generate a new id for each request.
REQUEST_ID = str(uuid.uuid4())
accessID = getAccessId()
if isAccessIdAuthentic(accessID):
#########Get the File IDs from the request parameters#########
inputFileID = request.json
userFilesDir = dc.getUserFilesDir(accessID)
try:
#Get the File IDs from the request parameters
xl_FileID = inputFileID["xlFileID"]
except Exception as e:
print("ERROR: CM_Validation: " + str(e))
xl_FileID = "x"
try:
a2l_FileID = inputFileID["a2lFileID"]
except Exception as e:
print("ERROR: CM_Validation: " + str(e))
a2l_FileID = "x"
#########File IDs to FileName translation#########
fileName = dc.translateFileidToFilename(xl_FileID)
if fileName != False:
UPLOAD_XSLX_PATH = dc.getExcelUploadPath(accessID)
xl_file = os.path.join(UPLOAD_XSLX_PATH, fileName)
print(xl_file)
else:
print(
"ERROR: CM_Validation: Excel file with the given FileID parameter not found"
)
status = "Excel file with the given FileID parameter not found"
httpCode = 404
res = createRequestDetails("404", xl_FileID, "404", status,
httpCode, REQUEST_ID)
return res
fileName = dc.translateFileidToFilename(a2l_FileID)
if fileName != False:
UPLOAD_A2L_PATH = dc.getA2lUploadPath(accessID)
a2l_file = os.path.join(UPLOAD_A2L_PATH, fileName)
print(a2l_file)
else:
print(
"ERROR: CM_Validation: a2l file with the given FileID parameter not found"
)
status = "a2l file with the given FileID parameter not found"
httpCode = 404
res = createRequestDetails("404", a2l_FileID, "404", status,
httpCode, REQUEST_ID)
return res
OUT_DIR = CMConst.OUTPUT_DIR
# Call the service.
tool = CMConst.CLI_INTERFACE
PARAMS = tool + " " + a2l_file + " " + xl_file + " " + OUT_DIR + " " + OUT_DIR
#cmd = PYTHON_PATH + " " + tool + " " + PARAMS
try:
#subprocess.Popen(cmd)
subprocess.call([tool, a2l_file, xl_file, OUT_DIR, OUT_DIR])
except Exception as e:
print("ERROR: CM_Validation: " + str(e))
status = "Error(s) encountered in executing the service."
httpCode = 500
res = createRequestDetails("404", "404", "404", status, httpCode,
REQUEST_ID)
return res
fileName, md5chkSum = postProcessing(userFilesDir)
if fileName != False:
print("INFO: CM_Validation: Service executed successfully")
status = "Service executed successfully."
httpCode = 201
Generated_fileID = str(fileNameToHash(REQUEST_ID + getTimeStamp()))
res = createRequestDetails(fileName, Generated_fileID, md5chkSum,
status, httpCode, REQUEST_ID)
if createFileDetails(Generated_fileID, fileName, accessID):
res = make_response(res, httpCode)
return res
else:
print("ERROR: CM_Validation: Error Saving results to server")
status = "[ERR] Error saving file to disk."
httpCode = 500
res["Status"] = status
res["HttpResponse"] = httpCode
res = make_response(res, httpCode)
#abort(500, "Error saving file to disk")
return res
else:
status = "Error(s) encountered in post processing the results."
httpCode = 500
res = createRequestDetails("404", "404", "404", status, httpCode,
REQUEST_ID)
return res
else:
print("ERROR: CM_Validation: Invalid Access-ID, request denied.")
status = "[ERR] Invalid Access-ID, request denied."
httpCode = 403
res = createRequestDetails("xxx", "xxx", "xxx", status, httpCode,
REQUEST_ID)
res = make_response(res, httpCode)
return res
def generateUmlDiagram():
# declare as global to update the original variable.
global REQUEST_ID
# generate a new id for each request.
REQUEST_ID = str(uuid.uuid4())
accessID = getAccessId()
if isAccessIdAuthentic(accessID):
#Get the File IDs from the request parameters
inputFileID = request.json
#This is the folder in workspace where all the files generated by services shoudl be stored.
userFilesDir = dc.getUserFilesDir(accessID)
sourceFilePath = dc.getTextUploadPath(accessID)
try:
inputFile_ID = inputFileID["InputFileID"]
except Exception as e:
print("ERROR: GEN_UML: " + str(e))
inputFile_ID = "x"
print("INFO_GEN_UML: " + inputFile_ID)
# File IDs to FileName translation
sourceFileName = dc.translateFileidToFilename(inputFile_ID)
print("INFO_GEN_UML: FileName on Server: " + sourceFileName)
if sourceFileName != False:
sourcefile = os.path.join(sourceFilePath, sourceFileName)
else:
print(
"ERROR: GEN_UML: Configuration(source code) with the given FileID parameter not found"
)
status = "Configuration(source code) with the given FileID parameter not found"
httpCode = 404
res = createRequestDetails("N/A", inputFile_ID, "N/A", status,
httpCode, REQUEST_ID)
return res
# Call the service.
tool = TOOL_PATH
# the cmd script takes in two parameters Input .TXT file and the path to
# the output directory which in this case is the "USER FILES DIRECTORY"
#PARAMS = sourcefile + " " + OUTPUT_DIR
#cmd = tool + " " + PARAMS
try:
subprocess.call(
['java', '-jar', tool, sourcefile, "-o", OUTPUT_DIR])
except Exception as e:
print("ERROR: GEN_UML: " + str(e))
status = "Error(s) encountered in executing the service."
httpCode = 500
res = createRequestDetails("N/A", inputFile_ID, "N/A", status,
httpCode, REQUEST_ID)
return res
#processing on generate file
#Plant uml names the generated file the same as input source file.
fileName, md5chkSum = postProcessing(sourceFileName, userFilesDir)
if fileName != False:
print("INFO: GEN_UML: Service executed successfully.")
status = "UML diagram generated successfully."
Generated_fileID = str(fileNameToHash(REQUEST_ID + getTimeStamp()))
httpCode = 201
res = createRequestDetails(fileName, Generated_fileID, md5chkSum,
status, httpCode, REQUEST_ID)
if createFileDetails(Generated_fileID, fileName, accessID):
res = make_response(res, httpCode)
return res
else:
print("ERROR: GEN_UML: Error Saving file to server")
status = "[ERR] Error saving file to disk."
httpCode = 500
res["Status"] = status
res["HttpResponse"] = httpCode
res = make_response(res, httpCode)
#abort(500, "Error saving file to disk")
return res
else:
print(
"ERROR: GEN_UML: Error(s) encountered in post processing the results."
)
status = "Error(s) encountered in post processing the results."
httpCode = 500
res = createRequestDetails(fileName, inputFile_ID, "N/A", status,
httpCode, REQUEST_ID)
return res
else:
print("ERROR: GEN_UML: Invalid Access-ID, request denied.")
status = "[ERR] Invalid Access-ID, request denied."
httpCode = 403
res = createRequestDetails("N/A", "N/A", "N/A", status, httpCode,
REQUEST_ID)
res = make_response(res, httpCode)
return res
def trainModel(workflowID, inputFileID):
# declare as global to update the original variable.
global REQUEST_ID
# generate a new id for each request.
REQUEST_ID = str(uuid.uuid4())
accessID = getAccessId()
if fh.isAccessIdAuthentic(accessID):
InputFileID = inputFileID
inputFilePath = dc.getArchiveUploadPath((accessID))
#######################################################
if workflowID != "" and InputFileID != "":
print("INFO: HyAPI_ML: " + InputFileID)
# File IDs to FileName translation
inputFileName = dc.translateFileidToFilename(InputFileID)
if inputFileName != False:
print("INFO: HyAPI_ML: FileName on Server: " + inputFileName)
inputFile = os.path.join(inputFilePath, inputFileName)
else:
print(
"ERROR: HyAPI_ML: Input image with the given FileID parameter not found"
)
status = "Input image with the given FileID parameter not found"
httpCode = MLC.HTTP_NOT_FOUND
res = createReqDetails_training(workflowID, "NA", "NI",
REQUEST_ID, status, httpCode)
return res
# Get the path to the user's workspace and the specified workflow:
# STEP-1: EXTRACT THE INPUT ZIP TO THE TEMP DIRECTORY.
tempDirectory = MLDC.getUserTF_tmp_dir(accessID, workflowID)
if extractZip(inputFile, tempDirectory):
print("INFO: HyAPI_ML: Inputs extracted to temp directory.")
else:
status = "Error extracting input .zip file to workflow."
print("ERROR: HyAPI_ML: " + status)
httpCode = MLC.HTTP_SERVER_ERROR
res = createReqDetails_training(workflowID, "NA", "NI",
REQUEST_ID, status, httpCode)
return make_response(res, httpCode)
#STEP-2: MOVE THE FILES TO THE RELEVANT DIRECTORIES IN THE WORKFLOW STRUCTURE.
userWorkflow = MLDC.getUserTF_workflow(accessID, workflowID)
workflow_Dataset = MLDC.getDatasetDirectory(accessID, workflowID)
workflow_Annotations = MLDC.getAnnotationsDirectory(
accessID, workflowID)
workflow_Training = MLDC.getTrainingDirectory(accessID, workflowID)
try:
testData_tmp = os.path.join(tempDirectory,
MLC.USR_INPUT_TEST_IMAGES_DIR)
testData_final = os.path.join(workflow_Dataset,
MLC.USR_INPUT_TEST_IMAGES_DIR)
trainData_tmp = os.path.join(tempDirectory,
MLC.USR_INPUT_TRAIN_IMAGES_DIR)
trainData_final = os.path.join(workflow_Dataset,
MLC.USR_INPUT_TRAIN_IMAGES_DIR)
labelMap_tmp = os.path.join(tempDirectory,
MLC.USR_INPUT_LABEL_MAP_FILE)
labelMap_final = os.path.join(workflow_Annotations,
MLC.USR_INPUT_LABEL_MAP_FILE)
pipeline_tmp = os.path.join(tempDirectory,
MLC.USR_INPUT_PIPELINE_FILE)
pipeline_final = os.path.join(workflow_Training,
MLC.USR_INPUT_PIPELINE_FILE)
# (source,destination)
if fh.moveFile(testData_tmp, testData_final) and fh.moveFile(
trainData_tmp, trainData_final) and fh.moveFile(
labelMap_tmp, labelMap_final) and fh.moveFile(
pipeline_tmp, pipeline_final):
print("INFO: HyAPI_ML:Files moved successfully")
else:
print("INFO: HyAPI_ML: Error moving files.")
status = "Error(s) while populating the workflow with new files"
httpCode = MLC.HTTP_SERVER_ERROR
res = createReqDetails_training(workflowID, "NA", "NI",
REQUEST_ID, status,
httpCode)
return make_response(res, httpCode)
except Exception as e:
print(str(e))
status = "Error(s) while populating the workflow with new files"
httpCode = MLC.HTTP_SERVER_ERROR
res = createReqDetails_training(workflowID, "NA", "NI",
REQUEST_ID, status, httpCode)
return make_response(res, httpCode)
#STEP-3-a: GENERATE UNIFIED CSV FILES FOR TEST AND TRAIN DATA.
train_csv_path = os.path.join(workflow_Annotations,
MLC.USR_OUTPUT_TRAIN_CSV)
test_csv_path = os.path.join(workflow_Annotations,
MLC.USR_OUTPUT_TEST_CSV)
try:
xml_to_csv.generateCsvFromXml(trainData_final, train_csv_path)
xml_to_csv.generateCsvFromXml(testData_final, test_csv_path)
print("Dataset XMLs converted to CSV files successfully.")
except Exception as e:
print(str(e))
status = "Error(s) while generating annotaion CSV files from the dataset XMLs."
httpCode = MLC.HTTP_SERVER_ERROR
res = createReqDetails_training(workflowID, "NA", "NI",
REQUEST_ID, status, httpCode)
return make_response(res, httpCode)
#STEP-3-b: GENERATE TF-RECORDS FILES FOR TEST AND TRAIN DATA
train_records_path = os.path.join(workflow_Annotations,
MLC.USR_OUTPUT_TRAIN_RECORD)
test_records_path = os.path.join(workflow_Annotations,
MLC.USR_OUTPUT_TEST_RECORD)
try:
generate_tfrecord.generateTfRecords(train_records_path,
trainData_final,
train_csv_path)
generate_tfrecord.generateTfRecords(test_records_path,
testData_final,
test_csv_path)
print("TF-Records file generated successfully.")
except Exception as e:
print(str(e))
status = "Error(s) occured while generating tf-records file."
httpCode = MLC.HTTP_SERVER_ERROR
res = createReqDetails_training(workflowID, "NA", "NI",
REQUEST_ID, status, httpCode)
return make_response(res, httpCode)
#STEP-4: START THE TRAINING JOB
try:
containerID = DockerInterface.startModelTraining(
REQUEST_ID, userWorkflow)
if containerID != False:
print("Training process started, Container ID: " +
str(containerID))
status = "Training process started, Model will be available for export once this process finishes."
httpCode = MLC.HTTP_CREATED
res = createReqDetails_training(workflowID, containerID,
"NI", REQUEST_ID, status,
httpCode)
if MLDC.updateWorkflow(accessID, workflowID, status,
containerID):
return make_response(res, httpCode)
else:
status = "Error saving workflow details to db."
print("ERROR: HyAPI_ML: " + status)
httpCode = MLC.HTTP_SERVER_ERROR
res = createReqDetails_training(
workflowID, containerID, "NI", REQUEST_ID, status,
httpCode)
return make_response(res, httpCode)
else:
status = "Error(s) encountered while starting the training job."
print(status)
httpCode = MLC.HTTP_SERVER_ERROR
res = createReqDetails_training(workflowID, "NA", "NI",
REQUEST_ID, status,
httpCode)
return make_response(res, httpCode)
except Exception as e:
print(str(e))
status = "Error(s) encountered while starting the training job."
httpCode = MLC.HTTP_SERVER_ERROR
res = createReqDetails_training(workflowID, "NA", "NI",
REQUEST_ID, status, httpCode)
return make_response(res, httpCode)
#test = DockerInterface.checkContainerStatus(containerID)
#print(str(test))
#If one of the required parameters in the request are missing.
else:
status = "WorkflowName & BaseModel values are required in input json. One or both are missing in the request."
print("ERROR: HyAPI_ML: " + status)
httpCode = MLC.HTTP_BAD_REQUEST
res = createReqDetails_newWorkflow("N/A", "N/A", "N/A", REQUEST_ID,
status, httpCode)
return make_response(res, httpCode)
def detectStopSign():
# declare as global to update the original variable.
global REQUEST_ID
# generate a new id for each request.
REQUEST_ID = str(uuid.uuid4())
accessID = getAccessId()
if fh.isAccessIdAuthentic(accessID):
#Get the File IDs from the request parameters
inputFileID = request.json
#This is the folder in workspace where all the files generated by services shoudl be stored.
userFilesDir = dc.getUserFilesDir(accessID)
inputFilePath = dc.getImageUploadPath((accessID))
try:
inputFile_ID = inputFileID["InputFileID"]
except Exception as e:
print("ERROR: HyAPI_ML: " + str(e))
inputFile_ID = "x"
print("INFO: HyAPI_ML: " + inputFile_ID)
# File IDs to FileName translation
inputFileName = dc.translateFileidToFilename(inputFile_ID)
if inputFileName != False:
print("INFO: HyAPI_ML FileName on Server: " + inputFileName)
inputFile = os.path.join(inputFilePath, inputFileName)
else:
print(
"ERROR: HyAPI_ML: Input image with the given FileID parameter not found"
)
status = "Input image with the given FileID parameter not found"
httpCode = MLC.HTTP_NOT_FOUND
res = createRequestDetails(inputFile_ID, "N/A", status, httpCode,
REQUEST_ID)
return res
#procStatus holds the filename
procStatus = ml_od.detectStopSign(inputFile, FGB, LMB, userFilesDir)
if procStatus != False:
outputFileID = str(fh.generateHash(REQUEST_ID + fh.getTimeStamp()))
status = "Object detection process successful"
httpCode = MLC.HTTP_CREATED
res = createRequestDetails(inputFileName, outputFileID, status,
httpCode, REQUEST_ID)
if dc.createFileDetails(outputFileID, procStatus, accessID):
print("INFO: HyAPI_ML: Object detection process successful")
res = make_response(res, httpCode)
return res
else: #Internal server error
print("ERROR: HyAPI_ML: Error Saving file to server")
status = "[ERR] Error saving file to disk."
httpCode = MLC.HTTP_SERVER_ERROR
res["Status"] = status
res["HttpResponse"] = httpCode
res = make_response(res, httpCode)
#abort(500, "Error saving file to disk")
return res
else:
print("ERROR: HyAPI_ML: Invalid Access-ID, request denied.")
status = "[ERR] Invalid Access-ID, request denied."
httpCode = MLC.HTTP_FORBIDDEN
res = createRequestDetails("N/A", "N/A", status, httpCode, REQUEST_ID)
res = make_response(res, httpCode)
return res