def getFile(fileID):
# declare as global to update the original variable.
global REQUEST_ID
# generate a new id for each request.
REQUEST_ID = str(uuid.uuid4())
accessID = getAccessId()
mfileID = fileID
if isAccessIdAuthentic(accessID):
fileName = dc.translateFileidToFilename(mfileID)
userFilesDir = dc.getUserFilesDir(accessID)
print("INFO: FILEDOWNLOAD: Requested File Name: " + str(fileName))
if fileName != False:
try:
md5Checksum = generateMD5(os.path.join(userFilesDir, fileName))
print(
"INFO: FILEDOWNLOAD: File with the given FileID found on server."
)
status = "File with the given FileID found on server."
httpCode = 200
res = createRequestDetails(fileName, mfileID, md5Checksum,
status, httpCode, REQUEST_ID)
return send_from_directory(userFilesDir,
filename=fileName,
as_attachment=True)
except Exception as e:
print(
"ERROR: FILEDOWNLOAD: Error(s) encountered while transferring file."
)
print(str(e))
status = "Error(s) encountered while transferring file."
httpCode = 500
res = createRequestDetails("N/A", mfileID, "N/A", status,
httpCode, REQUEST_ID)
else:
print(
"ERROR: FILEDOWNLOAD: File with the given FileID parameter not found"
)
status = "File with the given FileID parameter not found"
httpCode = 404
res = createRequestDetails("N/A", mfileID, "N/A", status, httpCode,
REQUEST_ID)
else:
print("ERROR: FILEDOWNLOAD: Invalid Access-ID, request denied.")
status = "[ERR] Invalid Access-ID, request denied."
httpCode = 403
res = createRequestDetails("N/A", "N/A", "N/A", status, httpCode,
REQUEST_ID)
res = make_response(res, httpCode)
return res
############################ END OF SCRIPT ###################################
def authenticateCredentials(userName, password):
accessID = dc.encryptWithSecretKey(userName)
password = dc.encryptWithSecretKey(password)
status = isAccessIdAuthentic(accessID)
if status != False:
dbPass = status["password"]
if password == dbPass:
return 0
else:
return 2
else:
return 1
def createRequestDetails(mStatus, mResponse, requestID):
#create a dictionary with details of the uploaded file
requestDetails = {
"Service": "New User Registration",
"RequestID": requestID,
"Status": mStatus,
"HttpResponse": mResponse,
"timeStamp": dc.getTimeStamp()
}
if dc.updateHistory(requestDetails, requestID):
print("INFO: SIGNUP: History log updated.")
else:
print("ERROR: SIGNUP: Request could not be added to history log.")
return requestDetails
def createRequestDetails(fileName, fileID, md5Checksum, mStatus, mResponse,
requestID):
try:
ext = fileName.rsplit(".", 1)[1].upper()
except:
print("ERROR: GEN_UML: Could not extract file's extension.")
ext = "-"
#create a dictionary with details of the uploaded file
requestDetails = {
"FileName": fileName,
"FileType": ext,
"FileID": fileID,
"RequestID": requestID,
"MD5": md5Checksum,
"Status": mStatus,
"HttpResponse": mResponse,
"timeStamp": getTimeStamp()
}
if dc.updateHistory(requestDetails, requestID):
print("INFO: GEN_UML: History log updated.")
else:
print("ERROR: GEN_UML: Request could not be added to history log.")
return requestDetails
def showWorkSpace(userName, accessID, accessToken):
loginForm = forms.LoginForm()
workflowForm = forms.newWorkflowForm()
if fh.isAccessIdAuthentic(accessID):
encodingKey = dc.generateHash(accessID)
if isAccessTokenAuthentic(accessID, accessToken):
userWorkspace = MLDC.getWorkspace(accessID)
url = url_for("mlGUI_bp.showWorkSpace",
userName=userName,
accessID=accessID,
accessToken=accessToken)
return render_template('workspace.html',
userName=userName,
apiKey=accessID,
encKey=encodingKey,
accessToken=accessToken,
workflowsList=userWorkspace,
workflowUrl=url,
status_message=status_msg,
form=workflowForm)
else:
print("couldn't login")
return render_template('login.html',
form=loginForm,
status="login-failed.")
def getSavePath(fileName,accessID):
extension = fileName.rsplit(".",1)[1].upper()
userUploadsDir = dc.getUserUploadsDir(accessID)
if extension in ["JPEG","JPG","PNG"]:
UPLOAD_IMG_PATH = os.path.join(userUploadsDir,CONSTANTS.USER_UPLOADS_IMAGES_DIR)
savePath = os.path.join(UPLOAD_IMG_PATH,fileName)
elif extension in ["ZIP"]:
UPLOAD_ZIP_PATH = os.path.join(userUploadsDir,CONSTANTS.USER_UPLOADS_ZIP_DIR)
savePath = os.path.join(UPLOAD_ZIP_PATH,fileName)
elif extension in ["ARXML"]:
UPLOAD_ARXML_PATH = os.path.join(userUploadsDir,CONSTANTS.USER_UPLOADS_ARXML_DIR)
savePath = os.path.join(UPLOAD_ARXML_PATH,fileName)
elif extension in ["JSON"]:
UPLOAD_JSON_PATH = os.path.join(userUploadsDir,CONSTANTS.USER_UPLOADS_JSON_DIR)
savePath = os.path.join(UPLOAD_JSON_PATH,fileName)
elif extension in ["XLSX","XLS"]:
UPLOAD_XSLX_PATH = os.path.join(userUploadsDir,CONSTANTS.USER_UPLOADS_XLSX_DIR)
savePath = os.path.join(UPLOAD_XSLX_PATH,fileName)
elif extension in ["A2L"]:
UPLOAD_A2L_PATH = os.path.join(userUploadsDir,CONSTANTS.USER_UPLOADS_A2L_DIR)
savePath = os.path.join(UPLOAD_A2L_PATH,fileName)
elif extension in ["TXT",".C"]:
UPLOAD_TXT_PATH = os.path.join(userUploadsDir,CONSTANTS.USER_UPLOADS_TEXT_DIR)
savePath = os.path.join(UPLOAD_TXT_PATH,fileName)
else:
return False
return savePath
def isAccessIdAuthentic(accessID):
API_KEY_DB = dc.getApiKeysFromDb()
if API_KEY_DB != False:
if accessID in API_KEY_DB.keys():
customerDataDict = API_KEY_DB[accessID]
print("INFO: FILEUPLOAD: Access ID authenticated. Requester identified as: "+ customerDataDict["customer"])
return True
else:
print("ERROR: FILEUPLOAD: Invalid AccessID, request denied.")
return False
def loginPage():
loginForm = forms.LoginForm()
if request.method == 'POST':
if loginForm.validate_on_submit():
print("INFO: ML_GUI: Form Data Validated.")
userName = loginForm.userName.data
password = loginForm.password.data
status = authenticateCredentials(userName, password)
if status == 0:
print("Username/password combination validated.")
accessID = dc.encryptWithSecretKey(userName)
encodingKey = dc.generateHash(accessID)
accessToken = generateAccessToken(userName, password, accessID,
encodingKey)
workspaces_url = url_for("mlGUI_bp.showWorkSpace",
userName=userName,
accessID=accessID,
accessToken=accessToken)
#workspaces_url = "192.168.0.192:5000" + workspaces_url
#print(workspaces_url)
return redirect(workspaces_url)
elif status == 1:
print("Invalid username.")
return render_template('login.html',
form=loginForm,
status="Invalid usename," + userName +
" does not exist in database.")
elif status == 2:
print("Incorrect password.")
return render_template(
'login.html',
form=loginForm,
status="Incorrect password given for user: "******"login.html", form=loginForm)
else:
return make_response("Invalid http method", 404)
def createRequestDetails(inFileID, outFileID, mStatus, mResponse, requestID):
#create a dictionary with details of the uploaded file
requestDetails = {
"In_FileName": str(inFileID),
"Out_FileID": str(outFileID),
"RequestID": requestID,
"Status": mStatus,
"HttpResponse": mResponse,
"timeStamp": fh.getTimeStamp()
}
if dc.updateHistory(requestDetails, requestID):
print("INFO: HyAPI_ML: History log updated.")
else:
print("ERROR: HyAPI_ML: Request could not be added to history log.")
return requestDetails
def createReqDetails_training(wfID, containerID, baseModel, requestID, mStatus,
httpCode):
#create a dictionary with details of the uploaded file
requestDetails = {
"WorkFlowID": str(wfID),
"ContainerID": containerID,
"BaseModel": baseModel,
"RequestID": requestID,
"Status": mStatus,
"HttpResponse": httpCode,
"timeStamp": fh.getTimeStamp()
}
if dc.updateHistory(requestDetails, requestID):
print("INFO: HyAPI_ML: History log updated.")
else:
print("ERROR: HyAPI_ML: Request could not be added to history log.")
return requestDetails
def isAccessTokenAuthentic(accessID, accessToken):
accountDetails = isAccessIdAuthentic(accessID)
if accountDetails != False:
try:
userName = accountDetails["userName"]
password = accountDetails["password"]
encodingKey = accountDetails["encodingKey"]
credentials = userName + accessID + password + encodingKey
internal_accessToken = dc.encryptWithSecretKey(credentials)
except Exception as e:
print(str(e))
return False
if internal_accessToken == accessToken:
return True
else:
print(credentials)
print(accessToken)
print(internal_accessToken)
return False
def showWorkflow(userName, accessID, accessToken, workflowID):
loginForm = forms.LoginForm()
if fh.isAccessIdAuthentic(accessID):
encodingKey = dc.generateHash(accessID)
if isAccessTokenAuthentic(accessID, accessToken):
userWorkspace = MLDC.getWorkspace(accessID)
workflow = userWorkspace[workflowID]
return render_template('workflow.html',
workflowName=workflow["workflow_Name"],
workflowID=workflow["workflow_ID"],
baseModel=workflow["baseModel"],
processID=workflow["process_ID"],
userName=userName,
accessID=accessID,
accessToken=accessToken,
encodingKey=encodingKey,
status_message=status_msg)
else:
print("couldn't login")
return render_template('login.html',
form=loginForm,
status="Not sure")
def createFileDetails(fileID, fileName, accessId):
try:
ext = fileName.rsplit(".", 1)[1].upper()
except:
print("ERROR: CAN_TEST: Could not extract file's extension.")
ext = "-"
#create a dictionary with details of the uploaded file
fileDetails = {
"FileName": fileName,
"FileType": ext,
"FileID": fileID,
"Access-id": accessId,
"timeStamp": getTimeStamp()
}
if dc.updateFilesOnServer(fileDetails, fileID):
print("INFO: CAN_TEST: History log updated.")
return True
else:
print("ERROR: CAN_TEST: Request could not be added to history log.")
return False
def checkFileStatus(fileID):
# declare as global to update the original variable.
global REQUEST_ID
# generate a new id for each request.
REQUEST_ID = str(uuid.uuid4())
accessID = getAccessId()
if isAccessIdAuthentic(accessID):
mfileID = fileID
userWorkspace = os.path.join(WORKSPACE_DIR, accessID)
userFilesDir = os.path.join(userWorkspace, USER_FILES)
fileName = dc.translateFileidToFilename(mfileID)
if fileName != False:
md5Checksum = generateMD5(os.path.join(userFilesDir, fileName))
status = "File with the given FileID found on server."
httpCode = 200
res = createRequestDetails(fileName, mfileID, md5Checksum, status,
httpCode, REQUEST_ID)
return res
else:
print("File with the given FileID parameter not found")
status = "File with the given FileID parameter not found"
httpCode = 404
res = createRequestDetails("N/A", mfileID, "N/A", status, httpCode,
REQUEST_ID)
return res
else:
print("Invalid Access-ID, request denied.")
status = "[ERR] Invalid Access-ID, request denied."
httpCode = 403
res = createRequestDetails("N/A", "N/A", "N/A", status, httpCode,
REQUEST_ID)
res = make_response(res, httpCode)
return res
def detectStopSign():
# declare as global to update the original variable.
global REQUEST_ID
# generate a new id for each request.
REQUEST_ID = str(uuid.uuid4())
accessID = getAccessId()
if fh.isAccessIdAuthentic(accessID):
#Get the File IDs from the request parameters
inputFileID = request.json
#This is the folder in workspace where all the files generated by services shoudl be stored.
userFilesDir = dc.getUserFilesDir(accessID)
inputFilePath = dc.getImageUploadPath((accessID))
try:
inputFile_ID = inputFileID["InputFileID"]
except Exception as e:
print("ERROR: HyAPI_ML: " + str(e))
inputFile_ID = "x"
print("INFO: HyAPI_ML: " + inputFile_ID)
# File IDs to FileName translation
inputFileName = dc.translateFileidToFilename(inputFile_ID)
if inputFileName != False:
print("INFO: HyAPI_ML FileName on Server: " + inputFileName)
inputFile = os.path.join(inputFilePath, inputFileName)
else:
print(
"ERROR: HyAPI_ML: Input image with the given FileID parameter not found"
)
status = "Input image with the given FileID parameter not found"
httpCode = MLC.HTTP_NOT_FOUND
res = createRequestDetails(inputFile_ID, "N/A", status, httpCode,
REQUEST_ID)
return res
#procStatus holds the filename
procStatus = ml_od.detectStopSign(inputFile, FGB, LMB, userFilesDir)
if procStatus != False:
outputFileID = str(fh.generateHash(REQUEST_ID + fh.getTimeStamp()))
status = "Object detection process successful"
httpCode = MLC.HTTP_CREATED
res = createRequestDetails(inputFileName, outputFileID, status,
httpCode, REQUEST_ID)
if dc.createFileDetails(outputFileID, procStatus, accessID):
print("INFO: HyAPI_ML: Object detection process successful")
res = make_response(res, httpCode)
return res
else: #Internal server error
print("ERROR: HyAPI_ML: Error Saving file to server")
status = "[ERR] Error saving file to disk."
httpCode = MLC.HTTP_SERVER_ERROR
res["Status"] = status
res["HttpResponse"] = httpCode
res = make_response(res, httpCode)
#abort(500, "Error saving file to disk")
return res
else:
print("ERROR: HyAPI_ML: Invalid Access-ID, request denied.")
status = "[ERR] Invalid Access-ID, request denied."
httpCode = MLC.HTTP_FORBIDDEN
res = createRequestDetails("N/A", "N/A", status, httpCode, REQUEST_ID)
res = make_response(res, httpCode)
return res
def createNewWorkflow():
# declare as global to update the original variable.
global REQUEST_ID
# generate a new id for each request.
REQUEST_ID = str(uuid.uuid4())
accessID = getAccessId()
if fh.isAccessIdAuthentic(accessID):
#This is the folder in workspace where all the files generated by services shoudl be stored.
#userTfDir = dc.getUserTfDir(accessID)
#Get the File IDs from the request parameters
newWorkFlowDetails = request.json
#Get the data from the json included in the request.
try:
workFlowName = newWorkFlowDetails["WorkflowName"]
try:
#TODO: Provision for multiple base models
#baseModel = newWorkFlowDetails["BaseModel"]
#baseModelID = baseModel
baseModel = "01"
baseModelID = "01"
# 01,02,03,04
except Exception as e:
print("ERROR: HyAPI_ML: " + str(e))
baseModel = "x"
except Exception as e:
print("ERROR: HyAPI_ML: " + str(e))
workFlowName = "x"
#######################################################
if baseModel != "x" and workFlowName != "x":
userWorkflows = MLC.USER_WORKFLOWS_PATH
workflowID = dc.encryptWithSecretKey(workFlowName)
# STEP-1 Create new workspace.
try:
if not os.path.exists("./" + userWorkflows + "/" +
str(accessID) + "/" + str(workflowID)):
Path("./" + userWorkflows + "/" + str(accessID) + "/" +
str(workflowID)).mkdir(parents=True, exist_ok=True)
else:
pass
except Exception as e:
print(str(e))
status = "Error creating new workflow directory."
print("ERROR: HyAPI_ML: " + status)
httpCode = MLC.HTTP_SERVER_ERROR
res = createReqDetails_newWorkflow(workFlowName, workflowID,
baseModelID, REQUEST_ID,
status, httpCode)
return make_response(res, httpCode)
# STEP-2 Populate workspace with template
templateFile = MLC.TF_WORKSPACE_TEMPLATE
#print(templateFile)
userTfWorkflow = MLDC.getUserTF_workflow(accessID, workflowID)
#The template will be extracted to the newly created User's Tensoflow workflow directory
if extractZip(templateFile, userTfWorkflow):
print(
"INFO: HyAPI_ML: New workflow directory clone successful.")
else:
status = "Error cloning new workflow from template."
print("ERROR: HyAPI_ML: " + status)
httpCode = MLC.HTTP_SERVER_ERROR
res = createReqDetails_newWorkflow(workFlowName, workflowID,
baseModelID, REQUEST_ID,
status, httpCode)
return make_response(res, httpCode)
# STEP-3 Extract selected model in the "pre-trained-model" folder
if str(baseModel) == "01":
baseModel = MLC.MODEL_1_ZIP_PATH
baseModelName = MLC.MODEL_NAME_1
usrBaseModelPath = MLDC.getBaseModelDirectory(
accessID, workflowID)
if extractZip(baseModel, usrBaseModelPath):
print(usrBaseModelPath)
print(
"INFO: HyAPI_ML: Pre-Trained-Model copied successfully."
)
else:
status = "Error copying pre-trained-model to new workflow."
print("ERROR: HyAPI_ML: " + status)
httpCode = MLC.HTTP_SERVER_ERROR
res = createReqDetails_newWorkflow(workFlowName,
workflowID, baseModelID,
REQUEST_ID, status,
httpCode)
return make_response(res, httpCode)
# STEP-4 create and return response to client
status = "New workflow created successfuly."
print("INFO: HyAPI_ML: " + status)
httpCode = MLC.HTTP_CREATED
#create workflow details as a dict to add in the json db
workflowDetails = {
"workflow_Name": workFlowName,
"workflow_ID": workflowID,
"process_ID": "Never Run",
"process_Status": "Never Run",
"baseModel": baseModelName,
"timeStamp": fh.getTimeStamp()
}
if MLDC.saveWorkflowDetails(accessID, workflowID,
workflowDetails):
res = createReqDetails_newWorkflow(workFlowName,
workflowID, baseModelID,
REQUEST_ID, status,
httpCode)
return make_response(res, httpCode)
else:
status = "Error saving workflow details to db."
print("ERROR: HyAPI_ML: " + status)
httpCode = MLC.HTTP_SERVER_ERROR
res = createReqDetails_newWorkflow(workFlowName,
workflowID, baseModelID,
REQUEST_ID, status,
httpCode)
return make_response(res, httpCode)
else:
status = "Wrong parameter for baseModel(acceptable values are 01 & 02)"
print("ERROR: HyAPI_ML: " + status)
httpCode = MLC.HTTP_BAD_REQUEST
res = createReqDetails_newWorkflow(workFlowName, workflowID,
baseModelID, REQUEST_ID,
status, httpCode)
return make_response("Error", 400)
else:
status = "WorkflowName & BaseModel values are required in input json. One or both are missing in the request."
print("ERROR: HyAPI_ML: " + status)
httpCode = MLC.HTTP_BAD_REQUEST
res = createReqDetails_newWorkflow("N/A", "N/A", "N/A", REQUEST_ID,
status, httpCode)
return make_response(res, httpCode)
def generateAccessToken(userName, password, accessID, encodingKey):
password = dc.encryptWithSecretKey(password)
credentials = userName + accessID + password + encodingKey
accessToken = dc.encryptWithSecretKey(credentials)
print(credentials)
return accessToken
DB_PATH = CONSTANTS.DB_DIR
WORKSPACE_DIR = CONSTANTS.WORKSPACE_DIR
# These are directory names NOT path strings.
UPLOAD_ZIP_PATH = CONSTANTS.USER_UPLOADS_ZIP_DIR
UPLOAD_TXT_PATH = CONSTANTS.USER_UPLOADS_TEXT_DIR
################################ Init Databases ##############################
try:
from application import Data_Controller as dc
print("INFO: HyAPI_ML: DB Controller initialized successfully.")
except:
import Data_Controller as dc
print("DEBUG: HyAPI_ML: Safe Mode: Imported alternative DBC.")
API_KEY_DB = dc.getApiKeysFromDb()
REQ_HISTORY = dc.getRequestHistory()
FILES_ON_SERVER = dc.getFilesOnServer()
def isInitSuccessful():
if API_KEY_DB == False:
return False
if REQ_HISTORY == False:
return False
return True
global status_msg
status_msg = ""
"""
def trainModel(accessID, workflowID, inputFileID):
isInitSuccessful()
REQUEST_ID = str(uuid.uuid4())
inputFileName = dc.translateFileidToFilename(inputFileID)
inputFilePath = dc.getArchiveUploadPath((accessID))
if inputFileName != False:
print("INFO: HyAPI_ML_INTERNAL: FileName: " + inputFileName)
inputFile = os.path.join(inputFilePath, inputFileName)
else:
print(
"ERROR: HyAPI_ML_INTERNAL: Input image with the given FileID parameter not found"
)
return False
# Get the path to the user's workspace and the specified workflow:
# STEP-1: EXTRACT THE INPUT ZIP TO THE TEMP DIRECTORY.
tempDirectory = MLDC.getUserTF_tmp_dir(accessID, workflowID)
if extractZip(inputFile, tempDirectory):
print("INFO: HyAPI_ML_INTERNAL: Inputs extracted to temp directory.")
else:
print(
"ERROR: HyAPI_ML_INTERNAL: Error extracting input .zip file to workflow."
)
return False
#STEP-2: MOVE THE FILES TO THE RELEVANT DIRECTORIES IN THE WORKFLOW STRUCTURE.
userWorkflow = MLDC.getUserTF_workflow(accessID, workflowID)
workflow_Dataset = MLDC.getDatasetDirectory(accessID, workflowID)
workflow_Annotations = MLDC.getAnnotationsDirectory(accessID, workflowID)
workflow_Training = MLDC.getTrainingDirectory(accessID, workflowID)
try:
testData_tmp = os.path.join(tempDirectory,
MLC.USR_INPUT_TEST_IMAGES_DIR)
testData_final = os.path.join(workflow_Dataset,
MLC.USR_INPUT_TEST_IMAGES_DIR)
trainData_tmp = os.path.join(tempDirectory,
MLC.USR_INPUT_TRAIN_IMAGES_DIR)
trainData_final = os.path.join(workflow_Dataset,
MLC.USR_INPUT_TRAIN_IMAGES_DIR)
labelMap_tmp = os.path.join(tempDirectory,
MLC.USR_INPUT_LABEL_MAP_FILE)
labelMap_final = os.path.join(workflow_Annotations,
MLC.USR_INPUT_LABEL_MAP_FILE)
pipeline_tmp = os.path.join(tempDirectory, MLC.USR_INPUT_PIPELINE_FILE)
pipeline_final = os.path.join(workflow_Training,
MLC.USR_INPUT_PIPELINE_FILE)
# (source,destination)
if fh.moveFile(testData_tmp, testData_final) and fh.moveFile(
trainData_tmp, trainData_final) and fh.moveFile(
labelMap_tmp, labelMap_final) and fh.moveFile(
pipeline_tmp, pipeline_final):
print("INFO: HyAPI_ML_INTERNAL:Files moved successfully")
else:
print("ERROR: HyAPI_ML_INTERNAL: Error moving files.")
return False
except Exception as e:
print(str(e))
return False
#STEP-3-a: GENERATE UNIFIED CSV FILES FOR TEST AND TRAIN DATA.
train_csv_path = os.path.join(workflow_Annotations,
MLC.USR_OUTPUT_TRAIN_CSV)
test_csv_path = os.path.join(workflow_Annotations, MLC.USR_OUTPUT_TEST_CSV)
try:
xml_to_csv.generateCsvFromXml(trainData_final, train_csv_path)
xml_to_csv.generateCsvFromXml(testData_final, test_csv_path)
print(
"INFO: HyAPI_ML_INTERNAL: Dataset XMLs converted to CSV files successfully."
)
except Exception as e:
print(str(e))
return False
#STEP-3-b: GENERATE TF-RECORDS FILES FOR TEST AND TRAIN DATA
train_records_path = os.path.join(workflow_Annotations,
MLC.USR_OUTPUT_TRAIN_RECORD)
test_records_path = os.path.join(workflow_Annotations,
MLC.USR_OUTPUT_TEST_RECORD)
try:
generate_tfrecord.generateTfRecords(train_records_path,
trainData_final, train_csv_path)
generate_tfrecord.generateTfRecords(test_records_path, testData_final,
test_csv_path)
print(
"INFO: HyAPI_ML_INTERNAL: TF-Records file generated successfully.")
except Exception as e:
print(str(e))
return False
#STEP-4: START THE TRAINING JOB
try:
containerID = DockerInterface.startModelTraining(
REQUEST_ID, userWorkflow)
if containerID != False:
print(
"INFO: HyAPI_ML_INTERNAL: Training process started, Container ID: "
+ str(containerID))
status = "Training started."
if MLDC.updateWorkflow(accessID, workflowID, status, containerID):
return True
else:
print(
"ERROR: HyAPI_ML_INTERNAL: Error saving workflow details to db."
)
return False
else:
print(
"ERROR: HyAPI_ML_INTERNAL: Error(s) encountered while starting the training job."
)
return False
except Exception as e:
print(str(e))
return False
def createNewWorkflow(accessID, workFlowName, baseModelID):
# declare as global to update the original variable.
global REQUEST_ID
# generate a new id for each request.
REQUEST_ID = str(uuid.uuid4())
userWorkflows = MLC.USER_WORKFLOWS_PATH
workflowID = dc.encryptWithSecretKey(workFlowName)
# STEP-1 Create new workspace.
try:
if not os.path.exists("./" + userWorkflows + "/" + str(accessID) +
"/" + str(workflowID)):
Path("./" + userWorkflows + "/" + str(accessID) + "/" +
str(workflowID)).mkdir(parents=True, exist_ok=True)
else:
pass
except Exception as e:
print("ERROR: HyAPI_ML_INTERNAL: " + str(e))
return False
# STEP-2 Populate workspace with template
templateFile = MLC.TF_WORKSPACE_TEMPLATE
#print(templateFile)
userTfWorkflow = MLDC.getUserTF_workflow(accessID, workflowID)
#The template will be extracted to the newly created User's Tensoflow workflow directory
if extractZip(templateFile, userTfWorkflow):
print(
"INFO: HyAPI_ML_INTERNAL: New workflow directory clone successful."
)
else:
print(
"ERROR: HyAPI_ML_INTERNAL: Error cloning new workflow from template."
)
return False
#TODO: implement other models
# STEP-3 Extract selected model in the "pre-trained-model" folder
if str(baseModelID) == "01":
baseModel = MLC.MODEL_1_ZIP_PATH
baseModelName = MLC.MODEL_NAME_1
usrBaseModelPath = MLDC.getBaseModelDirectory(accessID, workflowID)
if extractZip(baseModel, usrBaseModelPath):
print(
"INFO: HyAPI_ML_INTERNAL: Pre-Trained-Model copied successfully."
)
else:
print(
"ERROR: HyAPI_ML_INTERNAL: Error copying pre-trained-model to new workflow."
)
return False
# STEP-4 create and return response to client
print("INFO: HyAPI_ML_INTERNAL: New workflow created successfuly.")
#create workflow details as a dict to add in the json db
workflowDetails = {
"workflow_Name": workFlowName,
"workflow_ID": workflowID,
"process_ID": "Never Run",
"process_Status": "Never Run",
"baseModel": baseModelName,
"dataSet": "empty",
"timeStamp": fh.getTimeStamp()
}
if MLDC.saveWorkflowDetails(accessID, workflowID, workflowDetails):
return True
else:
print(
"ERROR: HyAPI_ML_INTERNAL: Error saving workflow details to db."
)
return False
else:
print(
"ERROR: HyAPI_ML_INTERNAL: Wrong parameter for baseModel(acceptable values are 01 & 02)"
)
return False
def trainModel(workflowID, inputFileID):
# declare as global to update the original variable.
global REQUEST_ID
# generate a new id for each request.
REQUEST_ID = str(uuid.uuid4())
accessID = getAccessId()
if fh.isAccessIdAuthentic(accessID):
InputFileID = inputFileID
inputFilePath = dc.getArchiveUploadPath((accessID))
#######################################################
if workflowID != "" and InputFileID != "":
print("INFO: HyAPI_ML: " + InputFileID)
# File IDs to FileName translation
inputFileName = dc.translateFileidToFilename(InputFileID)
if inputFileName != False:
print("INFO: HyAPI_ML: FileName on Server: " + inputFileName)
inputFile = os.path.join(inputFilePath, inputFileName)
else:
print(
"ERROR: HyAPI_ML: Input image with the given FileID parameter not found"
)
status = "Input image with the given FileID parameter not found"
httpCode = MLC.HTTP_NOT_FOUND
res = createReqDetails_training(workflowID, "NA", "NI",
REQUEST_ID, status, httpCode)
return res
# Get the path to the user's workspace and the specified workflow:
# STEP-1: EXTRACT THE INPUT ZIP TO THE TEMP DIRECTORY.
tempDirectory = MLDC.getUserTF_tmp_dir(accessID, workflowID)
if extractZip(inputFile, tempDirectory):
print("INFO: HyAPI_ML: Inputs extracted to temp directory.")
else:
status = "Error extracting input .zip file to workflow."
print("ERROR: HyAPI_ML: " + status)
httpCode = MLC.HTTP_SERVER_ERROR
res = createReqDetails_training(workflowID, "NA", "NI",
REQUEST_ID, status, httpCode)
return make_response(res, httpCode)
#STEP-2: MOVE THE FILES TO THE RELEVANT DIRECTORIES IN THE WORKFLOW STRUCTURE.
userWorkflow = MLDC.getUserTF_workflow(accessID, workflowID)
workflow_Dataset = MLDC.getDatasetDirectory(accessID, workflowID)
workflow_Annotations = MLDC.getAnnotationsDirectory(
accessID, workflowID)
workflow_Training = MLDC.getTrainingDirectory(accessID, workflowID)
try:
testData_tmp = os.path.join(tempDirectory,
MLC.USR_INPUT_TEST_IMAGES_DIR)
testData_final = os.path.join(workflow_Dataset,
MLC.USR_INPUT_TEST_IMAGES_DIR)
trainData_tmp = os.path.join(tempDirectory,
MLC.USR_INPUT_TRAIN_IMAGES_DIR)
trainData_final = os.path.join(workflow_Dataset,
MLC.USR_INPUT_TRAIN_IMAGES_DIR)
labelMap_tmp = os.path.join(tempDirectory,
MLC.USR_INPUT_LABEL_MAP_FILE)
labelMap_final = os.path.join(workflow_Annotations,
MLC.USR_INPUT_LABEL_MAP_FILE)
pipeline_tmp = os.path.join(tempDirectory,
MLC.USR_INPUT_PIPELINE_FILE)
pipeline_final = os.path.join(workflow_Training,
MLC.USR_INPUT_PIPELINE_FILE)
# (source,destination)
if fh.moveFile(testData_tmp, testData_final) and fh.moveFile(
trainData_tmp, trainData_final) and fh.moveFile(
labelMap_tmp, labelMap_final) and fh.moveFile(
pipeline_tmp, pipeline_final):
print("INFO: HyAPI_ML:Files moved successfully")
else:
print("INFO: HyAPI_ML: Error moving files.")
status = "Error(s) while populating the workflow with new files"
httpCode = MLC.HTTP_SERVER_ERROR
res = createReqDetails_training(workflowID, "NA", "NI",
REQUEST_ID, status,
httpCode)
return make_response(res, httpCode)
except Exception as e:
print(str(e))
status = "Error(s) while populating the workflow with new files"
httpCode = MLC.HTTP_SERVER_ERROR
res = createReqDetails_training(workflowID, "NA", "NI",
REQUEST_ID, status, httpCode)
return make_response(res, httpCode)
#STEP-3-a: GENERATE UNIFIED CSV FILES FOR TEST AND TRAIN DATA.
train_csv_path = os.path.join(workflow_Annotations,
MLC.USR_OUTPUT_TRAIN_CSV)
test_csv_path = os.path.join(workflow_Annotations,
MLC.USR_OUTPUT_TEST_CSV)
try:
xml_to_csv.generateCsvFromXml(trainData_final, train_csv_path)
xml_to_csv.generateCsvFromXml(testData_final, test_csv_path)
print("Dataset XMLs converted to CSV files successfully.")
except Exception as e:
print(str(e))
status = "Error(s) while generating annotaion CSV files from the dataset XMLs."
httpCode = MLC.HTTP_SERVER_ERROR
res = createReqDetails_training(workflowID, "NA", "NI",
REQUEST_ID, status, httpCode)
return make_response(res, httpCode)
#STEP-3-b: GENERATE TF-RECORDS FILES FOR TEST AND TRAIN DATA
train_records_path = os.path.join(workflow_Annotations,
MLC.USR_OUTPUT_TRAIN_RECORD)
test_records_path = os.path.join(workflow_Annotations,
MLC.USR_OUTPUT_TEST_RECORD)
try:
generate_tfrecord.generateTfRecords(train_records_path,
trainData_final,
train_csv_path)
generate_tfrecord.generateTfRecords(test_records_path,
testData_final,
test_csv_path)
print("TF-Records file generated successfully.")
except Exception as e:
print(str(e))
status = "Error(s) occured while generating tf-records file."
httpCode = MLC.HTTP_SERVER_ERROR
res = createReqDetails_training(workflowID, "NA", "NI",
REQUEST_ID, status, httpCode)
return make_response(res, httpCode)
#STEP-4: START THE TRAINING JOB
try:
containerID = DockerInterface.startModelTraining(
REQUEST_ID, userWorkflow)
if containerID != False:
print("Training process started, Container ID: " +
str(containerID))
status = "Training process started, Model will be available for export once this process finishes."
httpCode = MLC.HTTP_CREATED
res = createReqDetails_training(workflowID, containerID,
"NI", REQUEST_ID, status,
httpCode)
if MLDC.updateWorkflow(accessID, workflowID, status,
containerID):
return make_response(res, httpCode)
else:
status = "Error saving workflow details to db."
print("ERROR: HyAPI_ML: " + status)
httpCode = MLC.HTTP_SERVER_ERROR
res = createReqDetails_training(
workflowID, containerID, "NI", REQUEST_ID, status,
httpCode)
return make_response(res, httpCode)
else:
status = "Error(s) encountered while starting the training job."
print(status)
httpCode = MLC.HTTP_SERVER_ERROR
res = createReqDetails_training(workflowID, "NA", "NI",
REQUEST_ID, status,
httpCode)
return make_response(res, httpCode)
except Exception as e:
print(str(e))
status = "Error(s) encountered while starting the training job."
httpCode = MLC.HTTP_SERVER_ERROR
res = createReqDetails_training(workflowID, "NA", "NI",
REQUEST_ID, status, httpCode)
return make_response(res, httpCode)
#test = DockerInterface.checkContainerStatus(containerID)
#print(str(test))
#If one of the required parameters in the request are missing.
else:
status = "WorkflowName & BaseModel values are required in input json. One or both are missing in the request."
print("ERROR: HyAPI_ML: " + status)
httpCode = MLC.HTTP_BAD_REQUEST
res = createReqDetails_newWorkflow("N/A", "N/A", "N/A", REQUEST_ID,
status, httpCode)
return make_response(res, httpCode)
def generateUmlDiagram():
# declare as global to update the original variable.
global REQUEST_ID
# generate a new id for each request.
REQUEST_ID = str(uuid.uuid4())
accessID = getAccessId()
if isAccessIdAuthentic(accessID):
#Get the File IDs from the request parameters
inputFileID = request.json
#This is the folder in workspace where all the files generated by services shoudl be stored.
userFilesDir = dc.getUserFilesDir(accessID)
sourceFilePath = dc.getTextUploadPath(accessID)
try:
inputFile_ID = inputFileID["InputFileID"]
except Exception as e:
print("ERROR: GEN_UML: " + str(e))
inputFile_ID = "x"
print("INFO_GEN_UML: " + inputFile_ID)
# File IDs to FileName translation
sourceFileName = dc.translateFileidToFilename(inputFile_ID)
print("INFO_GEN_UML: FileName on Server: " + sourceFileName)
if sourceFileName != False:
sourcefile = os.path.join(sourceFilePath, sourceFileName)
else:
print(
"ERROR: GEN_UML: Configuration(source code) with the given FileID parameter not found"
)
status = "Configuration(source code) with the given FileID parameter not found"
httpCode = 404
res = createRequestDetails("N/A", inputFile_ID, "N/A", status,
httpCode, REQUEST_ID)
return res
# Call the service.
tool = TOOL_PATH
# the cmd script takes in two parameters Input .TXT file and the path to
# the output directory which in this case is the "USER FILES DIRECTORY"
#PARAMS = sourcefile + " " + OUTPUT_DIR
#cmd = tool + " " + PARAMS
try:
subprocess.call(
['java', '-jar', tool, sourcefile, "-o", OUTPUT_DIR])
except Exception as e:
print("ERROR: GEN_UML: " + str(e))
status = "Error(s) encountered in executing the service."
httpCode = 500
res = createRequestDetails("N/A", inputFile_ID, "N/A", status,
httpCode, REQUEST_ID)
return res
#processing on generate file
#Plant uml names the generated file the same as input source file.
fileName, md5chkSum = postProcessing(sourceFileName, userFilesDir)
if fileName != False:
print("INFO: GEN_UML: Service executed successfully.")
status = "UML diagram generated successfully."
Generated_fileID = str(fileNameToHash(REQUEST_ID + getTimeStamp()))
httpCode = 201
res = createRequestDetails(fileName, Generated_fileID, md5chkSum,
status, httpCode, REQUEST_ID)
if createFileDetails(Generated_fileID, fileName, accessID):
res = make_response(res, httpCode)
return res
else:
print("ERROR: GEN_UML: Error Saving file to server")
status = "[ERR] Error saving file to disk."
httpCode = 500
res["Status"] = status
res["HttpResponse"] = httpCode
res = make_response(res, httpCode)
#abort(500, "Error saving file to disk")
return res
else:
print(
"ERROR: GEN_UML: Error(s) encountered in post processing the results."
)
status = "Error(s) encountered in post processing the results."
httpCode = 500
res = createRequestDetails(fileName, inputFile_ID, "N/A", status,
httpCode, REQUEST_ID)
return res
else:
print("ERROR: GEN_UML: Invalid Access-ID, request denied.")
status = "[ERR] Invalid Access-ID, request denied."
httpCode = 403
res = createRequestDetails("N/A", "N/A", "N/A", status, httpCode,
REQUEST_ID)
res = make_response(res, httpCode)
return res
def signup_cli():
# declare as global to update the original variable.
global REQUEST_ID
# generate a new id for each request.
REQUEST_ID = str(uuid.uuid4())
if isInitSuccessful():
try:
userData = request.json
except Exception as e:
print("ERROR: Signup: " + str(e))
status = "[ERR] Bad request, json file not found in request"
httpCode = 400
res = createRequestDetails(status, httpCode, REQUEST_ID)
return res
if request.method == 'POST':
#TODO: verification email.
try:
newUserData = {
"fName": userData["fName"],
"lName": userData["lName"],
"fullName": userData["fName"] + " " + userData["lName"],
"company": userData["company"],
"department": userData["department"],
"emailAddr": userData["emailAddr"],
"userName": userData["userName"],
"password": userData["password"]
}
except Exception as e:
print("ERROR: Signup: " + str(e))
status = "[ERR] Bad request, missing/in-valid data in json file"
httpCode = 400
res = createRequestDetails(status, httpCode, REQUEST_ID)
return res
# 1- Generate a new API-key for the user.
try:
genApiKey = dc.addNewApiKey(newUserData)
except Exception as e:
print(str(e))
abort(500, "ERROR")
#2- Create a dedicated workspace for the new user on server.
Path("../" + CONSTANTS.DATABASE_DIR_NAME + "/" +
str(genApiKey)).mkdir(parents=True, exist_ok=True)
#2.1 - Unzip template workspace
thread = Thread(target=prepareWorkSpace, args=(genApiKey, ))
thread.start()
#3- Send a confirmation email with details.
#4- Redirect to "Success" URL/workspace
thread.join()
print("INFO: Signup: New workspace prepared successfuly.")
print("INFO: Signup: Registration Complete")
status = "Registration process complete,New user's API Key: " + str(
genApiKey)
httpCode = 201
res = createRequestDetails(status, httpCode, REQUEST_ID)
return res
#return redirect(url_for('signup_bp.home'))
else:
welcomeString = "ERROR : INIT FAILED : SignUp "
res = make_response(welcomeString, 500)
return res
def validateCompuMethods():
# declare as global to update the original variable.
global REQUEST_ID
# generate a new id for each request.
REQUEST_ID = str(uuid.uuid4())
accessID = getAccessId()
if isAccessIdAuthentic(accessID):
#########Get the File IDs from the request parameters#########
inputFileID = request.json
userFilesDir = dc.getUserFilesDir(accessID)
try:
#Get the File IDs from the request parameters
xl_FileID = inputFileID["xlFileID"]
except Exception as e:
print("ERROR: CM_Validation: " + str(e))
xl_FileID = "x"
try:
a2l_FileID = inputFileID["a2lFileID"]
except Exception as e:
print("ERROR: CM_Validation: " + str(e))
a2l_FileID = "x"
#########File IDs to FileName translation#########
fileName = dc.translateFileidToFilename(xl_FileID)
if fileName != False:
UPLOAD_XSLX_PATH = dc.getExcelUploadPath(accessID)
xl_file = os.path.join(UPLOAD_XSLX_PATH, fileName)
print(xl_file)
else:
print(
"ERROR: CM_Validation: Excel file with the given FileID parameter not found"
)
status = "Excel file with the given FileID parameter not found"
httpCode = 404
res = createRequestDetails("404", xl_FileID, "404", status,
httpCode, REQUEST_ID)
return res
fileName = dc.translateFileidToFilename(a2l_FileID)
if fileName != False:
UPLOAD_A2L_PATH = dc.getA2lUploadPath(accessID)
a2l_file = os.path.join(UPLOAD_A2L_PATH, fileName)
print(a2l_file)
else:
print(
"ERROR: CM_Validation: a2l file with the given FileID parameter not found"
)
status = "a2l file with the given FileID parameter not found"
httpCode = 404
res = createRequestDetails("404", a2l_FileID, "404", status,
httpCode, REQUEST_ID)
return res
OUT_DIR = CMConst.OUTPUT_DIR
# Call the service.
tool = CMConst.CLI_INTERFACE
PARAMS = tool + " " + a2l_file + " " + xl_file + " " + OUT_DIR + " " + OUT_DIR
#cmd = PYTHON_PATH + " " + tool + " " + PARAMS
try:
#subprocess.Popen(cmd)
subprocess.call([tool, a2l_file, xl_file, OUT_DIR, OUT_DIR])
except Exception as e:
print("ERROR: CM_Validation: " + str(e))
status = "Error(s) encountered in executing the service."
httpCode = 500
res = createRequestDetails("404", "404", "404", status, httpCode,
REQUEST_ID)
return res
fileName, md5chkSum = postProcessing(userFilesDir)
if fileName != False:
print("INFO: CM_Validation: Service executed successfully")
status = "Service executed successfully."
httpCode = 201
Generated_fileID = str(fileNameToHash(REQUEST_ID + getTimeStamp()))
res = createRequestDetails(fileName, Generated_fileID, md5chkSum,
status, httpCode, REQUEST_ID)
if createFileDetails(Generated_fileID, fileName, accessID):
res = make_response(res, httpCode)
return res
else:
print("ERROR: CM_Validation: Error Saving results to server")
status = "[ERR] Error saving file to disk."
httpCode = 500
res["Status"] = status
res["HttpResponse"] = httpCode
res = make_response(res, httpCode)
#abort(500, "Error saving file to disk")
return res
else:
status = "Error(s) encountered in post processing the results."
httpCode = 500
res = createRequestDetails("404", "404", "404", status, httpCode,
REQUEST_ID)
return res
else:
print("ERROR: CM_Validation: Invalid Access-ID, request denied.")
status = "[ERR] Invalid Access-ID, request denied."
httpCode = 403
res = createRequestDetails("xxx", "xxx", "xxx", status, httpCode,
REQUEST_ID)
res = make_response(res, httpCode)
return res
UPLOAD_JSON_PATH = CONSTANTS.USER_UPLOADS_JSON_DIR
UPLOAD_A2L_PATH = CONSTANTS.USER_UPLOADS_A2L_DIR
UPLOAD_TXT_PATH = CONSTANTS.USER_UPLOADS_TEXT_DIR
SUPPORTED_EXTENSIONS = CONSTANTS.SUPPORTED_EXTENSIONS_UPLOAD
MAX_UPLOAD_FILE_SIZE = CONSTANTS.MAX_UPLOAD_FILE_SIZE
################################ Init Databases ##############################
try:
from application import Data_Controller as dc
print("INFO: FILEUPLOAD: DB Controller initialized successfully.")
except:
import Data_Controller as dc
print("DEBUG: FILEUPLOAD: Safe Mode: Imported alternative DBC.")
API_KEY_DB = dc.getApiKeysFromDb()
REQ_HISTORY = dc.getRequestHistory()
def isInitSuccessful():
if API_KEY_DB == False:
return False
if REQ_HISTORY == False:
return False
return True
###############################################################################
"""
Returns timeStamp at the time of function call
"""
def getTimeStamp():
return datetime.now().strftime(("%Y-%m-%d %H:%M:%S"))