def validate_threaded():
"""Start the validation process on a new thread."""
@copy_current_request_context
def ThreadedFunction(arg):
"""The new thread."""
threadedManager = ValidationManager(local, error_report_path)
threadedManager.threadedValidateJob(arg)
try:
interfaces = InterfaceHolder()
jobTracker = interfaces.jobDb
except ResponseException as e:
open("errorLog","a").write(str(e) + "\n")
return JsonResponse.error(e,e.status,table = "cannot connect to job database")
except Exception as e:
open("errorLog","a").write(str(e) + "\n")
exc = ResponseException(str(e),StatusCode.INTERNAL_ERROR,type(e))
return JsonResponse.error(exc,exc.status,table= "cannot connect to job database")
jobId = None
manager = ValidationManager(local, error_report_path)
try:
jobId = manager.getJobID(request)
except ResponseException as e:
manager.markJob(jobId,jobTracker,"invalid",interfaces.errorDb,manager.filename)
CloudLogger.logError(str(e),e,traceback.extract_tb(sys.exc_info()[2]))
return JsonResponse.error(e,e.status,table ="")
except Exception as e:
exc = ResponseException(str(e),StatusCode.CLIENT_ERROR,type(e))
manager.markJob(jobId,jobTracker,"invalid",interfaces.errorDb,manager.filename)
CloudLogger.logError(str(e),exc,traceback.extract_tb(sys.exc_info()[2]))
return JsonResponse.error(exc,exc.status,table="")
try:
manager.testJobID(jobId,interfaces)
except ResponseException as e:
open("errorLog","a").write(str(e) + "\n")
# Job is not ready to run according to job tracker, do not change status of job in job tracker
interfaces.errorDb.writeFileError(jobId,manager.filename,ValidationError.jobError)
return JsonResponse.error(e,e.status,table="")
except Exception as e:
open("errorLog","a").write(str(e) + "\n")
exc = ResponseException(str(e),StatusCode.CLIENT_ERROR,type(e))
interfaces.errorDb.writeFileError(jobId,manager.filename,ValidationError.jobError)
return JsonResponse.error(exc,exc.status,table="")
thread = Thread(target=ThreadedFunction, args= (jobId,))
try :
jobTracker.markJobStatus(jobId,"running")
except Exception as e:
open("errorLog","a").write(str(e) + "\n")
exc = ResponseException(str(e),StatusCode.INTERNAL_ERROR,type(e))
return JsonResponse.error(exc,exc.status,table="could not start job")
interfaces.close()
thread.start()
return JsonResponse.create(StatusCode.OK,{"table":"job"+str(jobId)})
def validate():
"""Start the validation process on the same threads."""
interfaces = InterfaceHolder() # Create sessions for this route
try:
return validationManager.validateJob(request,interfaces)
except Exception as e:
# Something went wrong getting the flask request
open("errorLog","a").write(str(e) + "\n")
exc = ResponseException(str(e),StatusCode.INTERNAL_ERROR,type(e))
return JsonResponse.error(exc,exc.status,table="")
finally:
interfaces.close()
def threadedValidateJob(self, jobId):
"""
args
jobId -- (Integer) a valid jobId
This method runs on a new thread thus
there are zero error messages other then the
job status being updated
"""
# As this is the start of a new thread, first generate new connections to the databases
interfaces = InterfaceHolder()
self.filename = ""
jobTracker = interfaces.jobDb
errorDb = interfaces.errorDb
try:
jobType = interfaces.jobDb.checkJobType(jobId)
if jobType == interfaces.jobDb.getJobTypeId(
"csv_record_validation"):
self.runValidation(jobId, interfaces)
elif jobType == interfaces.jobDb.getJobTypeId("validation"):
self.runCrossValidation(jobId, interfaces)
else:
raise ResponseException("Bad job type for validator",
StatusCode.INTERNAL_ERROR)
self.runValidation(jobId, interfaces)
errorDb.markFileComplete(jobId, self.filename)
return
except ResponseException as e:
CloudLogger.logError(str(e), e,
traceback.extract_tb(sys.exc_info()[2]))
self.markJob(jobId, jobTracker, "invalid", errorDb, self.filename,
e.errorType, e.extraInfo)
except ValueError as e:
CloudLogger.logError(str(e), e,
traceback.extract_tb(sys.exc_info()[2]))
self.markJob(jobId, jobTracker, "invalid", errorDb, self.filename,
ValidationError.unknownError)
except Exception as e:
#Something unknown happened we may need to try again!
CloudLogger.logError(str(e), e,
traceback.extract_tb(sys.exc_info()[2]))
self.markJob(jobId, jobTracker, "failed", errorDb, self.filename,
ValidationError.unknownError)
finally:
interfaces.close()
def threadedValidateJob(self,jobId) :
"""
args
jobId -- (Integer) a valid jobId
This method runs on a new thread thus
there are zero error messages other then the
job status being updated
"""
# As this is the start of a new thread, first generate new connections to the databases
interfaces = InterfaceHolder()
self.filename = ""
jobTracker = interfaces.jobDb
errorDb = interfaces.errorDb
try:
jobType = interfaces.jobDb.checkJobType(jobId)
if jobType == interfaces.jobDb.getJobTypeId("csv_record_validation"):
self.runValidation(jobId,interfaces)
elif jobType == interfaces.jobDb.getJobTypeId("validation"):
self.runCrossValidation(jobId, interfaces)
else:
raise ResponseException("Bad job type for validator", StatusCode.INTERNAL_ERROR)
self.runValidation(jobId, interfaces)
errorDb.markFileComplete(jobId,self.filename)
return
except ResponseException as e:
CloudLogger.logError(str(e),e,traceback.extract_tb(sys.exc_info()[2]))
self.markJob(jobId,jobTracker,"invalid",errorDb,self.filename,e.errorType,e.extraInfo)
except ValueError as e:
CloudLogger.logError(str(e),e,traceback.extract_tb(sys.exc_info()[2]))
self.markJob(jobId,jobTracker,"invalid",errorDb,self.filename,ValidationError.unknownError)
except Exception as e:
#Something unknown happened we may need to try again!
CloudLogger.logError(str(e),e,traceback.extract_tb(sys.exc_info()[2]))
self.markJob(jobId,jobTracker,"failed",errorDb,self.filename,ValidationError.unknownError)
finally:
interfaces.close()
