def validate_threaded(): """Start the validation process on a new thread.""" @copy_current_request_context def ThreadedFunction(arg): """The new thread.""" threadedManager = ValidationManager(local, error_report_path) threadedManager.threadedValidateJob(arg) try: interfaces = InterfaceHolder() jobTracker = interfaces.jobDb except ResponseException as e: open("errorLog","a").write(str(e) + "\n") return JsonResponse.error(e,e.status,table = "cannot connect to job database") except Exception as e: open("errorLog","a").write(str(e) + "\n") exc = ResponseException(str(e),StatusCode.INTERNAL_ERROR,type(e)) return JsonResponse.error(exc,exc.status,table= "cannot connect to job database") jobId = None manager = ValidationManager(local, error_report_path) try: jobId = manager.getJobID(request) except ResponseException as e: manager.markJob(jobId,jobTracker,"invalid",interfaces.errorDb,manager.filename) CloudLogger.logError(str(e),e,traceback.extract_tb(sys.exc_info()[2])) return JsonResponse.error(e,e.status,table ="") except Exception as e: exc = ResponseException(str(e),StatusCode.CLIENT_ERROR,type(e)) manager.markJob(jobId,jobTracker,"invalid",interfaces.errorDb,manager.filename) CloudLogger.logError(str(e),exc,traceback.extract_tb(sys.exc_info()[2])) return JsonResponse.error(exc,exc.status,table="") try: manager.testJobID(jobId,interfaces) except ResponseException as e: open("errorLog","a").write(str(e) + "\n") # Job is not ready to run according to job tracker, do not change status of job in job tracker interfaces.errorDb.writeFileError(jobId,manager.filename,ValidationError.jobError) return JsonResponse.error(e,e.status,table="") except Exception as e: open("errorLog","a").write(str(e) + "\n") exc = ResponseException(str(e),StatusCode.CLIENT_ERROR,type(e)) interfaces.errorDb.writeFileError(jobId,manager.filename,ValidationError.jobError) return JsonResponse.error(exc,exc.status,table="") thread = Thread(target=ThreadedFunction, args= (jobId,)) try : jobTracker.markJobStatus(jobId,"running") except Exception as e: open("errorLog","a").write(str(e) + "\n") exc = ResponseException(str(e),StatusCode.INTERNAL_ERROR,type(e)) return JsonResponse.error(exc,exc.status,table="could not start job") interfaces.close() thread.start() return JsonResponse.create(StatusCode.OK,{"table":"job"+str(jobId)})
def validate(): """Start the validation process on the same threads.""" interfaces = InterfaceHolder() # Create sessions for this route try: return validationManager.validateJob(request,interfaces) except Exception as e: # Something went wrong getting the flask request open("errorLog","a").write(str(e) + "\n") exc = ResponseException(str(e),StatusCode.INTERNAL_ERROR,type(e)) return JsonResponse.error(exc,exc.status,table="") finally: interfaces.close()
def threadedValidateJob(self, jobId): """ args jobId -- (Integer) a valid jobId This method runs on a new thread thus there are zero error messages other then the job status being updated """ # As this is the start of a new thread, first generate new connections to the databases interfaces = InterfaceHolder() self.filename = "" jobTracker = interfaces.jobDb errorDb = interfaces.errorDb try: jobType = interfaces.jobDb.checkJobType(jobId) if jobType == interfaces.jobDb.getJobTypeId( "csv_record_validation"): self.runValidation(jobId, interfaces) elif jobType == interfaces.jobDb.getJobTypeId("validation"): self.runCrossValidation(jobId, interfaces) else: raise ResponseException("Bad job type for validator", StatusCode.INTERNAL_ERROR) self.runValidation(jobId, interfaces) errorDb.markFileComplete(jobId, self.filename) return except ResponseException as e: CloudLogger.logError(str(e), e, traceback.extract_tb(sys.exc_info()[2])) self.markJob(jobId, jobTracker, "invalid", errorDb, self.filename, e.errorType, e.extraInfo) except ValueError as e: CloudLogger.logError(str(e), e, traceback.extract_tb(sys.exc_info()[2])) self.markJob(jobId, jobTracker, "invalid", errorDb, self.filename, ValidationError.unknownError) except Exception as e: #Something unknown happened we may need to try again! CloudLogger.logError(str(e), e, traceback.extract_tb(sys.exc_info()[2])) self.markJob(jobId, jobTracker, "failed", errorDb, self.filename, ValidationError.unknownError) finally: interfaces.close()
def threadedValidateJob(self,jobId) : """ args jobId -- (Integer) a valid jobId This method runs on a new thread thus there are zero error messages other then the job status being updated """ # As this is the start of a new thread, first generate new connections to the databases interfaces = InterfaceHolder() self.filename = "" jobTracker = interfaces.jobDb errorDb = interfaces.errorDb try: jobType = interfaces.jobDb.checkJobType(jobId) if jobType == interfaces.jobDb.getJobTypeId("csv_record_validation"): self.runValidation(jobId,interfaces) elif jobType == interfaces.jobDb.getJobTypeId("validation"): self.runCrossValidation(jobId, interfaces) else: raise ResponseException("Bad job type for validator", StatusCode.INTERNAL_ERROR) self.runValidation(jobId, interfaces) errorDb.markFileComplete(jobId,self.filename) return except ResponseException as e: CloudLogger.logError(str(e),e,traceback.extract_tb(sys.exc_info()[2])) self.markJob(jobId,jobTracker,"invalid",errorDb,self.filename,e.errorType,e.extraInfo) except ValueError as e: CloudLogger.logError(str(e),e,traceback.extract_tb(sys.exc_info()[2])) self.markJob(jobId,jobTracker,"invalid",errorDb,self.filename,ValidationError.unknownError) except Exception as e: #Something unknown happened we may need to try again! CloudLogger.logError(str(e),e,traceback.extract_tb(sys.exc_info()[2])) self.markJob(jobId,jobTracker,"failed",errorDb,self.filename,ValidationError.unknownError) finally: interfaces.close()