def checkJobDependencies(self, jobId):
# raise exception if current job is not actually finished
if self.getStatus(jobId) != self.getStatusId('finished'):
raise ValueError(
'Current job not finished, unable to check dependencies')
# check if dependent jobs are finished
for depJobId in self.getDependentJobs(jobId):
isReady = True
if not (self.getStatus(depJobId) == self.getStatusId('waiting')):
CloudLogger.logError(
"Job dependency is not in a 'waiting' state",
ResponseException(
"Job dependency is not in a 'waiting' state",
StatusCode.CLIENT_ERROR, ValueError),
traceback.extract_stack())
continue
# if dependent jobs are finished, then check the jobs of which the current job is a dependent
for preReqJobId in self.getPrerequisiteJobs(depJobId):
if not (self.getStatus(preReqJobId)
== self.getStatusId('finished')):
# Do nothing
isReady = False
break
# The type check here is temporary and needs to be removed once the validator is able
# to handle cross-file validation job
if isReady and (self.getJobType(depJobId)
== 'csv_record_validation'
or self.getJobType(depJobId) == 'validation'):
# mark job as ready
self.markStatus(depJobId, 'ready')
# add to the job queue
jobQueueResult = self.jobQueue.enqueue.delay(depJobId)
def closeOne(interface):
""" Close all aspects of one interface """
if(interface == None):
# No need to close a nonexistent connection
return
try:
if(interface.session == None):
# If session is None, skip closing
return
except AttributeError as e:
# If interface has no session, skip closing
return
# Try to close the session and connection, on error try a rollback
try:
interface.session.close()
except:
try:
interface.session.rollback()
interface.session.close()
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
trace = traceback.extract_tb(exc_tb, 10)
CloudLogger.logError('Broker DB Interface Error: ', e, trace)
del exc_tb
raise
def closeOne(interface):
""" Close all aspects of one interface """
if (interface == None):
# No need to close a nonexistent connection
return
try:
if (interface.session == None):
# If session is None, skip closing
return
except AttributeError as e:
# If interface has no session, skip closing
return
# Try to close the session and connection, on error try a rollback
try:
interface.session.close()
except:
try:
interface.session.rollback()
interface.session.close()
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
trace = traceback.extract_tb(exc_tb, 10)
CloudLogger.logError('Broker DB Interface Error: ', e, trace)
del exc_tb
raise
def validate_threaded():
"""Start the validation process on a new thread."""
@copy_current_request_context
def ThreadedFunction(arg):
"""The new thread."""
threadedManager = ValidationManager(local, error_report_path)
threadedManager.threadedValidateJob(arg)
try:
interfaces = InterfaceHolder()
jobTracker = interfaces.jobDb
except ResponseException as e:
open("errorLog","a").write(str(e) + "\n")
return JsonResponse.error(e,e.status,table = "cannot connect to job database")
except Exception as e:
open("errorLog","a").write(str(e) + "\n")
exc = ResponseException(str(e),StatusCode.INTERNAL_ERROR,type(e))
return JsonResponse.error(exc,exc.status,table= "cannot connect to job database")
jobId = None
manager = ValidationManager(local, error_report_path)
try:
jobId = manager.getJobID(request)
except ResponseException as e:
manager.markJob(jobId,jobTracker,"invalid",interfaces.errorDb,manager.filename)
CloudLogger.logError(str(e),e,traceback.extract_tb(sys.exc_info()[2]))
return JsonResponse.error(e,e.status,table ="")
except Exception as e:
exc = ResponseException(str(e),StatusCode.CLIENT_ERROR,type(e))
manager.markJob(jobId,jobTracker,"invalid",interfaces.errorDb,manager.filename)
CloudLogger.logError(str(e),exc,traceback.extract_tb(sys.exc_info()[2]))
return JsonResponse.error(exc,exc.status,table="")
try:
manager.testJobID(jobId,interfaces)
except ResponseException as e:
open("errorLog","a").write(str(e) + "\n")
# Job is not ready to run according to job tracker, do not change status of job in job tracker
interfaces.errorDb.writeFileError(jobId,manager.filename,ValidationError.jobError)
return JsonResponse.error(e,e.status,table="")
except Exception as e:
open("errorLog","a").write(str(e) + "\n")
exc = ResponseException(str(e),StatusCode.CLIENT_ERROR,type(e))
interfaces.errorDb.writeFileError(jobId,manager.filename,ValidationError.jobError)
return JsonResponse.error(exc,exc.status,table="")
thread = Thread(target=ThreadedFunction, args= (jobId,))
try :
jobTracker.markJobStatus(jobId,"running")
except Exception as e:
open("errorLog","a").write(str(e) + "\n")
exc = ResponseException(str(e),StatusCode.INTERNAL_ERROR,type(e))
return JsonResponse.error(exc,exc.status,table="could not start job")
interfaces.close()
thread.start()
return JsonResponse.create(StatusCode.OK,{"table":"job"+str(jobId)})
def checkJobDependencies(self,jobId):
""" For specified job, check which of its dependencies are ready to be started, and add them to the queue """
# raise exception if current job is not actually finished
if self.getJobStatus(jobId) != self.getJobStatusId('finished'):
raise ValueError('Current job not finished, unable to check dependencies')
# check if dependent jobs are finished
for depJobId in self.getDependentJobs(jobId):
isReady = True
if not (self.getJobStatus(depJobId) == self.getJobStatusId('waiting')):
CloudLogger.logError("Job dependency is not in a 'waiting' state",
ResponseException("Job dependency is not in a 'waiting' state",StatusCode.CLIENT_ERROR, ValueError),
traceback.extract_stack())
continue
# if dependent jobs are finished, then check the jobs of which the current job is a dependent
for preReqJobId in self.getPrerequisiteJobs(depJobId):
if not (self.getJobStatus(preReqJobId) == self.getJobStatusId('finished')):
# Do nothing
isReady = False
break
# The type check here is temporary and needs to be removed once the validator is able
# to handle cross-file validation job
if isReady and (self.getJobType(depJobId) == 'csv_record_validation' or self.getJobType(depJobId) == 'validation'):
# mark job as ready
self.markJobStatus(depJobId, 'ready')
# add to the job queue
jobQueueResult = self.jobQueue.enqueue.delay(depJobId)
def error(exception, errorCode, **kwargs):
""" Create an http response object for specified error
Args:
exception: Exception to be represented by response object
errorCode: Status code to be used in response
kwargs: Extra fields and values to be included in response
Returns:
Http response object containing specified error
"""
responseDict = {}
for key in kwargs:
responseDict[key] = kwargs[key]
exc_type, exc_obj, exc_tb = sys.exc_info()
trace = traceback.extract_tb(exc_tb, 10)
CloudLogger.logError('Route Error : ', exception, trace)
if (JsonResponse.debugMode):
responseDict["message"] = str(exception)
responseDict["errorType"] = str(type(exception))
if (type(exception) == type(ResponseException(""))
and exception.wrappedException != None):
responseDict["wrappedType"] = str(
type(exception.wrappedException))
responseDict["wrappedMessage"] = str(
exception.wrappedException)
trace = list(map(lambda entry: str(entry), trace))
responseDict["trace"] = trace
if (JsonResponse.printDebug):
print(str(type(exception)))
print(str(exception))
print(str(trace))
if (JsonResponse.logDebug):
open("responseErrorLog",
"a").write(str(type(exception)) + ": ")
open("responseErrorLog", "a").write(str(exception) + "\n")
open("responseErrorLog", "a").write(str(trace) + "\n")
del exc_tb
return JsonResponse.create(errorCode, responseDict)
else:
responseDict["message"] = "An error has occurred"
del exc_tb
return JsonResponse.create(errorCode, responseDict)
def checkJobDependencies(self, jobId):
""" For specified job, check which of its dependencies are ready to be started, and add them to the queue """
# raise exception if current job is not actually finished
if self.getJobStatus(jobId) != self.getJobStatusId('finished'):
raise ValueError(
'Current job not finished, unable to check dependencies')
# check if dependent jobs are finished
for depJobId in self.getDependentJobs(jobId):
isReady = True
if not (self.getJobStatus(depJobId)
== self.getJobStatusId('waiting')):
CloudLogger.logError(
"Job dependency is not in a 'waiting' state",
ResponseException(
"Job dependency is not in a 'waiting' state",
StatusCode.CLIENT_ERROR, ValueError),
traceback.extract_stack())
continue
# if dependent jobs are finished, then check the jobs of which the current job is a dependent
for preReqJobId in self.getPrerequisiteJobs(depJobId):
if not (self.getJobStatus(preReqJobId)
== self.getJobStatusId('finished')):
# Do nothing
isReady = False
break
# The type check here is temporary and needs to be removed once the validator is able
# to handle cross-file validation job
if isReady and (self.getJobType(depJobId)
== 'csv_record_validation'
or self.getJobType(depJobId) == 'validation'):
# mark job as ready
self.markJobStatus(depJobId, 'ready')
# add to the job queue
CloudLogger.log("Sending job {} to the job manager".format(
str(depJobId)))
mp = ManagerProxy()
mp.sendJobRequest(depJobId)
def error(exception, errorCode, **kwargs):
""" Create an http response object for specified error
Args:
exception: Exception to be represented by response object
errorCode: Status code to be used in response
kwargs: Extra fields and values to be included in response
Returns:
Http response object containing specified error
"""
responseDict = {}
for key in kwargs:
responseDict[key] = kwargs[key]
exc_type, exc_obj, exc_tb = sys.exc_info()
trace = traceback.extract_tb(exc_tb, 10)
CloudLogger.logError("Route Error : ", exception, trace)
if JsonResponse.debugMode:
responseDict["message"] = str(exception)
responseDict["errorType"] = str(type(exception))
if type(exception) == type(ResponseException("")) and exception.wrappedException != None:
responseDict["wrappedType"] = str(type(exception.wrappedException))
responseDict["wrappedMessage"] = str(exception.wrappedException)
responseDict["trace"] = trace
if JsonResponse.printDebug:
print(str(type(exception)))
print(str(exception))
print(str(trace))
if JsonResponse.logDebug:
open("responseErrorLog", "a").write(str(type(exception)) + ": ")
open("responseErrorLog", "a").write(str(exception) + "\n")
open("responseErrorLog", "a").write(str(trace) + "\n")
del exc_tb
return JsonResponse.create(errorCode, responseDict)
else:
responseDict["message"] = "An error has occurred"
del exc_tb
return JsonResponse.create(errorCode, responseDict)
def threadedValidateJob(self, jobId):
"""
args
jobId -- (Integer) a valid jobId
This method runs on a new thread thus
there are zero error messages other then the
job status being updated
"""
# As this is the start of a new thread, first generate new connections to the databases
interfaces = InterfaceHolder()
self.filename = ""
jobTracker = interfaces.jobDb
errorDb = interfaces.errorDb
try:
jobType = interfaces.jobDb.checkJobType(jobId)
if jobType == interfaces.jobDb.getJobTypeId(
"csv_record_validation"):
self.runValidation(jobId, interfaces)
elif jobType == interfaces.jobDb.getJobTypeId("validation"):
self.runCrossValidation(jobId, interfaces)
else:
raise ResponseException("Bad job type for validator",
StatusCode.INTERNAL_ERROR)
self.runValidation(jobId, interfaces)
errorDb.markFileComplete(jobId, self.filename)
return
except ResponseException as e:
CloudLogger.logError(str(e), e,
traceback.extract_tb(sys.exc_info()[2]))
self.markJob(jobId, jobTracker, "invalid", errorDb, self.filename,
e.errorType, e.extraInfo)
except ValueError as e:
CloudLogger.logError(str(e), e,
traceback.extract_tb(sys.exc_info()[2]))
self.markJob(jobId, jobTracker, "invalid", errorDb, self.filename,
ValidationError.unknownError)
except Exception as e:
#Something unknown happened we may need to try again!
CloudLogger.logError(str(e), e,
traceback.extract_tb(sys.exc_info()[2]))
self.markJob(jobId, jobTracker, "failed", errorDb, self.filename,
ValidationError.unknownError)
finally:
interfaces.close()
def threadedValidateJob(self,jobId) :
"""
args
jobId -- (Integer) a valid jobId
This method runs on a new thread thus
there are zero error messages other then the
job status being updated
"""
# As this is the start of a new thread, first generate new connections to the databases
interfaces = InterfaceHolder()
self.filename = ""
jobTracker = interfaces.jobDb
errorDb = interfaces.errorDb
try:
jobType = interfaces.jobDb.checkJobType(jobId)
if jobType == interfaces.jobDb.getJobTypeId("csv_record_validation"):
self.runValidation(jobId,interfaces)
elif jobType == interfaces.jobDb.getJobTypeId("validation"):
self.runCrossValidation(jobId, interfaces)
else:
raise ResponseException("Bad job type for validator", StatusCode.INTERNAL_ERROR)
self.runValidation(jobId, interfaces)
errorDb.markFileComplete(jobId,self.filename)
return
except ResponseException as e:
CloudLogger.logError(str(e),e,traceback.extract_tb(sys.exc_info()[2]))
self.markJob(jobId,jobTracker,"invalid",errorDb,self.filename,e.errorType,e.extraInfo)
except ValueError as e:
CloudLogger.logError(str(e),e,traceback.extract_tb(sys.exc_info()[2]))
self.markJob(jobId,jobTracker,"invalid",errorDb,self.filename,ValidationError.unknownError)
except Exception as e:
#Something unknown happened we may need to try again!
CloudLogger.logError(str(e),e,traceback.extract_tb(sys.exc_info()[2]))
self.markJob(jobId,jobTracker,"failed",errorDb,self.filename,ValidationError.unknownError)
finally:
interfaces.close()
def createApp():
"""Create the Flask app."""
try:
app = Flask(__name__)
local = CONFIG_BROKER['local']
error_report_path = CONFIG_SERVICES['error_report_path']
app.config.from_object(__name__)
# Future: Override config w/ environment variable, if set
app.config.from_envvar('VALIDATOR_SETTINGS', silent=True)
validationManager = ValidationManager(local, error_report_path)
@app.route("/", methods=["GET"])
def testApp():
"""Confirm server running."""
return "Validator is running"
@app.route("/validate_threaded/", methods=["POST"])
def validate_threaded():
"""Start the validation process on a new thread."""
@copy_current_request_context
def ThreadedFunction(arg):
"""The new thread."""
threadedManager = ValidationManager(local, error_report_path)
threadedManager.threadedValidateJob(arg)
try:
interfaces = InterfaceHolder()
jobTracker = interfaces.jobDb
except ResponseException as e:
open("errorLog","a").write(str(e) + "\n")
return JsonResponse.error(e,e.status,table = "cannot connect to job database")
except Exception as e:
open("errorLog","a").write(str(e) + "\n")
exc = ResponseException(str(e),StatusCode.INTERNAL_ERROR,type(e))
return JsonResponse.error(exc,exc.status,table= "cannot connect to job database")
jobId = None
manager = ValidationManager(local, error_report_path)
try:
jobId = manager.getJobID(request)
except ResponseException as e:
manager.markJob(jobId,jobTracker,"invalid",interfaces.errorDb,manager.filename)
CloudLogger.logError(str(e),e,traceback.extract_tb(sys.exc_info()[2]))
return JsonResponse.error(e,e.status,table ="")
except Exception as e:
exc = ResponseException(str(e),StatusCode.CLIENT_ERROR,type(e))
manager.markJob(jobId,jobTracker,"invalid",interfaces.errorDb,manager.filename)
CloudLogger.logError(str(e),exc,traceback.extract_tb(sys.exc_info()[2]))
return JsonResponse.error(exc,exc.status,table="")
try:
manager.testJobID(jobId,interfaces)
except ResponseException as e:
open("errorLog","a").write(str(e) + "\n")
# Job is not ready to run according to job tracker, do not change status of job in job tracker
interfaces.errorDb.writeFileError(jobId,manager.filename,ValidationError.jobError)
return JsonResponse.error(e,e.status,table="")
except Exception as e:
open("errorLog","a").write(str(e) + "\n")
exc = ResponseException(str(e),StatusCode.CLIENT_ERROR,type(e))
interfaces.errorDb.writeFileError(jobId,manager.filename,ValidationError.jobError)
return JsonResponse.error(exc,exc.status,table="")
thread = Thread(target=ThreadedFunction, args= (jobId,))
try :
jobTracker.markJobStatus(jobId,"running")
except Exception as e:
open("errorLog","a").write(str(e) + "\n")
exc = ResponseException(str(e),StatusCode.INTERNAL_ERROR,type(e))
return JsonResponse.error(exc,exc.status,table="could not start job")
interfaces.close()
thread.start()
return JsonResponse.create(StatusCode.OK,{"table":"job"+str(jobId)})
@app.route("/validate/",methods=["POST"])
def validate():
"""Start the validation process on the same threads."""
interfaces = InterfaceHolder() # Create sessions for this route
try:
return validationManager.validateJob(request,interfaces)
except Exception as e:
# Something went wrong getting the flask request
open("errorLog","a").write(str(e) + "\n")
exc = ResponseException(str(e),StatusCode.INTERNAL_ERROR,type(e))
return JsonResponse.error(exc,exc.status,table="")
finally:
interfaces.close()
JsonResponse.debugMode = CONFIG_SERVICES['rest_trace']
return app
except Exception as e:
trace = traceback.extract_tb(sys.exc_info()[2], 10)
CloudLogger.logError('Validator App Level Error: ', e, trace)
raise
def createApp():
"""Create the Flask app."""
try:
app = Flask(__name__)
local = CONFIG_BROKER['local']
error_report_path = CONFIG_SERVICES['error_report_path']
app.config.from_object(__name__)
# Future: Override config w/ environment variable, if set
app.config.from_envvar('VALIDATOR_SETTINGS', silent=True)
validationManager = ValidationManager(local, error_report_path)
@app.route("/", methods=["GET"])
def testApp():
"""Confirm server running."""
return "Validator is running"
@app.route("/validate_threaded/", methods=["POST"])
def validate_threaded():
"""Start the validation process on a new thread."""
@copy_current_request_context
def ThreadedFunction(arg):
"""The new thread."""
threadedManager = ValidationManager(local, error_report_path)
threadedManager.threadedValidateJob(arg)
try:
interfaces = InterfaceHolder()
jobTracker = interfaces.jobDb
except ResponseException as e:
open("errorLog", "a").write(str(e) + "\n")
return JsonResponse.error(
e, e.status, table="cannot connect to job database")
except Exception as e:
open("errorLog", "a").write(str(e) + "\n")
exc = ResponseException(str(e), StatusCode.INTERNAL_ERROR,
type(e))
return JsonResponse.error(
exc, exc.status, table="cannot connect to job database")
jobId = None
manager = ValidationManager(local, error_report_path)
try:
jobId = manager.getJobID(request)
except ResponseException as e:
manager.markJob(jobId, jobTracker, "invalid",
interfaces.errorDb, manager.filename)
CloudLogger.logError(str(e), e,
traceback.extract_tb(sys.exc_info()[2]))
return JsonResponse.error(e, e.status, table="")
except Exception as e:
exc = ResponseException(str(e), StatusCode.CLIENT_ERROR,
type(e))
manager.markJob(jobId, jobTracker, "invalid",
interfaces.errorDb, manager.filename)
CloudLogger.logError(str(e), exc,
traceback.extract_tb(sys.exc_info()[2]))
return JsonResponse.error(exc, exc.status, table="")
try:
manager.testJobID(jobId, interfaces)
except ResponseException as e:
open("errorLog", "a").write(str(e) + "\n")
# Job is not ready to run according to job tracker, do not change status of job in job tracker
interfaces.errorDb.writeFileError(jobId, manager.filename,
ValidationError.jobError)
return JsonResponse.error(e, e.status, table="")
except Exception as e:
open("errorLog", "a").write(str(e) + "\n")
exc = ResponseException(str(e), StatusCode.CLIENT_ERROR,
type(e))
interfaces.errorDb.writeFileError(jobId, manager.filename,
ValidationError.jobError)
return JsonResponse.error(exc, exc.status, table="")
thread = Thread(target=ThreadedFunction, args=(jobId, ))
try:
jobTracker.markStatus(jobId, "running")
except Exception as e:
open("errorLog", "a").write(str(e) + "\n")
exc = ResponseException(str(e), StatusCode.INTERNAL_ERROR,
type(e))
return JsonResponse.error(exc,
exc.status,
table="could not start job")
interfaces.close()
thread.start()
return JsonResponse.create(StatusCode.OK,
{"table": "job" + str(jobId)})
@app.route("/validate/", methods=["POST"])
def validate():
"""Start the validation process on the same threads."""
interfaces = InterfaceHolder() # Create sessions for this route
try:
return validationManager.validateJob(request, interfaces)
except Exception as e:
# Something went wrong getting the flask request
open("errorLog", "a").write(str(e) + "\n")
exc = ResponseException(str(e), StatusCode.INTERNAL_ERROR,
type(e))
return JsonResponse.error(exc, exc.status, table="")
finally:
interfaces.close()
JsonResponse.debugMode = CONFIG_SERVICES['rest_trace']
return app
except Exception as e:
trace = traceback.extract_tb(sys.exc_info()[2], 10)
CloudLogger.logError('Validator App Level Error: ', e, trace)
raise
def createApp():
"""Set up the application."""
try:
# Create application
app = Flask(__name__, instance_path=CONFIG_PATH)
local = CONFIG_BROKER['local']
app.config.from_object(__name__)
app.config['LOCAL'] = local
app.config['REST_TRACE'] = CONFIG_SERVICES['rest_trace']
app.config['SYSTEM_EMAIL'] = CONFIG_BROKER['reply_to_email']
# Future: Override config w/ environment variable, if set
app.config.from_envvar('BROKER_SETTINGS', silent=True)
# Set parameters
broker_file_path = CONFIG_BROKER['broker_files']
AccountHandler.FRONT_END = CONFIG_BROKER['full_url']
sesEmail.SIGNING_KEY = CONFIG_BROKER['email_token_key']
sesEmail.isLocal = local
if sesEmail.isLocal:
sesEmail.emailLog = os.path.join(broker_file_path, 'email.log')
# If local, make the email directory if needed
if local and not os.path.exists(broker_file_path):
os.makedirs(broker_file_path)
# When runlocal is true, assume Dynamo is on the same server
# (should be false for prod)
JsonResponse.debugMode = app.config['REST_TRACE']
if CONFIG_SERVICES['cross_origin_url'] == "*":
cors = CORS(app, supports_credentials=True)
else:
cors = CORS(app,
supports_credentials=True,
origins=CONFIG_SERVICES['cross_origin_url'])
# Enable AWS Sessions
app.session_interface = DynamoInterface()
# Set up bcrypt
bcrypt = Bcrypt(app)
# Root will point to index.html
@app.route("/", methods=["GET"])
def root():
return "Broker is running"
if local:
localFiles = os.path.join(broker_file_path, "<path:filename>")
# Only define this route when running locally
@app.route(localFiles)
def sendFile(filename):
if (config["local"]):
return send_from_directory(broker_file_path, filename)
else:
# For non-local installs, set Dynamo Region
SessionTable.DYNAMO_REGION = CONFIG_BROKER['aws_region']
# Add routes for modules here
add_login_routes(app, bcrypt)
add_file_routes(app, CONFIG_BROKER['aws_create_temp_credentials'],
local, broker_file_path)
add_user_routes(app, app.config['SYSTEM_EMAIL'], bcrypt)
SessionTable.LOCAL_PORT = CONFIG_DB['dynamo_port']
SessionTable.setup(app, local)
return app
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
trace = traceback.extract_tb(exc_tb, 10)
CloudLogger.logError('Broker App Level Error: ', e, trace)
del exc_tb
raise
def validateFileBySql(cls, submissionId, fileType, interfaces):
""" Check all SQL rules
Args:
submissionId: submission to be checked
fileType: file type being checked
interfaces: database interface objects
Returns:
List of errors found, each element has:
field names
error message
values in fields involved
row number
rule label
source file id
target file id
severity id
"""
CloudLogger.logError(
"VALIDATOR_INFO: ",
"Beginning SQL validation rules on submissionID: " +
str(submissionId) + " fileType: " + fileType, "")
# Pull all SQL rules for this file type
fileId = interfaces.validationDb.getFileTypeIdByName(fileType)
rules = interfaces.validationDb.session.query(RuleSql).filter(
RuleSql.file_id == fileId).filter(
RuleSql.rule_cross_file_flag == False).all()
errors = []
# Get short to long colname dictionary
shortColnames = interfaces.validationDb.getShortToLongColname()
# For each rule, execute sql for rule
for rule in rules:
CloudLogger.logError(
"VALIDATOR_INFO: ", "Running query: " +
str(RuleSql.query_name) + " on submissionID: " +
str(submissionId) + " fileType: " + fileType, "")
failures = interfaces.stagingDb.connection.execute(
rule.rule_sql.format(submissionId))
if failures.rowcount:
# Create column list (exclude row_number)
cols = failures.keys()
cols.remove("row_number")
# Build error list
for failure in failures:
errorMsg = rule.rule_error_message
row = failure["row_number"]
# Create strings for fields and values
valueList = [
"{}: {}".format(shortColnames[field],
str(failure[field]))
if field in shortColnames else "{}: {}".format(
field, str(failure[field])) for field in cols
]
valueString = ", ".join(valueList)
fieldList = [
shortColnames[field]
if field in shortColnames else field for field in cols
]
fieldString = ", ".join(fieldList)
errors.append([
fieldString, errorMsg, valueString, row,
rule.rule_label, fileId, rule.target_file_id,
rule.rule_severity_id
])
CloudLogger.logError(
"VALIDATOR_INFO: ",
"Completed SQL validation rules on submissionID: " +
str(submissionId) + " fileType: " + fileType, "")
return errors
def validateJob(self, request,interfaces):
""" Gets file for job, validates each row, and sends valid rows to staging database
Args:
request -- HTTP request containing the jobId
interfaces -- InterfaceHolder object to the databases
Returns:
Http response object
"""
# Create connection to job tracker database
self.filename = None
tableName = ""
jobId = None
jobTracker = None
try:
jobTracker = interfaces.jobDb
requestDict = RequestDictionary(request)
tableName = ""
if(requestDict.exists("job_id")):
jobId = requestDict.getValue("job_id")
else:
# Request does not have a job ID, can't validate
raise ResponseException("No job ID specified in request",StatusCode.CLIENT_ERROR)
# Check that job exists and is ready
if(not (jobTracker.runChecks(jobId))):
raise ResponseException("Checks failed on Job ID",StatusCode.CLIENT_ERROR)
tableName = interfaces.stagingDb.getTableName(jobId)
jobType = interfaces.jobDb.checkJobType(jobId)
except ResponseException as e:
CloudLogger.logError(str(e),e,traceback.extract_tb(sys.exc_info()[2]))
if(e.errorType == None):
# Error occurred while trying to get and check job ID
e.errorType = ValidationError.jobError
interfaces.errorDb.writeFileError(jobId,self.filename,e.errorType,e.extraInfo)
return JsonResponse.error(e,e.status,table=tableName)
except Exception as e:
exc = ResponseException(str(e),StatusCode.INTERNAL_ERROR,type(e))
CloudLogger.logError(str(e),e,traceback.extract_tb(sys.exc_info()[2]))
self.markJob(jobId,jobTracker,"failed",interfaces.errorDb,self.filename,ValidationError.unknownError)
return JsonResponse.error(exc,exc.status,table=tableName)
try:
jobTracker.markJobStatus(jobId,"running")
if jobType == interfaces.jobDb.getJobTypeId("csv_record_validation"):
self.runValidation(jobId,interfaces)
elif jobType == interfaces.jobDb.getJobTypeId("validation"):
self.runCrossValidation(jobId, interfaces)
else:
raise ResponseException("Bad job type for validator", StatusCode.INTERNAL_ERROR)
interfaces.errorDb.markFileComplete(jobId,self.filename)
return JsonResponse.create(StatusCode.OK,{"table":tableName})
except ResponseException as e:
CloudLogger.logError(str(e),e,traceback.extract_tb(sys.exc_info()[2]))
self.markJob(jobId,jobTracker,"invalid",interfaces.errorDb,self.filename,e.errorType,e.extraInfo)
return JsonResponse.error(e,e.status,table=tableName)
except ValueError as e:
CloudLogger.logError(str(e),e,traceback.extract_tb(sys.exc_info()[2]))
# Problem with CSV headers
exc = ResponseException(str(e),StatusCode.CLIENT_ERROR,type(e),ValidationError.unknownError) #"Internal value error"
self.markJob(jobId,jobTracker,"invalid",interfaces.errorDb,self.filename,ValidationError.unknownError)
return JsonResponse.error(exc,exc.status,table=tableName)
except Error as e:
CloudLogger.logError(str(e),e,traceback.extract_tb(sys.exc_info()[2]))
# CSV file not properly formatted (usually too much in one field)
exc = ResponseException("Internal error",StatusCode.CLIENT_ERROR,type(e),ValidationError.unknownError)
self.markJob(jobId,jobTracker,"invalid",interfaces.errorDb,self.filename,ValidationError.unknownError)
return JsonResponse.error(exc,exc.status,table=tableName)
except Exception as e:
CloudLogger.logError(str(e),e,traceback.extract_tb(sys.exc_info()[2]))
exc = ResponseException(str(e),StatusCode.INTERNAL_ERROR,type(e),ValidationError.unknownError)
self.markJob(jobId,jobTracker,"failed",interfaces.errorDb,self.filename,ValidationError.unknownError)
return JsonResponse.error(exc,exc.status,table=tableName)
def runValidation(self, jobId, interfaces):
""" Run validations for specified job
Args:
jobId: Job to be validated
jobTracker: Interface for job tracker
Returns:
True if successful
"""
sess = GlobalDB.db().session
# get the job object here so we can call the refactored getReportPath
# todo: replace other db access functions with job object attributes
job = sess.query(Job).filter(Job.job_id == jobId).one()
CloudLogger.logError("VALIDATOR_INFO: ",
"Beginning runValidation on jobID: " + str(jobId),
"")
jobTracker = interfaces.jobDb
submissionId = jobTracker.getSubmissionId(jobId)
rowNumber = 1
fileType = jobTracker.getFileType(jobId)
# Clear existing records for this submission
interfaces.stagingDb.clearFileBySubmission(submissionId, fileType)
# Get short to long colname dictionary
shortColnames = interfaces.validationDb.getShortToLongColname()
# If local, make the error report directory
if self.isLocal and not os.path.exists(self.directory):
os.makedirs(self.directory)
# Get bucket name and file name
fileName = jobTracker.getFileName(jobId)
self.filename = fileName
bucketName = CONFIG_BROKER['aws_bucket']
regionName = CONFIG_BROKER['aws_region']
errorFileName = self.getFileName(getReportPath(job, 'error'))
warningFileName = self.getFileName(getReportPath(job, 'warning'))
# Create File Status object
interfaces.errorDb.createFileIfNeeded(jobId, fileName)
validationDB = interfaces.validationDb
fieldList = validationDB.getFieldsByFileList(fileType)
csvSchema = validationDB.getFieldsByFile(fileType, shortCols=True)
reader = self.getReader()
# Get file size and write to jobs table
if CONFIG_BROKER["use_aws"]:
fileSize = s3UrlHandler.getFileSize(errorFileName)
else:
fileSize = os.path.getsize(jobTracker.getFileName(jobId))
jobTracker.setFileSizeById(jobId, fileSize)
fields = interfaces.validationDb.getFileColumnsByFile(fileType)
try:
# Pull file and return info on whether it's using short or long col headers
reader.openFile(regionName, bucketName, fileName, fieldList,
bucketName, errorFileName)
errorInterface = interfaces.errorDb
self.longToShortDict = interfaces.validationDb.getLongToShortColname(
)
# rowErrorPresent becomes true if any row error occurs, used for determining file status
rowErrorPresent = False
# list to keep track of rows that fail validations
errorRows = []
# While not done, pull one row and put it into staging table if it passes
# the Validator
with self.getWriter(regionName, bucketName, errorFileName, self.reportHeaders) as writer, \
self.getWriter(regionName, bucketName, warningFileName, self.reportHeaders) as warningWriter:
while not reader.isFinished:
rowNumber += 1
if (rowNumber % 100) == 0:
CloudLogger.logError(
"VALIDATOR_INFO: ", "JobId: " + str(jobId) +
" loading row " + str(rowNumber), "")
#
# first phase of validations: read record and record a
# formatting error if there's a problem
#
(record, reduceRow, skipRow, doneReading,
rowErrorHere) = self.readRecord(reader, writer, fileType,
interfaces, rowNumber,
jobId, fields)
if reduceRow:
rowNumber -= 1
if rowErrorHere:
rowErrorPresent = True
errorRows.append(rowNumber)
if doneReading:
# Stop reading from input file
break
elif skipRow:
# Do not write this row to staging, but continue processing future rows
continue
#
# second phase of validations: do basic schema checks
# (e.g., require fields, field length, data type)
#
# D files are obtained from upstream systems (ASP and FPDS) that perform their own basic validations,
# so these validations are not repeated here
if fileType in ["award", "award_procurement"]:
# Skip basic validations for D files, set as valid to trigger write to staging
passedValidations = True
valid = True
else:
passedValidations, failures, valid = Validator.validate(
record, csvSchema)
if valid:
skipRow = self.writeToStaging(record, jobId,
submissionId,
passedValidations,
interfaces, writer,
rowNumber, fileType)
if skipRow:
errorRows.append(rowNumber)
continue
if not passedValidations:
if self.writeErrors(failures, interfaces, jobId,
shortColnames, writer,
warningWriter, rowNumber):
errorRows.append(rowNumber)
CloudLogger.logError(
"VALIDATOR_INFO: ",
"Loading complete on jobID: " + str(jobId) +
". Total rows added to staging: " + str(rowNumber), "")
#
# third phase of validations: run validation rules as specified
# in the schema guidance. these validations are sql-based.
#
sqlErrorRows = self.runSqlValidations(interfaces, jobId,
fileType, shortColnames,
writer, warningWriter,
rowNumber)
errorRows.extend(sqlErrorRows)
# Write unfinished batch
writer.finishBatch()
warningWriter.finishBatch()
# Calculate total number of rows in file
# that passed validations
errorRowsUnique = set(errorRows)
totalRowsExcludingHeader = rowNumber - 1
validRows = totalRowsExcludingHeader - len(errorRowsUnique)
# Update job metadata
jobTracker.setJobRowcounts(jobId, rowNumber, validRows)
errorInterface.writeAllRowErrors(jobId)
# Update error info for submission
jobTracker.populateSubmissionErrorInfo(submissionId)
# Mark validation as finished in job tracker
jobTracker.markJobStatus(jobId, "finished")
interfaces.errorDb.markFileComplete(jobId, self.filename)
finally:
# Ensure the file always closes
reader.close()
CloudLogger.logError(
"VALIDATOR_INFO: ",
"Completed L1 and SQL rule validations on jobID: " +
str(jobId), "")
return True
def validateJob(self, request, interfaces):
""" Gets file for job, validates each row, and sends valid rows to staging database
Args:
request -- HTTP request containing the jobId
interfaces -- InterfaceHolder object to the databases
Returns:
Http response object
"""
# Create connection to job tracker database
self.filename = None
tableName = ""
jobId = None
jobTracker = None
try:
jobTracker = interfaces.jobDb
requestDict = RequestDictionary(request)
tableName = ""
if (requestDict.exists("job_id")):
jobId = requestDict.getValue("job_id")
else:
# Request does not have a job ID, can't validate
raise ResponseException("No job ID specified in request",
StatusCode.CLIENT_ERROR)
# Check that job exists and is ready
if (not (jobTracker.runChecks(jobId))):
raise ResponseException("Checks failed on Job ID",
StatusCode.CLIENT_ERROR)
tableName = interfaces.stagingDb.getTableName(jobId)
jobType = interfaces.jobDb.checkJobType(jobId)
except ResponseException as e:
CloudLogger.logError(str(e), e,
traceback.extract_tb(sys.exc_info()[2]))
if (e.errorType == None):
# Error occurred while trying to get and check job ID
e.errorType = ValidationError.jobError
interfaces.errorDb.writeFileError(jobId, self.filename,
e.errorType, e.extraInfo)
return JsonResponse.error(e, e.status, table=tableName)
except Exception as e:
exc = ResponseException(str(e), StatusCode.INTERNAL_ERROR, type(e))
CloudLogger.logError(str(e), e,
traceback.extract_tb(sys.exc_info()[2]))
self.markJob(jobId, jobTracker, "failed", interfaces.errorDb,
self.filename, ValidationError.unknownError)
return JsonResponse.error(exc, exc.status, table=tableName)
try:
jobTracker.markJobStatus(jobId, "running")
if jobType == interfaces.jobDb.getJobTypeId(
"csv_record_validation"):
self.runValidation(jobId, interfaces)
elif jobType == interfaces.jobDb.getJobTypeId("validation"):
self.runCrossValidation(jobId, interfaces)
else:
raise ResponseException("Bad job type for validator",
StatusCode.INTERNAL_ERROR)
interfaces.errorDb.markFileComplete(jobId, self.filename)
return JsonResponse.create(StatusCode.OK, {"table": tableName})
except ResponseException as e:
CloudLogger.logError(str(e), e,
traceback.extract_tb(sys.exc_info()[2]))
self.markJob(jobId, jobTracker, "invalid", interfaces.errorDb,
self.filename, e.errorType, e.extraInfo)
return JsonResponse.error(e, e.status, table=tableName)
except ValueError as e:
CloudLogger.logError(str(e), e,
traceback.extract_tb(sys.exc_info()[2]))
# Problem with CSV headers
exc = ResponseException(
str(e), StatusCode.CLIENT_ERROR, type(e),
ValidationError.unknownError) #"Internal value error"
self.markJob(jobId, jobTracker, "invalid", interfaces.errorDb,
self.filename, ValidationError.unknownError)
return JsonResponse.error(exc, exc.status, table=tableName)
except Error as e:
CloudLogger.logError(str(e), e,
traceback.extract_tb(sys.exc_info()[2]))
# CSV file not properly formatted (usually too much in one field)
exc = ResponseException("Internal error", StatusCode.CLIENT_ERROR,
type(e), ValidationError.unknownError)
self.markJob(jobId, jobTracker, "invalid", interfaces.errorDb,
self.filename, ValidationError.unknownError)
return JsonResponse.error(exc, exc.status, table=tableName)
except Exception as e:
CloudLogger.logError(str(e), e,
traceback.extract_tb(sys.exc_info()[2]))
exc = ResponseException(str(e), StatusCode.INTERNAL_ERROR, type(e),
ValidationError.unknownError)
self.markJob(jobId, jobTracker, "failed", interfaces.errorDb,
self.filename, ValidationError.unknownError)
return JsonResponse.error(exc, exc.status, table=tableName)
def runCrossValidation(self, jobId, interfaces):
""" Cross file validation job, test all rules with matching rule_timing """
# Create File Status object
interfaces.errorDb.createFileIfNeeded(jobId)
validationDb = interfaces.validationDb
errorDb = interfaces.errorDb
submissionId = interfaces.jobDb.getSubmissionId(jobId)
bucketName = CONFIG_BROKER['aws_bucket']
regionName = CONFIG_BROKER['aws_region']
CloudLogger.logError(
"VALIDATOR_INFO: ",
"Beginning runCrossValidation on submissionID: " +
str(submissionId), "")
# Delete existing cross file errors for this submission
errorDb.resetErrorsByJobId(jobId)
# use db to get a list of the cross-file combinations
targetFiles = validationDb.session.query(FileTypeValidation).subquery()
crossFileCombos = validationDb.session.query(
FileTypeValidation.name.label('first_file_name'),
FileTypeValidation.file_id.label('first_file_id'),
targetFiles.c.name.label('second_file_name'),
targetFiles.c.file_id.label('second_file_id')).filter(
FileTypeValidation.file_order < targetFiles.c.file_order)
# get all cross file rules from db
crossFileRules = validationDb.session.query(RuleSql).filter(
RuleSql.rule_cross_file_flag == True)
# for each cross-file combo, run associated rules and create error report
for row in crossFileCombos:
comboRules = crossFileRules.filter(
or_(
and_(RuleSql.file_id == row.first_file_id,
RuleSql.target_file_id == row.second_file_id),
and_(RuleSql.file_id == row.second_file_id,
RuleSql.target_file_id == row.first_file_id)))
# send comboRules to validator.crossValidate sql
failures = Validator.crossValidateSql(comboRules.all(),
submissionId)
# get error file name
reportFilename = self.getFileName(
getCrossReportName(submissionId, row.first_file_name,
row.second_file_name))
warningReportFilename = self.getFileName(
getCrossWarningReportName(submissionId, row.first_file_name,
row.second_file_name))
# loop through failures to create the error report
with self.getWriter(regionName, bucketName, reportFilename, self.crossFileReportHeaders) as writer, \
self.getWriter(regionName, bucketName, warningReportFilename, self.crossFileReportHeaders) as warningWriter:
for failure in failures:
if failure[9] == interfaces.validationDb.getRuleSeverityId(
"fatal"):
writer.write(failure[0:7])
if failure[9] == interfaces.validationDb.getRuleSeverityId(
"warning"):
warningWriter.write(failure[0:7])
errorDb.recordRowError(jobId,
"cross_file",
failure[0],
failure[3],
failure[5],
failure[6],
failure[7],
failure[8],
severity_id=failure[9])
writer.finishBatch()
warningWriter.finishBatch()
errorDb.writeAllRowErrors(jobId)
interfaces.jobDb.markJobStatus(jobId, "finished")
CloudLogger.logError(
"VALIDATOR_INFO: ",
"Completed runCrossValidation on submissionID: " +
str(submissionId), "")
# Update error info for submission
interfaces.jobDb.populateSubmissionErrorInfo(submissionId)
# TODO: Remove temporary step below
# Temporarily set publishable flag at end of cross file, remove this once users are able to mark their submissions
# as publishable
# Publish only if no errors are present
if interfaces.jobDb.getSubmissionById(
submissionId).number_of_errors == 0:
interfaces.jobDb.setPublishableFlag(submissionId, True)
# Mark validation complete
interfaces.errorDb.markFileComplete(jobId)