def runCrossValidation(self, job):
""" Cross file validation job, test all rules with matching rule_timing """
sess = GlobalDB.db().session
job_id = job.job_id
# Create File Status object
createFileIfNeeded(job_id)
error_list = ErrorInterface()
submission_id = job.submission_id
bucketName = CONFIG_BROKER['aws_bucket']
regionName = CONFIG_BROKER['aws_region']
_exception_logger.info(
'VALIDATOR_INFO: Beginning runCrossValidation on submission_id: '
'%s', submission_id)
# Delete existing cross file errors for this submission
sess.query(ErrorMetadata).filter(
ErrorMetadata.job_id == job_id).delete()
sess.commit()
# get all cross file rules from db
crossFileRules = sess.query(RuleSql).filter(
RuleSql.rule_cross_file_flag == True)
# for each cross-file combo, run associated rules and create error report
for c in get_cross_file_pairs():
first_file = c[0]
second_file = c[1]
comboRules = crossFileRules.filter(
or_(
and_(RuleSql.file_id == first_file.id,
RuleSql.target_file_id == second_file.id),
and_(RuleSql.file_id == second_file.id,
RuleSql.target_file_id == first_file.id)))
# send comboRules to validator.crossValidate sql
failures = Validator.crossValidateSql(comboRules.all(),
submission_id,
self.short_to_long_dict)
# get error file name
reportFilename = self.getFileName(
get_cross_report_name(submission_id, first_file.name,
second_file.name))
warningReportFilename = self.getFileName(
get_cross_warning_report_name(submission_id, first_file.name,
second_file.name))
# loop through failures to create the error report
with self.getWriter(regionName, bucketName, reportFilename, self.crossFileReportHeaders) as writer, \
self.getWriter(regionName, bucketName, warningReportFilename, self.crossFileReportHeaders) as warningWriter:
for failure in failures:
if failure[9] == RULE_SEVERITY_DICT['fatal']:
writer.write(failure[0:7])
if failure[9] == RULE_SEVERITY_DICT['warning']:
warningWriter.write(failure[0:7])
error_list.recordRowError(job_id,
"cross_file",
failure[0],
failure[3],
failure[5],
failure[6],
failure[7],
failure[8],
severity_id=failure[9])
writer.finishBatch()
warningWriter.finishBatch()
error_list.writeAllRowErrors(job_id)
mark_job_status(job_id, "finished")
_exception_logger.info(
'VALIDATOR_INFO: Completed runCrossValidation on submission_id: '
'%s', submission_id)
submission = sess.query(Submission).filter_by(
submission_id=submission_id).one()
# Update error info for submission
submission.number_of_errors = sumNumberOfErrorsForJobList(
submission_id)
submission.number_of_warnings = sumNumberOfErrorsForJobList(
submission_id, errorType="warning")
# TODO: Remove temporary step below
# Temporarily set publishable flag at end of cross file, remove this once users are able to mark their submissions
# as publishable
# Publish only if no errors are present
if submission.number_of_errors == 0:
submission.publishable = True
sess.commit()
# Mark validation complete
markFileComplete(job_id)
def runValidation(self, job):
""" Run validations for specified job
Args:
job: Job to be validated
Returns:
True if successful
"""
sess = GlobalDB.db().session
job_id = job.job_id
error_list = ErrorInterface()
_exception_logger.info(
'VALIDATOR_INFO: Beginning runValidation on job_id: %s', job_id)
submission_id = job.submission_id
rowNumber = 1
fileType = job.file_type.name
# Get orm model for this file
model = [ft.model for ft in FILE_TYPE if ft.name == fileType][0]
# Clear existing records for this submission
sess.query(model).filter_by(submission_id=submission_id).delete()
sess.commit()
# If local, make the error report directory
if self.isLocal and not os.path.exists(self.directory):
os.makedirs(self.directory)
# Get bucket name and file name
fileName = job.filename
bucketName = CONFIG_BROKER['aws_bucket']
regionName = CONFIG_BROKER['aws_region']
errorFileName = self.getFileName(get_report_path(job, 'error'))
warningFileName = self.getFileName(get_report_path(job, 'warning'))
# Create File Status object
createFileIfNeeded(job_id, fileName)
reader = self.getReader()
# Get file size and write to jobs table
if CONFIG_BROKER["use_aws"]:
fileSize = s3UrlHandler.getFileSize(errorFileName)
else:
fileSize = os.path.getsize(fileName)
job.file_size = fileSize
sess.commit()
# Get fields for this file
fields = sess.query(FileColumn). \
filter(FileColumn.file_id == FILE_TYPE_DICT[fileType]). \
all()
for field in fields:
sess.expunge(field)
csvSchema = {row.name_short: row for row in fields}
try:
# Pull file and return info on whether it's using short or long col headers
reader.open_file(regionName, bucketName, fileName, fields,
bucketName, errorFileName,
self.long_to_short_dict)
# list to keep track of rows that fail validations
errorRows = []
# While not done, pull one row and put it into staging table if it passes
# the Validator
with self.getWriter(regionName, bucketName, errorFileName, self.reportHeaders) as writer, \
self.getWriter(regionName, bucketName, warningFileName, self.reportHeaders) as warningWriter:
while not reader.is_finished:
rowNumber += 1
if rowNumber % 10 == 0:
logger.info('loading row %s', rowNumber)
#
# first phase of validations: read record and record a
# formatting error if there's a problem
#
(record, reduceRow, skipRow, doneReading, rowErrorHere,
flex_cols) = self.readRecord(reader, writer, fileType,
rowNumber, job, fields,
error_list)
if reduceRow:
rowNumber -= 1
if rowErrorHere:
errorRows.append(rowNumber)
if doneReading:
# Stop reading from input file
break
elif skipRow:
# Do not write this row to staging, but continue processing future rows
continue
#
# second phase of validations: do basic schema checks
# (e.g., require fields, field length, data type)
#
# D files are obtained from upstream systems (ASP and FPDS) that perform their own basic validations,
# so these validations are not repeated here
if fileType in ["award", "award_procurement"]:
# Skip basic validations for D files, set as valid to trigger write to staging
passedValidations = True
valid = True
else:
passedValidations, failures, valid = Validator.validate(
record, csvSchema)
if valid:
skipRow = self.writeToStaging(record, job,
submission_id,
passedValidations,
writer, rowNumber, model,
error_list)
if flex_cols:
self.write_to_flex(flex_cols, job_id,
submission_id, fileType)
if skipRow:
errorRows.append(rowNumber)
continue
if not passedValidations:
if self.writeErrors(failures, job,
self.short_to_long_dict, writer,
warningWriter, rowNumber,
error_list):
errorRows.append(rowNumber)
_exception_logger.info(
'VALIDATOR_INFO: Loading complete on job_id: %s. '
'Total rows added to staging: %s', job_id, rowNumber)
if fileType in ('appropriations', 'program_activity',
'award_financial'):
update_tas_ids(model, submission_id)
#
# third phase of validations: run validation rules as specified
# in the schema guidance. these validations are sql-based.
#
sqlErrorRows = self.runSqlValidations(job, fileType,
self.short_to_long_dict,
writer, warningWriter,
rowNumber, error_list)
errorRows.extend(sqlErrorRows)
# Write unfinished batch
writer.finishBatch()
warningWriter.finishBatch()
# Calculate total number of rows in file
# that passed validations
errorRowsUnique = set(errorRows)
totalRowsExcludingHeader = rowNumber - 1
validRows = totalRowsExcludingHeader - len(errorRowsUnique)
# Update job metadata
job.number_of_rows = rowNumber
job.number_of_rows_valid = validRows
sess.commit()
error_list.writeAllRowErrors(job_id)
# Update error info for submission
populateSubmissionErrorInfo(submission_id)
# Mark validation as finished in job tracker
mark_job_status(job_id, "finished")
markFileComplete(job_id, fileName)
finally:
# Ensure the file always closes
reader.close()
_exception_logger.info(
'VALIDATOR_INFO: Completed L1 and SQL rule validations on '
'job_id: %s', job_id)
return True