def write_file_error(job_id, filename, error_type, extra_info=None):
""" Write a file-level error to the file table
Args:
job_id: ID of job in job tracker
filename: name of error report in S3
error_type: type of error, value will be mapped to ValidationError class
extra_info: list of extra information to be included in file
"""
sess = GlobalDB.db().session
try:
int(job_id)
except Exception:
logger.error({
'message': 'Bad job_id: {}'.format(job_id),
'message_type': 'CoreError',
'job_id': job_id,
'function': 'write_file_error'
})
raise ValueError('Bad job_id: {}'.format(job_id))
# Get File object for this job ID or create it if it doesn't exist
file_rec = create_file_if_needed(job_id, filename)
# Mark error type and add header info if present
file_rec.file_status_id = FILE_STATUS_DICT[
ValidationError.get_error_type_string(error_type)]
if extra_info is not None:
if 'missing_headers' in extra_info:
file_rec.headers_missing = extra_info['missing_headers']
if 'duplicated_headers' in extra_info:
file_rec.headers_duplicated = extra_info['duplicated_headers']
sess.add(file_rec)
sess.commit()
def write_file_error(job_id, filename, error_type, extra_info=None):
""" Write a file-level error to the file table
Args:
job_id: ID of job in job tracker
filename: name of error report in S3
error_type: type of error, value will be mapped to ValidationError class
extra_info: list of extra information to be included in file
"""
sess = GlobalDB.db().session
try:
int(job_id)
except:
raise ValueError("".join(["Bad jobId: ", str(job_id)]))
# Get File object for this job ID or create it if it doesn't exist
file_rec = create_file_if_needed(job_id, filename)
# Mark error type and add header info if present
file_rec.file_status_id = FILE_STATUS_DICT[ValidationError.get_error_type_string(error_type)]
if extra_info is not None:
if "missing_headers" in extra_info:
file_rec.headers_missing = extra_info["missing_headers"]
if "duplicated_headers" in extra_info:
file_rec.headers_duplicated = extra_info["duplicated_headers"]
sess.add(file_rec)
sess.commit()
def write_all_row_errors(error_list, job_id):
""" Writes all recorded errors to database
Args:
error_list: dict keeping track of error metadata to be updated
job_id: ID to write errors for
"""
sess = GlobalDB.db().session
for key in error_list.keys():
error_dict = error_list[key]
# Set info for this error
this_job = error_dict["jobId"]
if int(job_id) != int(this_job):
# This row is for a different job, skip it
continue
field_name = error_dict["fieldName"]
try:
# If last part of key is an int, it's one of our prestored messages
error_type = int(error_dict["errorType"])
except ValueError:
# For rule failures, it will hold the error message
error_msg = error_dict["errorType"]
if "Field must be no longer than specified limit" in error_msg:
rule_failed_id = ERROR_TYPE_DICT['length_error']
else:
rule_failed_id = ERROR_TYPE_DICT['rule_failed']
error_row = ErrorMetadata(
job_id=this_job,
filename=error_dict["filename"],
field_name=field_name,
error_type_id=rule_failed_id,
rule_failed=error_msg,
occurrences=error_dict["numErrors"],
first_row=error_dict["firstRow"],
original_rule_label=error_dict["originalRuleLabel"],
file_type_id=error_dict["fileTypeId"],
target_file_type_id=error_dict["targetFileId"],
severity_id=error_dict["severity"])
else:
# This happens if cast to int was successful
error_string = ValidationError.get_error_type_string(error_type)
error_id = ERROR_TYPE_DICT[error_string]
# Create error metadata
error_row = ErrorMetadata(
job_id=this_job,
filename=error_dict["filename"],
field_name=field_name,
error_type_id=error_id,
occurrences=error_dict["numErrors"],
first_row=error_dict["firstRow"],
rule_failed=ValidationError.get_error_message(error_type),
original_rule_label=error_dict["originalRuleLabel"],
file_type_id=error_dict["fileTypeId"],
target_file_type_id=error_dict["targetFileId"],
severity_id=error_dict["severity"])
sess.add(error_row)
# Commit the session to write all rows
sess.commit()
def write_all_row_errors(self, job_id):
""" Writes all recorded errors to database
Args:
job_id: ID to write errors for
"""
sess = GlobalDB.db().session
for key in self.rowErrors.keys():
error_dict = self.rowErrors[key]
# Set info for this error
this_job = error_dict["jobId"]
if int(job_id) != int(this_job):
# This row is for a different job, skip it
continue
field_name = error_dict["fieldName"]
try:
# If last part of key is an int, it's one of our prestored messages
error_type = int(error_dict["errorType"])
except ValueError:
# For rule failures, it will hold the error message
error_msg = error_dict["errorType"]
if "Field must be no longer than specified limit" in error_msg:
rule_failed_id = ERROR_TYPE_DICT['length_error']
else:
rule_failed_id = ERROR_TYPE_DICT['rule_failed']
error_row = ErrorMetadata(job_id=this_job, filename=error_dict["filename"], field_name=field_name,
error_type_id=rule_failed_id, rule_failed=error_msg,
occurrences=error_dict["numErrors"], first_row=error_dict["firstRow"],
original_rule_label=error_dict["originalRuleLabel"],
file_type_id=error_dict["fileTypeId"],
target_file_type_id=error_dict["targetFileId"],
severity_id=error_dict["severity"])
else:
# This happens if cast to int was successful
error_string = ValidationError.get_error_type_string(error_type)
error_id = ERROR_TYPE_DICT[error_string]
# Create error metadata
error_row = ErrorMetadata(job_id=this_job, filename=error_dict["filename"], field_name=field_name,
error_type_id=error_id, occurrences=error_dict["numErrors"],
first_row=error_dict["firstRow"],
rule_failed=ValidationError.get_error_message(error_type),
original_rule_label=error_dict["originalRuleLabel"],
file_type_id=error_dict["fileTypeId"],
target_file_type_id=error_dict["targetFileId"],
severity_id=error_dict["severity"])
sess.add(error_row)
# Commit the session to write all rows
sess.commit()
# Clear the dictionary
self.rowErrors = {}
