def writeFileError(job_id, filename, error_type, extra_info=None):
""" Write a file-level error to the file table
Args:
job_id: ID of job in job tracker
filename: name of error report in S3
error_type: type of error, value will be mapped to ValidationError class
extra_info: list of extra information to be included in file
"""
sess = GlobalDB.db().session
try:
int(job_id)
except:
raise ValueError("".join(["Bad jobId: ", str(job_id)]))
# Get File object for this job ID or create it if it doesn't exist
fileRec = createFileIfNeeded(job_id, filename)
# Mark error type and add header info if present
fileRec.file_status_id = FILE_STATUS_DICT[ValidationError.getErrorTypeString(error_type)]
if extra_info is not None:
if "missing_headers" in extra_info:
fileRec.headers_missing = extra_info["missing_headers"]
if "duplicated_headers" in extra_info:
fileRec.headers_duplicated = extra_info["duplicated_headers"]
sess.add(fileRec)
sess.commit()
def writeFileError(self, jobId, filename, errorType, extraInfo = None):
""" Write a file-level error to the file table
Args:
jobId: ID of job in job tracker
filename: name of error report in S3
errorType: type of error, value will be mapped to ValidationError class
Returns:
True if successful
"""
try:
int(jobId)
except:
raise ValueError("".join(["Bad jobId: ", str(jobId)]))
# Get File object for this job ID or create it if it doesn't exist
fileRec = self.createFileIfNeeded(jobId, filename)
# Mark error type and add header info if present
fileRec.file_status_id = self.getFileStatusId(
ValidationError.getErrorTypeString(errorType))
if extraInfo is not None:
if "missing_headers" in extraInfo:
fileRec.headers_missing = extraInfo["missing_headers"]
if "duplicated_headers" in extraInfo:
fileRec.headers_duplicated = extraInfo["duplicated_headers"]
self.session.add(fileRec)
self.session.commit()
return True
def writeFileError(self, jobId, filename, errorType, extraInfo=None):
""" Write a file-level error to the file status table
Args:
jobId: ID of job in job tracker
filename: name of error report in S3
errorType: type of error, value will be mapped to ValidationError class
Returns:
True if successful
"""
try:
int(jobId)
except:
raise ValueError("".join(["Bad jobId: ", str(jobId)]))
# Get File Status for this job ID or create it if it doesn't exist
fileStatus = self.createFileStatusIfNeeded(jobId, filename)
# Mark error type and add header info if present
fileStatus.status_id = self.getStatusId(
ValidationError.getErrorTypeString(errorType))
if extraInfo is not None:
if "missing_headers" in extraInfo:
fileStatus.headers_missing = extraInfo["missing_headers"]
if "duplicated_headers" in extraInfo:
fileStatus.headers_duplicated = extraInfo["duplicated_headers"]
self.session.add(fileStatus)
self.session.commit()
return True
def writeAllRowErrors(self, job_id):
""" Writes all recorded errors to database
Args:
job_id: ID to write errors for
"""
sess = GlobalDB.db().session
for key in self.rowErrors.keys():
errorDict = self.rowErrors[key]
# Set info for this error
thisJob = errorDict["jobId"]
if int(job_id) != int(thisJob):
# This row is for a different job, skip it
continue
field_name = errorDict["fieldName"]
try:
# If last part of key is an int, it's one of our prestored messages
error_type = int(errorDict["errorType"])
except ValueError:
# For rule failures, it will hold the error message
errorMsg = errorDict["errorType"]
if "Field must be no longer than specified limit" in errorMsg:
ruleFailedId = ERROR_TYPE_DICT['length_error']
else:
ruleFailedId = ERROR_TYPE_DICT['rule_failed']
errorRow = ErrorMetadata(job_id=thisJob, filename=errorDict["filename"], field_name=field_name,
error_type_id=ruleFailedId, rule_failed=errorMsg,
occurrences=errorDict["numErrors"], first_row=errorDict["firstRow"],
original_rule_label=errorDict["originalRuleLabel"],
file_type_id=errorDict["fileTypeId"],
target_file_type_id=errorDict["targetFileId"],
severity_id=errorDict["severity"])
else:
# This happens if cast to int was successful
errorString = ValidationError.getErrorTypeString(error_type)
errorId = ERROR_TYPE_DICT[errorString]
# Create error metadata
errorRow = ErrorMetadata(job_id=thisJob, filename=errorDict["filename"], field_name=field_name,
error_type_id=errorId, occurrences=errorDict["numErrors"],
first_row=errorDict["firstRow"],
rule_failed=ValidationError.getErrorMessage(error_type),
original_rule_label=errorDict["originalRuleLabel"],
file_type_id=errorDict["fileTypeId"],
target_file_type_id=errorDict["targetFileId"],
severity_id=errorDict["severity"])
sess.add(errorRow)
# Commit the session to write all rows
sess.commit()
# Clear the dictionary
self.rowErrors = {}
def writeAllRowErrors(self, jobId):
""" Writes all recorded errors to database
Args:
jobId: ID to write errors for
Returns:
True if successful
"""
for key in self.rowErrors.keys():
errorDict = self.rowErrors[key]
# Set info for this error
thisJob = errorDict["jobId"]
if (int(jobId) != int(thisJob)):
# This row is for a different job, skip it
continue
fieldName = errorDict["fieldName"]
try:
# If last part of key is an int, it's one of our prestored messages
errorType = int(errorDict["errorType"])
except ValueError:
# For rule failures, it will hold the error message
errorMsg = errorDict["errorType"]
ruleFailedId = self.getTypeId("rule_failed")
errorRow = ErrorData(job_id=thisJob,
filename=errorDict["filename"],
field_name=fieldName,
error_type_id=ruleFailedId,
rule_failed=errorMsg,
occurrences=errorDict["numErrors"],
first_row=errorDict["firstRow"])
else:
# This happens if cast to int was successful
errorString = ValidationError.getErrorTypeString(errorType)
errorId = self.getTypeId(errorString)
# Create error data
errorRow = ErrorData(
job_id=thisJob,
filename=errorDict["filename"],
field_name=fieldName,
error_type_id=errorId,
occurrences=errorDict["numErrors"],
first_row=errorDict["firstRow"],
rule_failed=ValidationError.getErrorMessage(errorType))
self.session.add(errorRow)
# Commit the session to write all rows
self.session.commit()
# Clear the dictionary
self.rowErrors = {}
def writeAllRowErrors(self, jobId):
""" Writes all recorded errors to database
Args:
jobId: ID to write errors for
Returns:
True if successful
"""
for key in self.rowErrors.keys():
errorDict = self.rowErrors[key]
# Set info for this error
thisJob = errorDict["jobId"]
if(int(jobId) != int(thisJob)):
# This row is for a different job, skip it
continue
fieldName = errorDict["fieldName"]
try:
# If last part of key is an int, it's one of our prestored messages
errorType = int(errorDict["errorType"])
except ValueError:
# For rule failures, it will hold the error message
errorMsg = errorDict["errorType"]
ruleFailedId = self.getTypeId("rule_failed")
errorRow = ErrorMetadata(job_id=thisJob, filename=errorDict["filename"], field_name=fieldName, error_type_id=ruleFailedId, rule_failed=errorMsg, occurrences=errorDict["numErrors"], first_row=errorDict["firstRow"])
else:
# This happens if cast to int was successful
errorString = ValidationError.getErrorTypeString(errorType)
errorId = self.getTypeId(errorString)
# Create error metadata
errorRow = ErrorMetadata(job_id=thisJob, filename=errorDict["filename"], field_name=fieldName, error_type_id=errorId, occurrences=errorDict["numErrors"], first_row=errorDict["firstRow"], rule_failed=ValidationError.getErrorMessage(errorType))
self.session.add(errorRow)
# Commit the session to write all rows
self.session.commit()
# Clear the dictionary
self.rowErrors = {}
