def write_file_error(job_id, filename, error_type, extra_info=None): """ Write a file-level error to the file table Args: job_id: ID of job in job tracker filename: name of error report in S3 error_type: type of error, value will be mapped to ValidationError class extra_info: list of extra information to be included in file """ sess = GlobalDB.db().session try: int(job_id) except Exception: logger.error({ 'message': 'Bad job_id: {}'.format(job_id), 'message_type': 'CoreError', 'job_id': job_id, 'function': 'write_file_error' }) raise ValueError('Bad job_id: {}'.format(job_id)) # Get File object for this job ID or create it if it doesn't exist file_rec = create_file_if_needed(job_id, filename) # Mark error type and add header info if present file_rec.file_status_id = FILE_STATUS_DICT[ ValidationError.get_error_type_string(error_type)] if extra_info is not None: if 'missing_headers' in extra_info: file_rec.headers_missing = extra_info['missing_headers'] if 'duplicated_headers' in extra_info: file_rec.headers_duplicated = extra_info['duplicated_headers'] sess.add(file_rec) sess.commit()
def write_file_error(job_id, filename, error_type, extra_info=None): """ Write a file-level error to the file table Args: job_id: ID of job in job tracker filename: name of error report in S3 error_type: type of error, value will be mapped to ValidationError class extra_info: list of extra information to be included in file """ sess = GlobalDB.db().session try: int(job_id) except: raise ValueError("".join(["Bad jobId: ", str(job_id)])) # Get File object for this job ID or create it if it doesn't exist file_rec = create_file_if_needed(job_id, filename) # Mark error type and add header info if present file_rec.file_status_id = FILE_STATUS_DICT[ValidationError.get_error_type_string(error_type)] if extra_info is not None: if "missing_headers" in extra_info: file_rec.headers_missing = extra_info["missing_headers"] if "duplicated_headers" in extra_info: file_rec.headers_duplicated = extra_info["duplicated_headers"] sess.add(file_rec) sess.commit()
def write_all_row_errors(error_list, job_id): """ Writes all recorded errors to database Args: error_list: dict keeping track of error metadata to be updated job_id: ID to write errors for """ sess = GlobalDB.db().session for key in error_list.keys(): error_dict = error_list[key] # Set info for this error this_job = error_dict["jobId"] if int(job_id) != int(this_job): # This row is for a different job, skip it continue field_name = error_dict["fieldName"] try: # If last part of key is an int, it's one of our prestored messages error_type = int(error_dict["errorType"]) except ValueError: # For rule failures, it will hold the error message error_msg = error_dict["errorType"] if "Field must be no longer than specified limit" in error_msg: rule_failed_id = ERROR_TYPE_DICT['length_error'] else: rule_failed_id = ERROR_TYPE_DICT['rule_failed'] error_row = ErrorMetadata( job_id=this_job, filename=error_dict["filename"], field_name=field_name, error_type_id=rule_failed_id, rule_failed=error_msg, occurrences=error_dict["numErrors"], first_row=error_dict["firstRow"], original_rule_label=error_dict["originalRuleLabel"], file_type_id=error_dict["fileTypeId"], target_file_type_id=error_dict["targetFileId"], severity_id=error_dict["severity"]) else: # This happens if cast to int was successful error_string = ValidationError.get_error_type_string(error_type) error_id = ERROR_TYPE_DICT[error_string] # Create error metadata error_row = ErrorMetadata( job_id=this_job, filename=error_dict["filename"], field_name=field_name, error_type_id=error_id, occurrences=error_dict["numErrors"], first_row=error_dict["firstRow"], rule_failed=ValidationError.get_error_message(error_type), original_rule_label=error_dict["originalRuleLabel"], file_type_id=error_dict["fileTypeId"], target_file_type_id=error_dict["targetFileId"], severity_id=error_dict["severity"]) sess.add(error_row) # Commit the session to write all rows sess.commit()
def write_all_row_errors(self, job_id): """ Writes all recorded errors to database Args: job_id: ID to write errors for """ sess = GlobalDB.db().session for key in self.rowErrors.keys(): error_dict = self.rowErrors[key] # Set info for this error this_job = error_dict["jobId"] if int(job_id) != int(this_job): # This row is for a different job, skip it continue field_name = error_dict["fieldName"] try: # If last part of key is an int, it's one of our prestored messages error_type = int(error_dict["errorType"]) except ValueError: # For rule failures, it will hold the error message error_msg = error_dict["errorType"] if "Field must be no longer than specified limit" in error_msg: rule_failed_id = ERROR_TYPE_DICT['length_error'] else: rule_failed_id = ERROR_TYPE_DICT['rule_failed'] error_row = ErrorMetadata(job_id=this_job, filename=error_dict["filename"], field_name=field_name, error_type_id=rule_failed_id, rule_failed=error_msg, occurrences=error_dict["numErrors"], first_row=error_dict["firstRow"], original_rule_label=error_dict["originalRuleLabel"], file_type_id=error_dict["fileTypeId"], target_file_type_id=error_dict["targetFileId"], severity_id=error_dict["severity"]) else: # This happens if cast to int was successful error_string = ValidationError.get_error_type_string(error_type) error_id = ERROR_TYPE_DICT[error_string] # Create error metadata error_row = ErrorMetadata(job_id=this_job, filename=error_dict["filename"], field_name=field_name, error_type_id=error_id, occurrences=error_dict["numErrors"], first_row=error_dict["firstRow"], rule_failed=ValidationError.get_error_message(error_type), original_rule_label=error_dict["originalRuleLabel"], file_type_id=error_dict["fileTypeId"], target_file_type_id=error_dict["targetFileId"], severity_id=error_dict["severity"]) sess.add(error_row) # Commit the session to write all rows sess.commit() # Clear the dictionary self.rowErrors = {}