def read_record(self, reader, writer, row_number, job, fields, error_list):
""" Read and process the next record
Args:
reader: CsvReader object
writer: CsvWriter object
row_number: Next row number to be read
job: current job
fields: List of FileColumn objects for this file type
error_list: instance of ErrorInterface to keep track of errors
Returns:
Tuple with six elements:
1. Dict of record after preprocessing
2. Boolean indicating whether to reduce row count
3. Boolean indicating whether to skip row
4. Boolean indicating whether to stop reading
5. Row error has been found
6. Dict of flex columns
"""
reduce_row = False
row_error_found = False
job_id = job.job_id
try:
(next_record, flex_fields) = reader.get_next_record()
record = FieldCleaner.clean_row(
next_record, self.long_to_short_dict[job.file_type_id], fields)
record["row_number"] = row_number
for flex_field in flex_fields:
flex_field.submission_id = job.submission_id
flex_field.job_id = job.job_id
flex_field.row_number = row_number
flex_field.file_type_id = job.file_type_id
if reader.is_finished and len(record) < 2:
# This is the last line and is empty, don't record an error
return {}, True, True, True, False, [] # Don't count this row
except ResponseException:
if reader.is_finished and reader.extra_line:
# Last line may be blank don't record an error,
# reader.extra_line indicates a case where the last valid line has extra line breaks
# Don't count last row if empty
reduce_row = True
else:
writer.writerow([
"Formatting Error", ValidationError.readErrorMsg,
str(row_number), ""
])
error_list.record_row_error(
job_id,
job.filename,
"Formatting Error",
ValidationError.readError,
row_number,
severity_id=RULE_SEVERITY_DICT['fatal'])
row_error_found = True
return {}, reduce_row, True, False, row_error_found, []
return record, reduce_row, False, False, row_error_found, flex_fields
def read_record(self, reader, writer, row_number, job, fields, error_list):
""" Read and process the next record
Args:
reader: CsvReader object
writer: CsvWriter object
row_number: Next row number to be read
job: current job
fields: List of FileColumn objects for this file type
error_list: instance of ErrorInterface to keep track of errors
Returns:
Tuple with six elements:
1. Dict of record after preprocessing
2. Boolean indicating whether to reduce row count
3. Boolean indicating whether to skip row
4. Boolean indicating whether to stop reading
5. Row error has been found
6. Dict of flex columns
"""
reduce_row = False
row_error_found = False
job_id = job.job_id
try:
(next_record, flex_fields) = reader.get_next_record()
record = FieldCleaner.clean_row(next_record, self.long_to_short_dict, fields)
record["row_number"] = row_number
for flex_field in flex_fields:
flex_field.submission_id = job.submission_id
flex_field.job_id = job.job_id
flex_field.row_number = row_number
flex_field.file_type_id = job.file_type_id
if reader.is_finished and len(record) < 2:
# This is the last line and is empty, don't record an error
return {}, True, True, True, False, [] # Don't count this row
except ResponseException:
if reader.is_finished and reader.extra_line:
# Last line may be blank don't record an error,
# reader.extra_line indicates a case where the last valid line has extra line breaks
# Don't count last row if empty
reduce_row = True
else:
writer.write(["Formatting Error", ValidationError.readErrorMsg, str(row_number), ""])
error_list.record_row_error(job_id, job.filename, "Formatting Error", ValidationError.readError,
row_number, severity_id=RULE_SEVERITY_DICT['fatal'])
row_error_found = True
return {}, reduce_row, True, False, row_error_found, []
return record, reduce_row, False, False, row_error_found, flex_fields
