def test_tas_concats():
# everything
tas_dict = {
'allocation_transfer_agency': '097',
'agency_identifier': '017',
'beginning_period_of_availa': '2017',
'ending_period_of_availabil': '2017',
'availability_type_code': 'D',
'main_account_code': '0001',
'sub_account_code': '001'
}
tas2_str = concat_tas_dict(tas_dict)
tas2_dstr = concat_display_tas_dict(tas_dict)
assert tas2_str == '09701720172017D0001001'
assert tas2_dstr == '097-017-D-0001-001'
# everything sans type code
tas_dict = {
'allocation_transfer_agency': '097',
'agency_identifier': '017',
'beginning_period_of_availa': '2017',
'ending_period_of_availabil': '2017',
'availability_type_code': None,
'main_account_code': '0001',
'sub_account_code': '001'
}
tas2_str = concat_tas_dict(tas_dict)
tas2_dstr = concat_display_tas_dict(tas_dict)
assert tas2_str == '09701720172017 0001001'
assert tas2_dstr == '097-017-2017/2017-0001-001'
# everything sans ata
tas_dict = {
'allocation_transfer_agency': None,
'agency_identifier': '017',
'beginning_period_of_availa': '2017',
'ending_period_of_availabil': '2017',
'availability_type_code': 'D',
'main_account_code': '0001',
'sub_account_code': '001'
}
tas2_str = concat_tas_dict(tas_dict)
tas2_dstr = concat_display_tas_dict(tas_dict)
assert tas2_str == '00001720172017D0001001'
assert tas2_dstr == '017-D-0001-001'
# everything sans aid
tas_dict = {
'allocation_transfer_agency': '097',
'agency_identifier': None,
'beginning_period_of_availa': '2017',
'ending_period_of_availabil': '2017',
'availability_type_code': 'D',
'main_account_code': '0001',
'sub_account_code': '001'
}
tas2_str = concat_tas_dict(tas_dict)
tas2_dstr = concat_display_tas_dict(tas_dict)
assert tas2_str == '09700020172017D0001001'
assert tas2_dstr == '097-D-0001-001'
# everything sans periods
tas_dict = {
'allocation_transfer_agency': '097',
'agency_identifier': '017',
'beginning_period_of_availa': None,
'ending_period_of_availabil': None,
'availability_type_code': None,
'main_account_code': '0001',
'sub_account_code': '001'
}
tas2_str = concat_tas_dict(tas_dict)
tas2_dstr = concat_display_tas_dict(tas_dict)
assert tas2_str == '09701700000000 0001001'
assert tas2_dstr == '097-017-0001-001'
# everything sans beginning period
tas_dict = {
'allocation_transfer_agency': '097',
'agency_identifier': '017',
'beginning_period_of_availa': None,
'ending_period_of_availabil': '2017',
'availability_type_code': None,
'main_account_code': '0001',
'sub_account_code': '001'
}
tas2_str = concat_tas_dict(tas_dict)
tas2_dstr = concat_display_tas_dict(tas_dict)
assert tas2_str == '09701700002017 0001001'
assert tas2_dstr == '097-017-2017-0001-001'
# everything sans codes
tas_dict = {
'allocation_transfer_agency': '097',
'agency_identifier': '017',
'beginning_period_of_availa': '2017',
'ending_period_of_availabil': '2017',
'availability_type_code': 'D',
'main_account_code': None,
'sub_account_code': None
}
tas2_str = concat_tas_dict(tas_dict)
tas2_dstr = concat_display_tas_dict(tas_dict)
assert tas2_str == '09701720172017D0000000'
assert tas2_dstr == '097-017-D'
# nothing
tas_dict = {
'allocation_transfer_agency': None,
'agency_identifier': None,
'beginning_period_of_availa': None,
'ending_period_of_availabil': None,
'availability_type_code': None,
'main_account_code': None,
'sub_account_code': None
}
tas2_str = concat_tas_dict(tas_dict)
tas2_dstr = concat_display_tas_dict(tas_dict)
assert tas2_str == '00000000000000 0000000'
assert tas2_dstr == ''
def test_generate_a(database):
sess = database.session
agency_cgac = '097'
year = 2017
tas1_dict = {
'allocation_transfer_agency': agency_cgac,
'agency_identifier': '000',
'beginning_period_of_availa': '2017',
'ending_period_of_availabil': '2017',
'availability_type_code': ' ',
'main_account_code': '0001',
'sub_account_code': '001'
}
tas1_str = concat_tas_dict(tas1_dict)
tas2_dict = {
'allocation_transfer_agency': None,
'agency_identifier': '017',
'beginning_period_of_availa': '2017',
'ending_period_of_availabil': '2017',
'availability_type_code': ' ',
'main_account_code': '0001',
'sub_account_code': '001'
}
tas2_str = concat_tas_dict(tas2_dict)
sf1 = SF133Factory(period=6,
fiscal_year=year,
tas=tas1_str,
line=1160,
amount='1.00',
**tas1_dict)
sf2 = SF133Factory(period=6,
fiscal_year=year,
tas=tas1_str,
line=1180,
amount='2.00',
**tas1_dict)
sf3 = SF133Factory(period=6,
fiscal_year=year,
tas=tas2_str,
line=1000,
amount='4.00',
**tas2_dict)
tas1 = TASFactory(financial_indicator2=' ', **tas1_dict)
tas2 = TASFactory(financial_indicator2=' ', **tas2_dict)
job = JobFactory(job_status_id=JOB_STATUS_DICT['running'],
job_type_id=JOB_TYPE_DICT['file_upload'],
file_type_id=FILE_TYPE_DICT['appropriations'],
filename=None,
start_date='01/01/2017',
end_date='03/31/2017',
submission_id=None)
sess.add_all([sf1, sf2, sf3, tas1, tas2, job])
sess.commit()
file_gen_manager = FileGenerationManager(sess,
CONFIG_BROKER['local'],
job=job)
# providing agency code here as it will be passed via SQS and detached file jobs don't store agency code
file_gen_manager.generate_file(agency_cgac)
assert job.filename is not None
# check headers
file_rows = read_file_rows(job.filename)
assert file_rows[0] == [
key for key in file_generation_manager.fileA.mapping
]
# check body
sf1 = sess.query(SF133).filter_by(tas=tas1_str).first()
sf2 = sess.query(SF133).filter_by(tas=tas2_str).first()
expected1 = []
expected2 = []
sum_cols = [
'total_budgetary_resources_cpe', 'budget_authority_appropria_cpe',
'budget_authority_unobligat_fyb', 'adjustments_to_unobligated_cpe',
'other_budgetary_resources_cpe', 'contract_authority_amount_cpe',
'borrowing_authority_amount_cpe', 'spending_authority_from_of_cpe',
'status_of_budgetary_resour_cpe', 'obligations_incurred_total_cpe',
'gross_outlay_amount_by_tas_cpe', 'unobligated_balance_cpe',
'deobligations_recoveries_r_cpe'
]
zero_sum_cols = {sum_col: '0' for sum_col in sum_cols}
expected1_sum_cols = zero_sum_cols.copy()
expected1_sum_cols['budget_authority_appropria_cpe'] = '3.00'
expected2_sum_cols = zero_sum_cols.copy()
expected2_sum_cols['budget_authority_unobligat_fyb'] = '4.00'
for value in file_generation_manager.fileA.db_columns:
# loop through all values and format date columns
if value in sf1.__dict__:
expected1.append(str(sf1.__dict__[value] or ''))
expected2.append(str(sf2.__dict__[value] or ''))
elif value in expected1_sum_cols:
expected1.append(expected1_sum_cols[value])
expected2.append(expected2_sum_cols[value])
assert expected1 in file_rows
assert expected2 in file_rows
def process_data_chunk(self, sess, chunk_df):
""" Loads in a chunk of the file and performs initial validations
Args:
sess: the database connection
bucket_name: the bucket to pull the file
region_name: the region to pull the file
"""
logger.info({
'message': 'Loading rows starting from {}'.format(self.max_row_number + 1),
'message_type': 'ValidatorInfo',
'submission_id': self.submission_id,
'job_id': self.job.job_id,
'file_type': self.file_type.name,
'action': 'data_loading',
'status': 'start'
})
# initializing warning/error files and dataframes
total_errors = pd.DataFrame(columns=self.report_headers)
total_warnings = pd.DataFrame(columns=self.report_headers)
flex_data = None
required_list = {}
type_list = {}
office_list = {}
# Replace whatever the user included so we're using the database headers
chunk_df.rename(columns=self.reader.header_dict, inplace=True)
empty_file = chunk_df.empty
if not empty_file:
chunk_df = chunk_df.applymap(clean_col)
# Adding row number
chunk_df = chunk_df.reset_index()
# index gets reset for each chunk, adding the header, and adding previous rows
chunk_df['row_number'] = chunk_df.index + 1 + self.max_row_number
self.total_rows += len(chunk_df.index)
# Increment row numbers if any were ignored being too long
# This syncs the row numbers back to their original values
for row in sorted(self.long_rows):
chunk_df.loc[chunk_df['row_number'] >= row, 'row_number'] = chunk_df['row_number'] + 1
# Setting max row number for chunking purposes
self.max_row_number = chunk_df['row_number'].max()
# Filtering out already processed long rows
self.long_rows = [row for row in self.long_rows if row > self.max_row_number]
# Drop rows that were too short and pandas filled in with Nones
chunk_df = chunk_df[~chunk_df['row_number'].isin(self.short_rows)]
# Drop the index column
chunk_df = chunk_df.drop(['index'], axis=1)
# Drop all rows that have 1 or less filled in values (row_number is always filled in so this is how
# we have to drop all rows that are just empty)
chunk_df.dropna(thresh=2, inplace=True)
empty_file = chunk_df.empty
if not empty_file:
if self.is_fabs:
# create a list of all required/type labels for FABS
labels = sess.query(ValidationLabel).all()
for label in labels:
if label.label_type == 'requirement':
required_list[label.column_name] = label.label
else:
type_list[label.column_name] = label.label
# Create a list of all offices
offices = sess.query(Office.office_code, Office.sub_tier_code).all()
for office in offices:
office_list[office.office_code] = office.sub_tier_code
# Clear out office list to save space
del offices
# Gathering flex data (must be done before chunk limiting)
if self.reader.flex_fields:
flex_data = chunk_df.loc[:, list(self.reader.flex_fields + ['row_number'])]
if flex_data is not None and not flex_data.empty:
flex_data['concatted'] = flex_data.apply(lambda x: concat_flex(x), axis=1)
# Dropping any extraneous fields included + flex data (must be done before file type checking)
chunk_df = chunk_df[list(self.expected_headers + ['row_number'])]
# Only do validations if it's not a D file
if self.file_type.name not in ['award', 'award_procurement']:
# Padding specific fields
for field in self.parsed_fields['padded']:
chunk_df[field] = chunk_df.apply(
lambda x: FieldCleaner.pad_field(self.csv_schema[field], x[field]), axis=1)
# Cleaning up numbers so they can be inserted properly
for field in self.parsed_fields['number']:
chunk_df[field] = chunk_df.apply(lambda x: clean_numbers(x[field]), axis=1)
if self.is_fabs:
chunk_df['is_valid'] = True
chunk_df['awarding_sub_tier_agency_c'] = chunk_df.apply(
lambda x: derive_fabs_awarding_sub_tier(x, office_list), axis=1)
chunk_df['afa_generated_unique'] = chunk_df.apply(
lambda x: derive_fabs_afa_generated_unique(x), axis=1)
chunk_df['unique_award_key'] = chunk_df.apply(
lambda x: derive_fabs_unique_award_key(x), axis=1)
else:
chunk_df['tas'] = chunk_df.apply(lambda x: concat_tas_dict(x), axis=1)
chunk_df['display_tas'] = chunk_df.apply(lambda x: concat_display_tas_dict(x), axis=1)
chunk_df['unique_id'] = chunk_df.apply(lambda x: derive_unique_id(x, self.is_fabs), axis=1)
# Separate each of the checks to their own dataframes, then concat them together
req_errors = check_required(chunk_df, self.parsed_fields['required'], required_list,
self.report_headers, self.short_to_long_dict[self.file_type.file_type_id],
flex_data, is_fabs=self.is_fabs)
type_errors = check_type(chunk_df, self.parsed_fields['number'] + self.parsed_fields['boolean'],
type_list, self.report_headers, self.csv_schema,
self.short_to_long_dict[self.file_type.file_type_id], flex_data,
is_fabs=self.is_fabs)
type_error_rows = type_errors['Row Number'].tolist()
length_errors = check_length(chunk_df, self.parsed_fields['length'], self.report_headers,
self.csv_schema, self.short_to_long_dict[self.file_type.file_type_id],
flex_data, type_error_rows)
if self.is_fabs:
error_dfs = [req_errors, type_errors, length_errors]
warning_dfs = [pd.DataFrame(columns=list(self.report_headers + ['error_type']))]
else:
error_dfs = [req_errors, type_errors]
warning_dfs = [length_errors]
total_errors = pd.concat(error_dfs, ignore_index=True)
total_warnings = pd.concat(warning_dfs, ignore_index=True)
# Converting these to ints because pandas likes to change them to floats randomly
total_errors[['Row Number', 'error_type']] = total_errors[['Row Number', 'error_type']].astype(int)
total_warnings[['Row Number', 'error_type']] = total_warnings[['Row Number', 'error_type']]. \
astype(int)
self.error_rows.extend([int(x) for x in total_errors['Row Number'].tolist()])
for index, row in total_errors.iterrows():
self.error_list.record_row_error(self.job.job_id, self.file_name, row['Field Name'],
row['error_type'], row['Row Number'], row['Rule Label'],
self.file_type.file_type_id, None, RULE_SEVERITY_DICT['fatal'])
for index, row in total_warnings.iterrows():
self.error_list.record_row_error(self.job.job_id, self.file_name, row['Field Name'],
row['error_type'], row['Row Number'], row['Rule Label'],
self.file_type.file_type_id, None, RULE_SEVERITY_DICT['warning'])
total_errors.drop(['error_type'], axis=1, inplace=True, errors='ignore')
total_warnings.drop(['error_type'], axis=1, inplace=True, errors='ignore')
# Remove type error rows from original dataframe
chunk_df = chunk_df[~chunk_df['row_number'].isin(type_error_rows)]
chunk_df.drop(['unique_id'], axis=1, inplace=True)
# Write all the errors/warnings to their files
total_errors.to_csv(self.error_file_path, columns=self.report_headers, index=False, quoting=csv.QUOTE_ALL,
mode='a', header=False)
total_warnings.to_csv(self.warning_file_path, columns=self.report_headers, index=False,
quoting=csv.QUOTE_ALL, mode='a', header=False)
# Finally load the data into the database
if not empty_file:
# The model data
now = datetime.now()
chunk_df['created_at'] = now
chunk_df['updated_at'] = now
chunk_df['job_id'] = self.job.job_id
chunk_df['submission_id'] = self.submission_id
insert_dataframe(chunk_df, self.model.__table__.name, sess.connection())
# Flex Fields
if flex_data is not None:
flex_data.drop(['concatted'], axis=1, inplace=True)
flex_data = flex_data[flex_data['row_number'].isin(chunk_df['row_number'])]
flex_rows = pd.melt(flex_data, id_vars=['row_number'], value_vars=self.reader.flex_fields,
var_name='header', value_name='cell')
# Filling in all the shared data for these flex fields
now = datetime.now()
flex_rows['created_at'] = now
flex_rows['updated_at'] = now
flex_rows['job_id'] = self.job.job_id
flex_rows['submission_id'] = self.submission_id
flex_rows['file_type_id'] = self.file_type.file_type_id
# Adding the entire set of flex fields
insert_dataframe(flex_rows, FlexField.__table__.name, sess.connection())
sess.commit()
logger.info({
'message': 'Loaded rows up to {}'.format(self.max_row_number),
'message_type': 'ValidatorInfo',
'submission_id': self.submission_id,
'job_id': self.job.job_id,
'file_type': self.file_type.name,
'action': 'data_loading',
'status': 'end'
})
def setUpClass(cls):
""" Set up class-wide resources (test data) """
super(ErrorWarningTests, cls).setUpClass()
logging.getLogger('dataactcore').setLevel(logging.ERROR)
logging.getLogger('dataactvalidator').setLevel(logging.ERROR)
with create_app().app_context():
# get the submission test users
sess = GlobalDB.db().session
cls.session = sess
# set up default e-mails for tests
admin_user = sess.query(User).filter(
User.email == cls.test_users['admin_user']).one()
cls.validator = ValidationManager(
directory=CONFIG_SERVICES['error_report_path'])
# Just have one valid submission and then keep on reloading files
cls.submission_id = insert_submission(sess,
admin_user.user_id,
cgac_code='SYS',
start_date='01/2001',
end_date='03/2001',
is_quarter=True)
cls.submission = sess.query(Submission).filter_by(
submission_id=cls.submission_id).one()
cls.val_job = insert_job(
cls.session,
FILE_TYPE_DICT['appropriations'],
JOB_STATUS_DICT['ready'],
JOB_TYPE_DICT['csv_record_validation'],
cls.submission_id,
filename=JOB_TYPE_DICT['csv_record_validation'])
cls.original_reports = set(
os.listdir(CONFIG_SERVICES['error_report_path']))
# adding TAS to ensure valid file is valid
tas1 = TASFactory(account_num=1,
allocation_transfer_agency='019',
agency_identifier='072',
beginning_period_of_availa=None,
ending_period_of_availabil=None,
availability_type_code='X',
main_account_code='0306',
sub_account_code='000',
internal_start_date='01-01-2000')
tas2 = TASFactory(account_num=2,
allocation_transfer_agency=None,
agency_identifier='019',
beginning_period_of_availa='2016',
ending_period_of_availabil='2016',
availability_type_code=None,
main_account_code='0113',
sub_account_code='000',
internal_start_date='01-01-2000')
tas3 = TASFactory(account_num=3,
allocation_transfer_agency=None,
agency_identifier='028',
beginning_period_of_availa=None,
ending_period_of_availabil=None,
availability_type_code='X',
main_account_code='0406',
sub_account_code='000',
internal_start_date='01-01-2000')
tas4 = TASFactory(account_num=4,
allocation_transfer_agency=None,
agency_identifier='028',
beginning_period_of_availa='2010',
ending_period_of_availabil='2011',
availability_type_code=None,
main_account_code='0406',
sub_account_code='000',
internal_start_date='01-01-2000')
tas5 = TASFactory(account_num=5,
allocation_transfer_agency='069',
agency_identifier='013',
beginning_period_of_availa=None,
ending_period_of_availabil=None,
availability_type_code='X',
main_account_code='2050',
sub_account_code='005',
internal_start_date='01-01-2000')
tas6 = TASFactory(account_num=6,
allocation_transfer_agency='028',
agency_identifier='028',
beginning_period_of_availa=None,
ending_period_of_availabil=None,
availability_type_code='X',
main_account_code='8007',
sub_account_code='000',
internal_start_date='01-01-2000')
tas7 = TASFactory(account_num=7,
allocation_transfer_agency=None,
agency_identifier='049',
beginning_period_of_availa=None,
ending_period_of_availabil=None,
availability_type_code='X',
main_account_code='0100',
sub_account_code='000',
internal_start_date='01-01-2000')
tas8 = TASFactory(account_num=8,
allocation_transfer_agency=None,
agency_identifier='049',
beginning_period_of_availa='2010',
ending_period_of_availabil='2011',
availability_type_code=None,
main_account_code='0100',
sub_account_code='000',
internal_start_date='01-01-2000')
tas9 = TASFactory(account_num=9,
allocation_transfer_agency=None,
agency_identifier='049',
beginning_period_of_availa='2014',
ending_period_of_availabil='2015',
availability_type_code=None,
main_account_code='0100',
sub_account_code='000',
internal_start_date='01-01-2000')
tas10 = TASFactory(account_num=10,
allocation_transfer_agency=None,
agency_identifier='049',
beginning_period_of_availa='2015',
ending_period_of_availabil='2016',
availability_type_code=None,
main_account_code='0100',
sub_account_code='000',
internal_start_date='01-01-2000')
sess.add_all(
[tas1, tas2, tas3, tas4, tas5, tas6, tas7, tas8, tas9, tas10])
# adding GTAS to ensure valid file is valid
gtas1 = SF133Factory(tas=concat_tas_dict(tas1.component_dict()),
allocation_transfer_agency='019',
agency_identifier='072',
beginning_period_of_availa=None,
line=1009,
ending_period_of_availabil=None,
availability_type_code='X',
main_account_code='0306',
sub_account_code='000',
period=6,
fiscal_year=2001)
gtas2 = SF133Factory(tas=concat_tas_dict(tas2.component_dict()),
allocation_transfer_agency=None,
agency_identifier='019',
beginning_period_of_availa='2016',
line=1009,
ending_period_of_availabil='2016',
availability_type_code=None,
main_account_code='0113',
sub_account_code='000',
period=6,
fiscal_year=2001)
gtas3 = SF133Factory(tas=concat_tas_dict(tas3.component_dict()),
allocation_transfer_agency=None,
agency_identifier='028',
beginning_period_of_availa=None,
line=1009,
ending_period_of_availabil=None,
availability_type_code='X',
main_account_code='0406',
sub_account_code='000',
period=6,
fiscal_year=2001)
gtas4 = SF133Factory(tas=concat_tas_dict(tas4.component_dict()),
allocation_transfer_agency=None,
agency_identifier='028',
beginning_period_of_availa='2010',
line=1009,
ending_period_of_availabil='2011',
availability_type_code=None,
main_account_code='0406',
sub_account_code='000',
period=6,
fiscal_year=2001)
gtas5 = SF133Factory(tas=concat_tas_dict(tas5.component_dict()),
allocation_transfer_agency='069',
agency_identifier='013',
beginning_period_of_availa=None,
line=1009,
ending_period_of_availabil=None,
availability_type_code='X',
main_account_code='2050',
sub_account_code='005',
period=6,
fiscal_year=2001)
gtas6 = SF133Factory(tas=concat_tas_dict(tas6.component_dict()),
allocation_transfer_agency='028',
agency_identifier='028',
beginning_period_of_availa=None,
line=1009,
ending_period_of_availabil=None,
availability_type_code='X',
main_account_code='8007',
sub_account_code='000',
period=6,
fiscal_year=2001)
gtas7 = SF133Factory(tas=concat_tas_dict(tas7.component_dict()),
allocation_transfer_agency=None,
agency_identifier='049',
beginning_period_of_availa=None,
line=1009,
ending_period_of_availabil=None,
availability_type_code='X',
main_account_code='0100',
sub_account_code='000',
period=6,
fiscal_year=2001)
gtas8 = SF133Factory(tas=concat_tas_dict(tas8.component_dict()),
allocation_transfer_agency=None,
agency_identifier='049',
beginning_period_of_availa='2010',
line=1009,
ending_period_of_availabil='2011',
availability_type_code=None,
main_account_code='0100',
sub_account_code='000',
period=6,
fiscal_year=2001)
gtas9 = SF133Factory(tas=concat_tas_dict(tas9.component_dict()),
allocation_transfer_agency=None,
agency_identifier='049',
beginning_period_of_availa='2014',
line=1009,
ending_period_of_availabil='2015',
availability_type_code=None,
main_account_code='0100',
sub_account_code='000',
period=6,
fiscal_year=2001)
gtas10 = SF133Factory(tas=concat_tas_dict(tas10.component_dict()),
allocation_transfer_agency=None,
agency_identifier='049',
beginning_period_of_availa='2015',
line=1009,
ending_period_of_availabil='2016',
availability_type_code=None,
main_account_code='0100',
sub_account_code='000',
period=6,
fiscal_year=2001)
sess.add_all([
gtas1, gtas2, gtas3, gtas4, gtas5, gtas6, gtas7, gtas8, gtas9,
gtas10
])
sess.commit()
def test_generate_a(mock_broker_config_paths, database):
sess = database.session
agency_cgac = '097'
year = 2017
tas1_dict = {
'allocation_transfer_agency': agency_cgac,
'agency_identifier': '000',
'beginning_period_of_availa': '2017',
'ending_period_of_availabil': '2017',
'availability_type_code': ' ',
'main_account_code': '0001',
'sub_account_code': '001'
}
tas1_str = concat_tas_dict(tas1_dict)
tas2_dict = {
'allocation_transfer_agency': None,
'agency_identifier': '017',
'beginning_period_of_availa': '2017',
'ending_period_of_availabil': '2017',
'availability_type_code': ' ',
'main_account_code': '0001',
'sub_account_code': '001'
}
tas2_str = concat_tas_dict(tas2_dict)
sf1 = SF133Factory(period=6, fiscal_year=year, tas=tas1_str, line=1160, amount='1.00', **tas1_dict)
sf2 = SF133Factory(period=6, fiscal_year=year, tas=tas1_str, line=1180, amount='2.00', **tas1_dict)
sf3 = SF133Factory(period=6, fiscal_year=year, tas=tas2_str, line=1000, amount='4.00', **tas2_dict)
tas1 = TASFactory(financial_indicator2=' ', **tas1_dict)
tas2 = TASFactory(financial_indicator2=' ', **tas2_dict)
job = JobFactory(job_status_id=JOB_STATUS_DICT['running'], job_type_id=JOB_TYPE_DICT['file_upload'],
file_type_id=FILE_TYPE_DICT['appropriations'], filename=None, start_date='01/01/2017',
end_date='03/31/2017', submission_id=None)
sess.add_all([sf1, sf2, sf3, tas1, tas2, job])
sess.commit()
file_gen_manager = FileGenerationManager(sess, CONFIG_BROKER['local'], job=job)
# providing agency code here as it will be passed via SQS and detached file jobs don't store agency code
file_gen_manager.generate_file(agency_cgac)
assert job.filename is not None
# check headers
file_rows = read_file_rows(job.filename)
assert file_rows[0] == [key for key in file_generation_manager.fileA.mapping]
# check body
sf1 = sess.query(SF133).filter_by(tas=tas1_str).first()
sf2 = sess.query(SF133).filter_by(tas=tas2_str).first()
expected1 = []
expected2 = []
sum_cols = [
'total_budgetary_resources_cpe',
'budget_authority_appropria_cpe',
'budget_authority_unobligat_fyb',
'adjustments_to_unobligated_cpe',
'other_budgetary_resources_cpe',
'contract_authority_amount_cpe',
'borrowing_authority_amount_cpe',
'spending_authority_from_of_cpe',
'status_of_budgetary_resour_cpe',
'obligations_incurred_total_cpe',
'gross_outlay_amount_by_tas_cpe',
'unobligated_balance_cpe',
'deobligations_recoveries_r_cpe'
]
zero_sum_cols = {sum_col: '0' for sum_col in sum_cols}
expected1_sum_cols = zero_sum_cols.copy()
expected1_sum_cols['budget_authority_appropria_cpe'] = '3.00'
expected2_sum_cols = zero_sum_cols.copy()
expected2_sum_cols['budget_authority_unobligat_fyb'] = '4.00'
for value in file_generation_manager.fileA.db_columns:
# loop through all values and format date columns
if value in sf1.__dict__:
expected1.append(str(sf1.__dict__[value] or ''))
expected2.append(str(sf2.__dict__[value] or ''))
elif value in expected1_sum_cols:
expected1.append(expected1_sum_cols[value])
expected2.append(expected2_sum_cols[value])
assert expected1 in file_rows
assert expected2 in file_rows