def test_generate_file_updates_jobs(monkeypatch, mock_broker_config_paths, database): sess = database.session job1 = JobFactory(job_status_id=JOB_STATUS_DICT['running'], job_type_id=JOB_TYPE_DICT['file_upload'], file_type_id=FILE_TYPE_DICT['award_procurement'], filename=None, original_filename=None, start_date='01/01/2017', end_date='01/31/2017') job2 = JobFactory(job_status_id=JOB_STATUS_DICT['running'], job_type_id=JOB_TYPE_DICT['file_upload'], file_type_id=FILE_TYPE_DICT['award_procurement'], filename=None, original_filename=None, start_date='01/01/2017', end_date='01/31/2017') job3 = JobFactory(job_status_id=JOB_STATUS_DICT['running'], job_type_id=JOB_TYPE_DICT['file_upload'], file_type_id=FILE_TYPE_DICT['award_procurement'], filename=None, original_filename=None, start_date='01/01/2017', end_date='01/31/2017') file_gen = FileGenerationFactory(request_date=datetime.now().date(), start_date='01/01/2017', end_date='01/31/2017', file_type='D1', agency_code='123', agency_type='awarding', is_cached_file=True, file_path=None) sess.add_all([job1, job2, job3, file_gen]) sess.commit() job1.file_generation_id = file_gen.file_generation_id job2.file_generation_id = file_gen.file_generation_id job3.file_generation_id = file_gen.file_generation_id sess.commit() monkeypatch.setattr(generation_helper, 'g', Mock(return_value={'is_local': CONFIG_BROKER['local']})) file_gen_manager = FileGenerationManager(sess, CONFIG_BROKER['local'], file_generation=file_gen) file_gen_manager.generate_file() sess.refresh(file_gen) original_filename = file_gen.file_path.split('/')[-1] assert job1.job_status_id == JOB_STATUS_DICT['finished'] assert job1.original_filename == original_filename assert job1.filename == '{}{}'.format( CONFIG_BROKER['broker_files'] if CONFIG_BROKER['local'] else job1.submission_id + '/', original_filename) assert job2.job_status_id == JOB_STATUS_DICT['finished'] assert job2.original_filename == original_filename assert job2.filename == '{}{}'.format( CONFIG_BROKER['broker_files'] if CONFIG_BROKER['local'] else job2.submission_id + '/', original_filename) assert job3.job_status_id == JOB_STATUS_DICT['finished'] assert job3.original_filename == original_filename assert job3.filename == '{}{}'.format( CONFIG_BROKER['broker_files'] if CONFIG_BROKER['local'] else job3.submission_id + '/', original_filename)
def test_generate_e_file_csv(monkeypatch, mock_broker_config_paths, database): """ Verify that an appropriate CSV is written, based on fileE.Row's structure """ # Create an award so that we have _a_ duns sub = SubmissionFactory() database.session.add(sub) database.session.commit() ap = AwardProcurementFactory(submission_id=sub.submission_id) database.session.add(ap) database.session.commit() file_path = str(mock_broker_config_paths['broker_files'].join('e_test1')) job = JobFactory(job_status_id=JOB_STATUS_DICT['running'], job_type_id=JOB_TYPE_DICT['file_upload'], file_type_id=FILE_TYPE_DICT['executive_compensation'], filename=file_path, original_filename='e_test1', submission_id=sub.submission_id) database.session.add(job) database.session.commit() monkeypatch.setattr(file_generation_manager.fileE, 'row_to_dict', Mock()) file_generation_manager.fileE.row_to_dict.return_value = {} monkeypatch.setattr(file_generation_manager.fileE, 'retrieve_rows', Mock()) file_generation_manager.fileE.retrieve_rows.return_value = [ fileE.Row('a', 'b', 'c', 'd', '1a', '1b', '2a', '2b', '3a', '3b', '4a', '4b', '5a', '5b'), fileE.Row('A', 'B', 'C', 'D', '1A', '1B', '2A', '2B', '3A', '3B', '4A', '4B', '5A', '5B') ] file_gen_manager = FileGenerationManager(database.session, CONFIG_BROKER['local'], job=job) file_gen_manager.generate_file() expected = [ ['AwardeeOrRecipientUniqueIdentifier', 'AwardeeOrRecipientLegalEntityName', 'UltimateParentUniqueIdentifier', 'UltimateParentLegalEntityName', 'HighCompOfficer1FullName', 'HighCompOfficer1Amount', 'HighCompOfficer2FullName', 'HighCompOfficer2Amount', 'HighCompOfficer3FullName', 'HighCompOfficer3Amount', 'HighCompOfficer4FullName', 'HighCompOfficer4Amount', 'HighCompOfficer5FullName', 'HighCompOfficer5Amount'], ['a', 'b', 'c', 'd', '1a', '1b', '2a', '2b', '3a', '3b', '4a', '4b', '5a', '5b'], ['A', 'B', 'C', 'D', '1A', '1B', '2A', '2B', '3A', '3B', '4A', '4B', '5A', '5B'] ] assert read_file_rows(file_path) == expected
def test_generate_funding_d2(mock_broker_config_paths, database): sess = database.session pafa = PublishedAwardFinancialAssistanceFactory pafa_1 = pafa(funding_agency_code='123', action_date='20170101', afa_generated_unique='unique1', is_active=True) pafa_2 = pafa(funding_agency_code='123', action_date='20170131', afa_generated_unique='unique2', is_active=True) pafa_3 = pafa(funding_agency_code='123', action_date='20161231', afa_generated_unique='unique3', is_active=True) pafa_4 = pafa(funding_agency_code='123', action_date='20170201', afa_generated_unique='unique4', is_active=True) pafa_5 = pafa(funding_agency_code='123', action_date='20170115', afa_generated_unique='unique5', is_active=False) pafa_6 = pafa(funding_agency_code='234', action_date='20170115', afa_generated_unique='unique6', is_active=True) file_gen = FileGenerationFactory(request_date=datetime.now().date(), start_date='01/01/2017', end_date='01/31/2017', file_type='D2', agency_code='123', agency_type='funding', is_cached_file=True, file_path=None) sess.add_all([pafa_1, pafa_2, pafa_3, pafa_4, pafa_5, pafa_6, file_gen]) sess.commit() file_gen_manager = FileGenerationManager(sess, CONFIG_BROKER['local'], file_generation=file_gen) file_gen_manager.generate_file() assert file_gen.file_path is not None # check headers file_rows = read_file_rows(file_gen.file_path) assert file_rows[0] == [key for key in file_generation_manager.fileD2.mapping] # check body pafa1 = sess.query(PublishedAwardFinancialAssistance).filter_by(afa_generated_unique='unique1').first() pafa2 = sess.query(PublishedAwardFinancialAssistance).filter_by(afa_generated_unique='unique2').first() expected1, expected2 = [], [] for value in file_generation_manager.fileD2.db_columns: # loop through all values and format date columns if value in ['period_of_performance_star', 'period_of_performance_curr', 'modified_at', 'action_date']: expected1.append(re.sub(r"[-]", r"", str(pafa1.__dict__[value]))[0:8]) expected2.append(re.sub(r"[-]", r"", str(pafa2.__dict__[value]))[0:8]) else: expected1.append(str(pafa1.__dict__[value])) expected2.append(str(pafa2.__dict__[value])) assert expected1 in file_rows assert expected2 in file_rows
def test_generate_funding_d1(mock_broker_config_paths, database): sess = database.session dap_model = DetachedAwardProcurementFactory dap_1 = dap_model(funding_agency_code='123', action_date='20170101', detached_award_proc_unique='unique1') dap_2 = dap_model(funding_agency_code='123', action_date='20170131', detached_award_proc_unique='unique2') dap_3 = dap_model(funding_agency_code='123', action_date='20170201', detached_award_proc_unique='unique3') dap_4 = dap_model(funding_agency_code='123', action_date='20161231', detached_award_proc_unique='unique4') dap_5 = dap_model(funding_agency_code='234', action_date='20170115', detached_award_proc_unique='unique5') file_gen = FileGenerationFactory(request_date=datetime.now().date(), start_date='01/01/2017', end_date='01/31/2017', file_type='D1', agency_code='123', agency_type='funding', is_cached_file=True, file_path=None) sess.add_all([dap_1, dap_2, dap_3, dap_4, dap_5, file_gen]) sess.commit() file_gen_manager = FileGenerationManager(sess, CONFIG_BROKER['local'], file_generation=file_gen) file_gen_manager.generate_file() assert file_gen.file_path is not None # check headers file_rows = read_file_rows(file_gen.file_path) assert file_rows[0] == [key for key in file_generation_manager.fileD1.mapping] # check body dap_one = sess.query(DetachedAwardProcurement).filter_by(detached_award_proc_unique='unique1').first() dap_two = sess.query(DetachedAwardProcurement).filter_by(detached_award_proc_unique='unique2').first() expected1, expected2 = [], [] for value in file_generation_manager.fileD1.db_columns: # loop through all values and format date columns if value in ['period_of_performance_star', 'period_of_performance_curr', 'period_of_perf_potential_e', 'ordering_period_end_date', 'action_date', 'last_modified', 'solicitation_date']: expected1.append(re.sub(r"[-]", r"", str(dap_one.__dict__[value]))[0:8]) expected2.append(re.sub(r"[-]", r"", str(dap_two.__dict__[value]))[0:8]) else: expected1.append(str(dap_one.__dict__[value])) expected2.append(str(dap_two.__dict__[value])) assert expected1 in file_rows assert expected2 in file_rows
def test_generate_e_file_query(monkeypatch, mock_broker_config_paths, database): """ Verify that generate_e_file makes an appropriate query (matching both D1 and D2 entries) """ # Generate several file D1 entries, largely with the same submission_id, and with two overlapping DUNS. Generate # several D2 entries with the same submission_id as well sess = database.session sub = SubmissionFactory() sub_2 = SubmissionFactory() sess.add_all([sub, sub_2]) sess.commit() file_path = str(mock_broker_config_paths['broker_files'].join('e_test1')) job = JobFactory(job_status_id=JOB_STATUS_DICT['running'], job_type_id=JOB_TYPE_DICT['file_upload'], file_type_id=FILE_TYPE_DICT['executive_compensation'], filename=file_path, original_filename='e_test1', submission_id=sub.submission_id) database.session.add(job) database.session.commit() model = AwardProcurementFactory(submission_id=sub.submission_id) aps = [AwardProcurementFactory(submission_id=sub.submission_id) for _ in range(4)] afas = [AwardFinancialAssistanceFactory(submission_id=sub.submission_id) for _ in range(5)] same_duns = AwardProcurementFactory( submission_id=sub.submission_id, awardee_or_recipient_uniqu=model.awardee_or_recipient_uniqu) unrelated = AwardProcurementFactory(submission_id=sub_2.submission_id) sess.add_all(aps + afas + [model, same_duns, unrelated]) sess.commit() monkeypatch.setattr(file_generation_manager.fileE, 'retrieve_rows', Mock(return_value=[])) file_gen_manager = FileGenerationManager(database.session, CONFIG_BROKER['local'], job=job) file_gen_manager.generate_file() # [0][0] gives us the first, non-keyword args call_args = file_generation_manager.fileE.retrieve_rows.call_args[0][0] expected = [ap.awardee_or_recipient_uniqu for ap in aps] expected.append(model.awardee_or_recipient_uniqu) expected.extend(afa.awardee_or_recipient_uniqu for afa in afas) assert list(sorted(call_args)) == list(sorted(expected))
def validator_process_job(job_id, agency_code, is_retry=False): """ Retrieves a Job based on its ID, and kicks off a validation. Handles errors by ensuring the Job (if exists) is no longer running. Args: job_id: ID of a Job agency_code: CGAC or FREC code for agency, only required for file generations by Job is_retry: If this is not the very first time handling execution of this job. If True, cleanup is performed before proceeding to retry the job Raises: Any Exceptions raised by the GenerationManager or ValidationManager, excluding those explicitly handled """ if is_retry: if cleanup_validation(job_id): log_job_message( logger=logger, message= "Attempting a retry of {} after successful retry-cleanup.". format(inspect.stack()[0][3]), job_id=job_id, is_debug=True) else: log_job_message( logger=logger, message="Retry of {} found to be not necessary after cleanup. " "Returning from job with success.".format( inspect.stack()[0][3]), job_id=job_id, is_debug=True) return sess = GlobalDB.db().session job = None try: # Get the job job = sess.query(Job).filter_by(job_id=job_id).one_or_none() if job is None: validation_error_type = ValidationError.jobError write_file_error(job_id, None, validation_error_type) raise ResponseException( 'Job ID {} not found in database'.format(job_id), StatusCode.CLIENT_ERROR, None, validation_error_type) mark_job_status(job_id, 'ready') # We can either validate or generate a file based on Job ID if job.job_type.name == 'file_upload': # Generate A, E, or F file file_generation_manager = FileGenerationManager(sess, g.is_local, job=job) file_generation_manager.generate_file(agency_code) else: # Run validations validation_manager = ValidationManager( g.is_local, CONFIG_SERVICES['error_report_path']) validation_manager.validate_job(job.job_id) except (ResponseException, csv.Error, UnicodeDecodeError, ValueError) as e: # Handle exceptions explicitly raised during validation error_data = { 'message': 'An exception occurred in the Validator', 'message_type': 'ValidatorInfo', 'job_id': job_id, 'traceback': traceback.format_exc() } if job: error_data.update({ 'submission_id': job.submission_id, 'file_type': job.file_type.name }) logger.error(error_data) sess.refresh(job) job.error_message = str(e) if job.filename is not None: error_type = ValidationError.unknownError if isinstance(e, UnicodeDecodeError): error_type = ValidationError.encodingError elif isinstance(e, ResponseException): error_type = e.errorType write_file_error(job.job_id, job.filename, error_type) mark_job_status(job.job_id, 'invalid') else: logger.error(error_data) raise e except Exception as e: # Log uncaught exceptions and fail the Job error_data = { 'message': 'An unhandled exception occurred in the Validator', 'message_type': 'ValidatorInfo', 'job_id': job_id, 'traceback': traceback.format_exc() } if job: error_data.update({ 'submission_id': job.submission_id, 'file_type': job.file_type.name }) logger.error(error_data) # Try to mark the Job as failed, but continue raising the original Exception if not possible try: mark_job_status(job_id, 'failed') sess.refresh(job) job.error_message = str(e) sess.commit() except: pass raise e
def validator_process_file_generation(file_gen_id, is_retry=False): """ Retrieves a FileGeneration object based on its ID, and kicks off a file generation. Handles errors by ensuring the FileGeneration (if exists) is no longer cached. Args: file_gen_id: ID of a FileGeneration object is_retry: If this is not the very first time handling execution of this job. If True, cleanup is performed before proceeding to retry the job Raises: Any Exceptions raised by the FileGenerationManager """ if is_retry: if cleanup_generation(file_gen_id): log_job_message( logger=logger, message= "Attempting a retry of {} after successful retry-cleanup.". format(inspect.stack()[0][3]), job_id=file_gen_id, is_debug=True) else: log_job_message( logger=logger, message="Retry of {} found to be not necessary after cleanup. " "Returning from job with success.".format( inspect.stack()[0][3]), job_id=file_gen_id, is_debug=True) return sess = GlobalDB.db().session file_generation = None try: file_generation = sess.query(FileGeneration).filter_by( file_generation_id=file_gen_id).one_or_none() if file_generation is None: raise ResponseException( 'FileGeneration ID {} not found in database'.format( file_gen_id), StatusCode.CLIENT_ERROR, None) file_generation_manager = FileGenerationManager( sess, g.is_local, file_generation=file_generation) file_generation_manager.generate_file() except Exception as e: # Log uncaught exceptions and fail all Jobs referencing this FileGeneration error_data = { 'message': 'An unhandled exception occurred in the Validator during file generation', 'message_type': 'ValidatorInfo', 'file_generation_id': file_gen_id, 'traceback': traceback.format_exc() } if file_generation: error_data.update({ 'agency_code': file_generation.agency_code, 'agency_type': file_generation.agency_type, 'start_date': file_generation.start_date, 'end_date': file_generation.end_date, 'file_type': file_generation.file_type, 'file_path': file_generation.file_path, }) logger.error(error_data) # Try to mark the Jobs as failed, but continue raising the original Exception if not possible try: if file_generation: # Uncache the FileGeneration sess.refresh(file_generation) file_generation.is_cached_file = False # Mark all Jobs waiting on this FileGeneration as failed generation_jobs = sess.query(Job).filter_by( file_generation_id=file_gen_id).all() for job in generation_jobs: if job.job_status in [ JOB_STATUS_DICT['waiting'], JOB_STATUS_DICT['ready'], JOB_STATUS_DICT['running'] ]: mark_job_status(job.job_id, 'failed') sess.refresh(job) job.file_generation_id = None job.error_message = str(e) sess.commit() except: pass # ResponseExceptions only occur at very specific times, and should not affect the Validator's future attempts # at handling messages from SQS if not isinstance(e, ResponseException): raise e
def test_generate_e_file(mock_broker_config_paths, database): """ Verify that generate_e_file makes an appropriate query (matching both D1 and D2 entries) and creates a file matching the expected DUNS """ # Generate several file D1 entries, largely with the same submission_id, and with two overlapping DUNS. Generate # several D2 entries with the same submission_id as well sess = database.session sub = SubmissionFactory() sub_2 = SubmissionFactory() sess.add_all([sub, sub_2]) sess.commit() file_path = str(mock_broker_config_paths['broker_files'].join('e_test1')) job = JobFactory(job_status_id=JOB_STATUS_DICT['running'], job_type_id=JOB_TYPE_DICT['file_upload'], file_type_id=FILE_TYPE_DICT['executive_compensation'], filename=file_path, original_filename='e_test1', submission_id=sub.submission_id) database.session.add(job) database.session.commit() model = AwardProcurementFactory(submission_id=sub.submission_id) aps = [ AwardProcurementFactory(submission_id=sub.submission_id) for _ in range(4) ] afas = [ AwardFinancialAssistanceFactory(submission_id=sub.submission_id) for _ in range(5) ] same_duns = AwardProcurementFactory( submission_id=sub.submission_id, awardee_or_recipient_uniqu=model.awardee_or_recipient_uniqu) unrelated = AwardProcurementFactory(submission_id=sub_2.submission_id) duns_list = [ DunsFactory( awardee_or_recipient_uniqu=model.awardee_or_recipient_uniqu) ] duns_list.extend([ DunsFactory(awardee_or_recipient_uniqu=ap.awardee_or_recipient_uniqu) for ap in aps ]) duns_list.extend([ DunsFactory(awardee_or_recipient_uniqu=afa.awardee_or_recipient_uniqu) for afa in afas ]) sess.add_all(aps + afas + duns_list + [model, same_duns, unrelated]) sess.commit() file_gen_manager = FileGenerationManager(database.session, CONFIG_BROKER['local'], job=job) file_gen_manager.generate_file() # check headers file_rows = read_file_rows(file_path) assert file_rows[0] == [ 'AwardeeOrRecipientUniqueIdentifier', 'AwardeeOrRecipientLegalEntityName', 'UltimateParentUniqueIdentifier', 'UltimateParentLegalEntityName', 'HighCompOfficer1FullName', 'HighCompOfficer1Amount', 'HighCompOfficer2FullName', 'HighCompOfficer2Amount', 'HighCompOfficer3FullName', 'HighCompOfficer3Amount', 'HighCompOfficer4FullName', 'HighCompOfficer4Amount', 'HighCompOfficer5FullName', 'HighCompOfficer5Amount' ] # Check listed DUNS expected = [[ duns.awardee_or_recipient_uniqu, duns.legal_business_name, duns.ultimate_parent_unique_ide, duns.ultimate_parent_legal_enti, duns.high_comp_officer1_full_na, duns.high_comp_officer1_amount, duns.high_comp_officer2_full_na, duns.high_comp_officer2_amount, duns.high_comp_officer3_full_na, duns.high_comp_officer3_amount, duns.high_comp_officer4_full_na, duns.high_comp_officer4_amount, duns.high_comp_officer5_full_na, duns.high_comp_officer5_amount ] for duns in duns_list] received = [file_row for file_row in file_rows[1:]] assert sorted(received) == list(sorted(expected))
def test_generate_a(database): sess = database.session agency_cgac = '097' year = 2017 tas1_dict = { 'allocation_transfer_agency': agency_cgac, 'agency_identifier': '000', 'beginning_period_of_availa': '2017', 'ending_period_of_availabil': '2017', 'availability_type_code': ' ', 'main_account_code': '0001', 'sub_account_code': '001' } tas1_str = concat_tas_dict(tas1_dict) tas2_dict = { 'allocation_transfer_agency': None, 'agency_identifier': '017', 'beginning_period_of_availa': '2017', 'ending_period_of_availabil': '2017', 'availability_type_code': ' ', 'main_account_code': '0001', 'sub_account_code': '001' } tas2_str = concat_tas_dict(tas2_dict) sf1 = SF133Factory(period=6, fiscal_year=year, tas=tas1_str, line=1160, amount='1.00', **tas1_dict) sf2 = SF133Factory(period=6, fiscal_year=year, tas=tas1_str, line=1180, amount='2.00', **tas1_dict) sf3 = SF133Factory(period=6, fiscal_year=year, tas=tas2_str, line=1000, amount='4.00', **tas2_dict) tas1 = TASFactory(financial_indicator2=' ', **tas1_dict) tas2 = TASFactory(financial_indicator2=' ', **tas2_dict) job = JobFactory(job_status_id=JOB_STATUS_DICT['running'], job_type_id=JOB_TYPE_DICT['file_upload'], file_type_id=FILE_TYPE_DICT['appropriations'], filename=None, start_date='01/01/2017', end_date='03/31/2017', submission_id=None) sess.add_all([sf1, sf2, sf3, tas1, tas2, job]) sess.commit() file_gen_manager = FileGenerationManager(sess, CONFIG_BROKER['local'], job=job) # providing agency code here as it will be passed via SQS and detached file jobs don't store agency code file_gen_manager.generate_file(agency_cgac) assert job.filename is not None # check headers file_rows = read_file_rows(job.filename) assert file_rows[0] == [ key for key in file_generation_manager.fileA.mapping ] # check body sf1 = sess.query(SF133).filter_by(tas=tas1_str).first() sf2 = sess.query(SF133).filter_by(tas=tas2_str).first() expected1 = [] expected2 = [] sum_cols = [ 'total_budgetary_resources_cpe', 'budget_authority_appropria_cpe', 'budget_authority_unobligat_fyb', 'adjustments_to_unobligated_cpe', 'other_budgetary_resources_cpe', 'contract_authority_amount_cpe', 'borrowing_authority_amount_cpe', 'spending_authority_from_of_cpe', 'status_of_budgetary_resour_cpe', 'obligations_incurred_total_cpe', 'gross_outlay_amount_by_tas_cpe', 'unobligated_balance_cpe', 'deobligations_recoveries_r_cpe' ] zero_sum_cols = {sum_col: '0' for sum_col in sum_cols} expected1_sum_cols = zero_sum_cols.copy() expected1_sum_cols['budget_authority_appropria_cpe'] = '3.00' expected2_sum_cols = zero_sum_cols.copy() expected2_sum_cols['budget_authority_unobligat_fyb'] = '4.00' for value in file_generation_manager.fileA.db_columns: # loop through all values and format date columns if value in sf1.__dict__: expected1.append(str(sf1.__dict__[value] or '')) expected2.append(str(sf2.__dict__[value] or '')) elif value in expected1_sum_cols: expected1.append(expected1_sum_cols[value]) expected2.append(expected2_sum_cols[value]) assert expected1 in file_rows assert expected2 in file_rows
def test_generate_a(mock_broker_config_paths, database): sess = database.session agency_cgac = '097' year = 2017 tas1_dict = { 'allocation_transfer_agency': agency_cgac, 'agency_identifier': '000', 'beginning_period_of_availa': '2017', 'ending_period_of_availabil': '2017', 'availability_type_code': ' ', 'main_account_code': '0001', 'sub_account_code': '001' } tas1_str = concat_tas_dict(tas1_dict) tas2_dict = { 'allocation_transfer_agency': None, 'agency_identifier': '017', 'beginning_period_of_availa': '2017', 'ending_period_of_availabil': '2017', 'availability_type_code': ' ', 'main_account_code': '0001', 'sub_account_code': '001' } tas2_str = concat_tas_dict(tas2_dict) sf1 = SF133Factory(period=6, fiscal_year=year, tas=tas1_str, line=1160, amount='1.00', **tas1_dict) sf2 = SF133Factory(period=6, fiscal_year=year, tas=tas1_str, line=1180, amount='2.00', **tas1_dict) sf3 = SF133Factory(period=6, fiscal_year=year, tas=tas2_str, line=1000, amount='4.00', **tas2_dict) tas1 = TASFactory(financial_indicator2=' ', **tas1_dict) tas2 = TASFactory(financial_indicator2=' ', **tas2_dict) job = JobFactory(job_status_id=JOB_STATUS_DICT['running'], job_type_id=JOB_TYPE_DICT['file_upload'], file_type_id=FILE_TYPE_DICT['appropriations'], filename=None, start_date='01/01/2017', end_date='03/31/2017', submission_id=None) sess.add_all([sf1, sf2, sf3, tas1, tas2, job]) sess.commit() file_gen_manager = FileGenerationManager(sess, CONFIG_BROKER['local'], job=job) # providing agency code here as it will be passed via SQS and detached file jobs don't store agency code file_gen_manager.generate_file(agency_cgac) assert job.filename is not None # check headers file_rows = read_file_rows(job.filename) assert file_rows[0] == [key for key in file_generation_manager.fileA.mapping] # check body sf1 = sess.query(SF133).filter_by(tas=tas1_str).first() sf2 = sess.query(SF133).filter_by(tas=tas2_str).first() expected1 = [] expected2 = [] sum_cols = [ 'total_budgetary_resources_cpe', 'budget_authority_appropria_cpe', 'budget_authority_unobligat_fyb', 'adjustments_to_unobligated_cpe', 'other_budgetary_resources_cpe', 'contract_authority_amount_cpe', 'borrowing_authority_amount_cpe', 'spending_authority_from_of_cpe', 'status_of_budgetary_resour_cpe', 'obligations_incurred_total_cpe', 'gross_outlay_amount_by_tas_cpe', 'unobligated_balance_cpe', 'deobligations_recoveries_r_cpe' ] zero_sum_cols = {sum_col: '0' for sum_col in sum_cols} expected1_sum_cols = zero_sum_cols.copy() expected1_sum_cols['budget_authority_appropria_cpe'] = '3.00' expected2_sum_cols = zero_sum_cols.copy() expected2_sum_cols['budget_authority_unobligat_fyb'] = '4.00' for value in file_generation_manager.fileA.db_columns: # loop through all values and format date columns if value in sf1.__dict__: expected1.append(str(sf1.__dict__[value] or '')) expected2.append(str(sf2.__dict__[value] or '')) elif value in expected1_sum_cols: expected1.append(expected1_sum_cols[value]) expected2.append(expected2_sum_cols[value]) assert expected1 in file_rows assert expected2 in file_rows