def test_generate_new_d1_file_success(monkeypatch, mock_broker_config_paths, database): """ Testing that a new D1 file is generated """ sess = database.session job = JobFactory( job_status_id=JOB_STATUS_DICT['waiting'], job_type_id=JOB_TYPE_DICT['file_upload'], file_type_id=FILE_TYPE_DICT['award_procurement'], filename=str( mock_broker_config_paths['d_file_storage_path'].join('original')), start_date='01/01/2017', end_date='01/31/2017', original_filename='original', from_cached=True) sess.add(job) sess.commit() file_gen_manager = FileGenerationManager(job, '123', 'awarding', CONFIG_BROKER['local']) file_gen_manager.generate_from_job() sess.refresh(job) file_request = sess.query(FileRequest).filter( FileRequest.job_id == job.job_id).one_or_none() assert file_request is not None assert file_request.is_cached_file is True assert file_request.start_date == job.start_date assert file_request.end_date == job.end_date assert file_request.agency_code == '123' assert file_request.request_date == datetime.now().date() assert job.original_filename != 'original' assert job.from_cached is False assert job.job_status_id == JOB_STATUS_DICT['finished']
def test_generate_new_d1_file_different_dates_success(monkeypatch, mock_broker_config_paths, database): """ Testing that a new D1 file is generated using the same data except for the dates """ sess = database.session job = JobFactory( job_status_id=JOB_STATUS_DICT['waiting'], job_type_id=JOB_TYPE_DICT['file_upload'], file_type_id=FILE_TYPE_DICT['award_procurement'], filename=str( mock_broker_config_paths['d_file_storage_path'].join('original')), start_date='01/01/2017', end_date='01/31/2017', original_filename='original', from_cached=True) sess.add(job) sess.commit() fr = FileRequestFactory(job=job, is_cached_file=True, agency_code='123', agency_type='awarding', start_date=job.start_date, end_date=job.end_date, file_type='D1', request_date=datetime.now().date()) sess.add(fr) sess.commit() # Change the job start date old_start_date = job.start_date job.start_date = '01/02/2017' sess.commit() file_gen_manager = FileGenerationManager(job, '123', 'awarding', CONFIG_BROKER['local']) file_gen_manager.generate_from_job() sess.refresh(job) new_file_request = sess.query(FileRequest).filter( FileRequest.job_id == job.job_id, FileRequest.is_cached_file.is_(True)).one_or_none() assert new_file_request is not None assert new_file_request.is_cached_file is True assert new_file_request.start_date == job.start_date assert new_file_request.end_date == job.end_date assert new_file_request.agency_code == '123' assert new_file_request.agency_type == 'awarding' old_file_request = sess.query(FileRequest).filter( FileRequest.job_id == job.job_id, FileRequest.is_cached_file.is_(False)).one_or_none() assert old_file_request is not None assert old_file_request.is_cached_file is False assert old_file_request.start_date == old_start_date assert old_file_request.end_date == job.end_date assert old_file_request.agency_code == '123' assert old_file_request.agency_type == 'awarding'
def test_generate_new_d1_file_keep_old_job_files_success( monkeypatch, mock_broker_config_paths, database): """ Testing that when a new file is generated by a child job, the parent job's files stay the same """ sess = database.session original_job = JobFactory( job_status_id=JOB_STATUS_DICT['waiting'], job_type_id=JOB_TYPE_DICT['file_upload'], file_type_id=FILE_TYPE_DICT['award_procurement'], filename=str( mock_broker_config_paths['d_file_storage_path'].join('original')), start_date='01/01/2017', end_date='01/31/2017', original_filename='original', from_cached=False) new_job = JobFactory( job_status_id=JOB_STATUS_DICT['finished'], job_type_id=JOB_TYPE_DICT['file_upload'], file_type_id=FILE_TYPE_DICT['award_procurement'], filename=str( mock_broker_config_paths['d_file_storage_path'].join('original')), start_date='01/01/2017', end_date='01/31/2017', original_filename='original', from_cached=False) sess.add_all([original_job, new_job]) sess.commit() fr = FileRequestFactory(job=original_job, parent_job_id=None, is_cached_file=False, agency_code='123', agency_type='awarding', start_date=original_job.start_date, end_date=original_job.end_date, file_type='D1', request_date=datetime.now().date()) fr_2 = FileRequestFactory(job=new_job, parent_job_id=original_job.job_id, is_cached_file=False, agency_code='123', agency_type='awarding', start_date=new_job.start_date, end_date=new_job.end_date, file_type='D1', request_date=datetime.now().date()) sess.add_all([fr, fr_2]) sess.commit() file_gen_manager = FileGenerationManager(original_job, '123', 'awarding', CONFIG_BROKER['local']) file_gen_manager.generate_from_job() sess.refresh(original_job) sess.refresh(new_job) assert original_job.original_filename != 'original' assert new_job.original_filename == 'original'
def test_generate_e_file_csv(monkeypatch, mock_broker_config_paths, database): """ Verify that an appropriate CSV is written, based on fileE.Row's structure """ # Create an award so that we have _a_ duns sub = SubmissionFactory() database.session.add(sub) database.session.commit() ap = AwardProcurementFactory(submission_id=sub.submission_id) database.session.add(ap) database.session.commit() file_path = str(mock_broker_config_paths['broker_files'].join('e_test1')) job = JobFactory(job_status_id=JOB_STATUS_DICT['running'], job_type_id=JOB_TYPE_DICT['file_upload'], file_type_id=FILE_TYPE_DICT['executive_compensation'], filename=file_path, original_filename='e_test1', submission_id=sub.submission_id) database.session.add(job) database.session.commit() monkeypatch.setattr(file_generation_manager.fileE, 'row_to_dict', Mock()) file_generation_manager.fileE.row_to_dict.return_value = {} monkeypatch.setattr(file_generation_manager.fileE, 'retrieve_rows', Mock()) file_generation_manager.fileE.retrieve_rows.return_value = [ fileE.Row('a', 'b', 'c', 'd', '1a', '1b', '2a', '2b', '3a', '3b', '4a', '4b', '5a', '5b'), fileE.Row('A', 'B', 'C', 'D', '1A', '1B', '2A', '2B', '3A', '3B', '4A', '4B', '5A', '5B') ] monkeypatch.setattr(file_generation_manager, 'mark_job_status', Mock()) file_gen_manager = FileGenerationManager(job, None, None, CONFIG_BROKER['local']) file_gen_manager.generate_e_file() expected = [[ 'AwardeeOrRecipientUniqueIdentifier', 'AwardeeOrRecipientLegalEntityName', 'UltimateParentUniqueIdentifier', 'UltimateParentLegalEntityName', 'HighCompOfficer1FullName', 'HighCompOfficer1Amount', 'HighCompOfficer2FullName', 'HighCompOfficer2Amount', 'HighCompOfficer3FullName', 'HighCompOfficer3Amount', 'HighCompOfficer4FullName', 'HighCompOfficer4Amount', 'HighCompOfficer5FullName', 'HighCompOfficer5Amount' ], [ 'a', 'b', 'c', 'd', '1a', '1b', '2a', '2b', '3a', '3b', '4a', '4b', '5a', '5b' ], [ 'A', 'B', 'C', 'D', '1A', '1B', '2A', '2B', '3A', '3B', '4A', '4B', '5A', '5B' ]] assert read_file_rows(file_path) == expected
def test_uncache_new_d1_file_fpds_success(monkeypatch, mock_broker_config_paths, database): """Testing that a new file is not generated if another job has already has a successfully generated file""" sess = database.session original_job = JobFactory( job_status_id=JOB_STATUS_DICT['finished'], job_type_id=JOB_TYPE_DICT['file_upload'], file_type_id=FILE_TYPE_DICT['award_procurement'], filename=str( mock_broker_config_paths['d_file_storage_path'].join('original')), start_date='01/01/2017', end_date='01/31/2017', original_filename='original', from_cached=True) sess.add(original_job) sess.commit() file_request = FileRequestFactory(job=original_job, is_cached_file=True, agency_code='123', agency_type='awarding', start_date='01/01/2017', end_date='01/31/2017', file_type='D1', request_date=(datetime.now().date() - timedelta(1))) new_job = JobFactory(job_status_id=JOB_STATUS_DICT['waiting'], job_type_id=JOB_TYPE_DICT['file_upload'], file_type_id=FILE_TYPE_DICT['award_procurement'], start_date='01/01/2017', end_date='01/31/2017') sess.add_all([file_request, new_job]) sess.commit() file_gen_manager = FileGenerationManager(new_job, '123', 'awarding', CONFIG_BROKER['local']) file_gen_manager.generate_from_job() sess.refresh(new_job) file_request = sess.query(FileRequest).filter( FileRequest.job_id == new_job.job_id).one_or_none() assert file_request is not None assert file_request.is_cached_file is True assert file_request.start_date == new_job.start_date assert file_request.end_date == new_job.end_date assert file_request.agency_code == '123' assert file_request.request_date == datetime.now().date() assert new_job.original_filename != 'original' assert new_job.from_cached is False assert new_job.job_status_id == JOB_STATUS_DICT['finished']
def test_generate_e_file_query(monkeypatch, mock_broker_config_paths, database): """ Verify that generate_e_file makes an appropriate query (matching both D1 and D2 entries) """ # Generate several file D1 entries, largely with the same submission_id, and with two overlapping DUNS. Generate # several D2 entries with the same submission_id as well sess = database.session sub = SubmissionFactory() sub_2 = SubmissionFactory() sess.add_all([sub, sub_2]) sess.commit() file_path = str(mock_broker_config_paths['broker_files'].join('e_test1')) job = JobFactory(job_status_id=JOB_STATUS_DICT['running'], job_type_id=JOB_TYPE_DICT['file_upload'], file_type_id=FILE_TYPE_DICT['executive_compensation'], filename=file_path, original_filename='e_test1', submission_id=sub.submission_id) database.session.add(job) database.session.commit() model = AwardProcurementFactory(submission_id=sub.submission_id) aps = [ AwardProcurementFactory(submission_id=sub.submission_id) for _ in range(4) ] afas = [ AwardFinancialAssistanceFactory(submission_id=sub.submission_id) for _ in range(5) ] same_duns = AwardProcurementFactory( submission_id=sub.submission_id, awardee_or_recipient_uniqu=model.awardee_or_recipient_uniqu) unrelated = AwardProcurementFactory(submission_id=sub_2.submission_id) sess.add_all(aps + afas + [model, same_duns, unrelated]) sess.commit() monkeypatch.setattr(file_generation_manager, 'mark_job_status', Mock()) monkeypatch.setattr(file_generation_manager.fileE, 'retrieve_rows', Mock(return_value=[])) file_gen_manager = FileGenerationManager(job, None, None, CONFIG_BROKER['local']) file_gen_manager.generate_e_file() # [0][0] gives us the first, non-keyword args call_args = file_generation_manager.fileE.retrieve_rows.call_args[0][0] expected = [ap.awardee_or_recipient_uniqu for ap in aps] expected.append(model.awardee_or_recipient_uniqu) expected.extend(afa.awardee_or_recipient_uniqu for afa in afas) assert list(sorted(call_args)) == list(sorted(expected))
def test_uncache_new_d1_file_fpds_success(monkeypatch, mock_broker_config_paths, database, job_constants): """Testing that a new file is not generated if another job has already has a successfully generated file""" sess = database.session original_job = JobFactory( job_status=sess.query(JobStatus).filter_by(name='finished').one(), job_type=sess.query(JobType).filter_by(name='file_upload').one(), file_type=sess.query(FileType).filter_by( name='award_procurement').one(), filename=str( mock_broker_config_paths['d_file_storage_path'].join('original')), start_date='01/01/2017', end_date='01/31/2017', original_filename='original', from_cached=True, ) sess.add(original_job) sess.commit() file_request = FileRequestFactory( job=original_job, is_cached_file=True, agency_code='123', start_date='01/01/2017', end_date='01/31/2017', file_type='D1', request_date=(datetime.now().date() - timedelta(1)), ) new_job = JobFactory( job_status=sess.query(JobStatus).filter_by(name='waiting').one(), job_type=sess.query(JobType).filter_by(name='file_upload').one(), file_type=sess.query(FileType).filter_by( name='award_procurement').one(), start_date='01/01/2017', end_date='01/31/2017', ) sess.add_all([file_request, new_job]) sess.commit() monkeypatch.setattr(file_generation_handler, 'retrieve_job_context_data', Mock(return_value=(sess, new_job))) FileGenerationManager().generate_from_job(new_job.job_id, '123') sess.refresh(new_job) file_request = sess.query(FileRequest).filter( FileRequest.job_id == new_job.job_id).one_or_none() assert file_request is not None assert file_request.is_cached_file is True assert file_request.start_date == new_job.start_date assert file_request.end_date == new_job.end_date assert file_request.agency_code == '123' assert file_request.request_date == datetime.now().date() assert new_job.original_filename != 'original' assert new_job.from_cached is False assert new_job.job_status_id == sess.query(JobStatus).filter_by( name='finished').one().job_status_id
def test_generate_file_updates_jobs(monkeypatch, mock_broker_config_paths, database): sess = database.session job1 = JobFactory(job_status_id=JOB_STATUS_DICT['running'], job_type_id=JOB_TYPE_DICT['file_upload'], file_type_id=FILE_TYPE_DICT['award_procurement'], filename=None, original_filename=None, start_date='01/01/2017', end_date='01/31/2017') job2 = JobFactory(job_status_id=JOB_STATUS_DICT['running'], job_type_id=JOB_TYPE_DICT['file_upload'], file_type_id=FILE_TYPE_DICT['award_procurement'], filename=None, original_filename=None, start_date='01/01/2017', end_date='01/31/2017') job3 = JobFactory(job_status_id=JOB_STATUS_DICT['running'], job_type_id=JOB_TYPE_DICT['file_upload'], file_type_id=FILE_TYPE_DICT['award_procurement'], filename=None, original_filename=None, start_date='01/01/2017', end_date='01/31/2017') file_gen = FileGenerationFactory(request_date=datetime.now().date(), start_date='01/01/2017', end_date='01/31/2017', file_type='D1', agency_code='123', agency_type='awarding', is_cached_file=True, file_path=None) sess.add_all([job1, job2, job3, file_gen]) sess.commit() job1.file_generation_id = file_gen.file_generation_id job2.file_generation_id = file_gen.file_generation_id job3.file_generation_id = file_gen.file_generation_id sess.commit() monkeypatch.setattr(generation_helper, 'g', Mock(return_value={'is_local': CONFIG_BROKER['local']})) file_gen_manager = FileGenerationManager(sess, CONFIG_BROKER['local'], file_generation=file_gen) file_gen_manager.generate_file() sess.refresh(file_gen) original_filename = file_gen.file_path.split('/')[-1] assert job1.job_status_id == JOB_STATUS_DICT['finished'] assert job1.original_filename == original_filename assert job1.filename == '{}{}'.format( CONFIG_BROKER['broker_files'] if CONFIG_BROKER['local'] else job1.submission_id + '/', original_filename) assert job2.job_status_id == JOB_STATUS_DICT['finished'] assert job2.original_filename == original_filename assert job2.filename == '{}{}'.format( CONFIG_BROKER['broker_files'] if CONFIG_BROKER['local'] else job2.submission_id + '/', original_filename) assert job3.job_status_id == JOB_STATUS_DICT['finished'] assert job3.original_filename == original_filename assert job3.filename == '{}{}'.format( CONFIG_BROKER['broker_files'] if CONFIG_BROKER['local'] else job3.submission_id + '/', original_filename)
def test_generate_e_file_csv(monkeypatch, mock_broker_config_paths, database): """ Verify that an appropriate CSV is written, based on fileE.Row's structure """ # Create an award so that we have _a_ duns sub = SubmissionFactory() database.session.add(sub) database.session.commit() ap = AwardProcurementFactory(submission_id=sub.submission_id) database.session.add(ap) database.session.commit() file_path = str(mock_broker_config_paths['broker_files'].join('e_test1')) job = JobFactory(job_status_id=JOB_STATUS_DICT['running'], job_type_id=JOB_TYPE_DICT['file_upload'], file_type_id=FILE_TYPE_DICT['executive_compensation'], filename=file_path, original_filename='e_test1', submission_id=sub.submission_id) database.session.add(job) database.session.commit() monkeypatch.setattr(file_generation_manager.fileE, 'row_to_dict', Mock()) file_generation_manager.fileE.row_to_dict.return_value = {} monkeypatch.setattr(file_generation_manager.fileE, 'retrieve_rows', Mock()) file_generation_manager.fileE.retrieve_rows.return_value = [ fileE.Row('a', 'b', 'c', 'd', '1a', '1b', '2a', '2b', '3a', '3b', '4a', '4b', '5a', '5b'), fileE.Row('A', 'B', 'C', 'D', '1A', '1B', '2A', '2B', '3A', '3B', '4A', '4B', '5A', '5B') ] file_gen_manager = FileGenerationManager(database.session, CONFIG_BROKER['local'], job=job) file_gen_manager.generate_file() expected = [ ['AwardeeOrRecipientUniqueIdentifier', 'AwardeeOrRecipientLegalEntityName', 'UltimateParentUniqueIdentifier', 'UltimateParentLegalEntityName', 'HighCompOfficer1FullName', 'HighCompOfficer1Amount', 'HighCompOfficer2FullName', 'HighCompOfficer2Amount', 'HighCompOfficer3FullName', 'HighCompOfficer3Amount', 'HighCompOfficer4FullName', 'HighCompOfficer4Amount', 'HighCompOfficer5FullName', 'HighCompOfficer5Amount'], ['a', 'b', 'c', 'd', '1a', '1b', '2a', '2b', '3a', '3b', '4a', '4b', '5a', '5b'], ['A', 'B', 'C', 'D', '1A', '1B', '2A', '2B', '3A', '3B', '4A', '4B', '5A', '5B'] ] assert read_file_rows(file_path) == expected
def test_generate_funding_d1(mock_broker_config_paths, database): sess = database.session dap_model = DetachedAwardProcurementFactory dap_1 = dap_model(funding_agency_code='123', action_date='20170101', detached_award_proc_unique='unique1') dap_2 = dap_model(funding_agency_code='123', action_date='20170131', detached_award_proc_unique='unique2') dap_3 = dap_model(funding_agency_code='123', action_date='20170201', detached_award_proc_unique='unique3') dap_4 = dap_model(funding_agency_code='123', action_date='20161231', detached_award_proc_unique='unique4') dap_5 = dap_model(funding_agency_code='234', action_date='20170115', detached_award_proc_unique='unique5') file_gen = FileGenerationFactory(request_date=datetime.now().date(), start_date='01/01/2017', end_date='01/31/2017', file_type='D1', agency_code='123', agency_type='funding', is_cached_file=True, file_path=None) sess.add_all([dap_1, dap_2, dap_3, dap_4, dap_5, file_gen]) sess.commit() file_gen_manager = FileGenerationManager(sess, CONFIG_BROKER['local'], file_generation=file_gen) file_gen_manager.generate_file() assert file_gen.file_path is not None # check headers file_rows = read_file_rows(file_gen.file_path) assert file_rows[0] == [key for key in file_generation_manager.fileD1.mapping] # check body dap_one = sess.query(DetachedAwardProcurement).filter_by(detached_award_proc_unique='unique1').first() dap_two = sess.query(DetachedAwardProcurement).filter_by(detached_award_proc_unique='unique2').first() expected1, expected2 = [], [] for value in file_generation_manager.fileD1.db_columns: # loop through all values and format date columns if value in ['period_of_performance_star', 'period_of_performance_curr', 'period_of_perf_potential_e', 'ordering_period_end_date', 'action_date', 'last_modified', 'solicitation_date']: expected1.append(re.sub(r"[-]", r"", str(dap_one.__dict__[value]))[0:8]) expected2.append(re.sub(r"[-]", r"", str(dap_two.__dict__[value]))[0:8]) else: expected1.append(str(dap_one.__dict__[value])) expected2.append(str(dap_two.__dict__[value])) assert expected1 in file_rows assert expected2 in file_rows
def test_generate_funding_d2(mock_broker_config_paths, database): sess = database.session pafa = PublishedAwardFinancialAssistanceFactory pafa_1 = pafa(funding_agency_code='123', action_date='20170101', afa_generated_unique='unique1', is_active=True) pafa_2 = pafa(funding_agency_code='123', action_date='20170131', afa_generated_unique='unique2', is_active=True) pafa_3 = pafa(funding_agency_code='123', action_date='20161231', afa_generated_unique='unique3', is_active=True) pafa_4 = pafa(funding_agency_code='123', action_date='20170201', afa_generated_unique='unique4', is_active=True) pafa_5 = pafa(funding_agency_code='123', action_date='20170115', afa_generated_unique='unique5', is_active=False) pafa_6 = pafa(funding_agency_code='234', action_date='20170115', afa_generated_unique='unique6', is_active=True) file_gen = FileGenerationFactory(request_date=datetime.now().date(), start_date='01/01/2017', end_date='01/31/2017', file_type='D2', agency_code='123', agency_type='funding', is_cached_file=True, file_path=None) sess.add_all([pafa_1, pafa_2, pafa_3, pafa_4, pafa_5, pafa_6, file_gen]) sess.commit() file_gen_manager = FileGenerationManager(sess, CONFIG_BROKER['local'], file_generation=file_gen) file_gen_manager.generate_file() assert file_gen.file_path is not None # check headers file_rows = read_file_rows(file_gen.file_path) assert file_rows[0] == [key for key in file_generation_manager.fileD2.mapping] # check body pafa1 = sess.query(PublishedAwardFinancialAssistance).filter_by(afa_generated_unique='unique1').first() pafa2 = sess.query(PublishedAwardFinancialAssistance).filter_by(afa_generated_unique='unique2').first() expected1, expected2 = [], [] for value in file_generation_manager.fileD2.db_columns: # loop through all values and format date columns if value in ['period_of_performance_star', 'period_of_performance_curr', 'modified_at', 'action_date']: expected1.append(re.sub(r"[-]", r"", str(pafa1.__dict__[value]))[0:8]) expected2.append(re.sub(r"[-]", r"", str(pafa2.__dict__[value]))[0:8]) else: expected1.append(str(pafa1.__dict__[value])) expected2.append(str(pafa2.__dict__[value])) assert expected1 in file_rows assert expected2 in file_rows
def test_generate_e_file_query(monkeypatch, mock_broker_config_paths, database): """ Verify that generate_e_file makes an appropriate query (matching both D1 and D2 entries) """ # Generate several file D1 entries, largely with the same submission_id, and with two overlapping DUNS. Generate # several D2 entries with the same submission_id as well sess = database.session sub = SubmissionFactory() sub_2 = SubmissionFactory() sess.add_all([sub, sub_2]) sess.commit() file_path = str(mock_broker_config_paths['broker_files'].join('e_test1')) job = JobFactory(job_status_id=JOB_STATUS_DICT['running'], job_type_id=JOB_TYPE_DICT['file_upload'], file_type_id=FILE_TYPE_DICT['executive_compensation'], filename=file_path, original_filename='e_test1', submission_id=sub.submission_id) database.session.add(job) database.session.commit() model = AwardProcurementFactory(submission_id=sub.submission_id) aps = [AwardProcurementFactory(submission_id=sub.submission_id) for _ in range(4)] afas = [AwardFinancialAssistanceFactory(submission_id=sub.submission_id) for _ in range(5)] same_duns = AwardProcurementFactory( submission_id=sub.submission_id, awardee_or_recipient_uniqu=model.awardee_or_recipient_uniqu) unrelated = AwardProcurementFactory(submission_id=sub_2.submission_id) sess.add_all(aps + afas + [model, same_duns, unrelated]) sess.commit() monkeypatch.setattr(file_generation_manager.fileE, 'retrieve_rows', Mock(return_value=[])) file_gen_manager = FileGenerationManager(database.session, CONFIG_BROKER['local'], job=job) file_gen_manager.generate_file() # [0][0] gives us the first, non-keyword args call_args = file_generation_manager.fileE.retrieve_rows.call_args[0][0] expected = [ap.awardee_or_recipient_uniqu for ap in aps] expected.append(model.awardee_or_recipient_uniqu) expected.extend(afa.awardee_or_recipient_uniqu for afa in afas) assert list(sorted(call_args)) == list(sorted(expected))
def test_generate_f_file(monkeypatch, mock_broker_config_paths, database): """ A CSV with fields in the right order should be written to the file system """ file_path1 = str(mock_broker_config_paths['broker_files'].join('f_test1')) job1 = JobFactory(job_status_id=JOB_STATUS_DICT['running'], job_type_id=JOB_TYPE_DICT['file_upload'], file_type_id=FILE_TYPE_DICT['sub_award'], filename=file_path1, original_filename='f_test1') file_path2 = str(mock_broker_config_paths['broker_files'].join('f_test2')) job2 = JobFactory(job_status_id=JOB_STATUS_DICT['running'], job_type_id=JOB_TYPE_DICT['file_upload'], file_type_id=FILE_TYPE_DICT['sub_award'], filename=file_path2, original_filename='f_test2') database.session.add(job1, job2) database.session.commit() file_f_mock = Mock() monkeypatch.setattr(file_generation_manager, 'fileF', file_f_mock) file_f_mock.generate_f_rows.return_value = [ dict(key4='a', key11='b'), dict(key4='c', key11='d') ] file_f_mock.mappings = OrderedDict([('key4', 'mapping4'), ('key11', 'mapping11')]) expected = [['key4', 'key11'], ['a', 'b'], ['c', 'd']] monkeypatch.setattr(file_generation_manager, 'mark_job_status', Mock()) file_gen_manager = FileGenerationManager(job1, None, None, CONFIG_BROKER['local']) file_gen_manager.generate_f_file() assert read_file_rows(file_path1) == expected # re-order file_f_mock.mappings = OrderedDict([('key11', 'mapping11'), ('key4', 'mapping4')]) expected = [['key11', 'key4'], ['b', 'a'], ['d', 'c']] monkeypatch.setattr(file_generation_manager, 'mark_job_status', Mock()) file_gen_manager = FileGenerationManager(job2, None, None, CONFIG_BROKER['local']) file_gen_manager.generate_f_file() assert read_file_rows(file_path2) == expected
def test_generate_new_d2_file_success(monkeypatch, mock_broker_config_paths, database, job_constants): """ Testing that a new D2 file is generated """ sess = database.session job = JobFactory( job_status=sess.query(JobStatus).filter_by(name='waiting').one(), job_type=sess.query(JobType).filter_by(name='file_upload').one(), file_type=sess.query(FileType).filter_by(name='award').one(), filename=str( mock_broker_config_paths['d_file_storage_path'].join('original')), start_date='01/01/2017', end_date='01/31/2017', original_filename='original', from_cached=True, ) sess.add(job) sess.commit() monkeypatch.setattr(file_generation_handler, 'retrieve_job_context_data', Mock(return_value=(sess, job))) FileGenerationManager().generate_from_job(job.job_id, '123') sess.refresh(job) file_request = sess.query(FileRequest).filter( FileRequest.job_id == job.job_id).one_or_none() assert file_request is not None assert file_request.is_cached_file is True assert file_request.start_date == job.start_date assert file_request.end_date == job.end_date assert file_request.agency_code == '123' assert file_request.request_date == datetime.now().date() assert job.original_filename != 'original' assert job.from_cached is False assert job.job_status_id == sess.query(JobStatus).filter_by( name='finished').one().job_status_id
def validator_process_job(job_id, agency_code, is_retry=False): """ Retrieves a Job based on its ID, and kicks off a validation. Handles errors by ensuring the Job (if exists) is no longer running. Args: job_id: ID of a Job agency_code: CGAC or FREC code for agency, only required for file generations by Job is_retry: If this is not the very first time handling execution of this job. If True, cleanup is performed before proceeding to retry the job Raises: Any Exceptions raised by the GenerationManager or ValidationManager, excluding those explicitly handled """ if is_retry: if cleanup_validation(job_id): log_job_message( logger=logger, message= "Attempting a retry of {} after successful retry-cleanup.". format(inspect.stack()[0][3]), job_id=job_id, is_debug=True) else: log_job_message( logger=logger, message="Retry of {} found to be not necessary after cleanup. " "Returning from job with success.".format( inspect.stack()[0][3]), job_id=job_id, is_debug=True) return sess = GlobalDB.db().session job = None try: # Get the job job = sess.query(Job).filter_by(job_id=job_id).one_or_none() if job is None: validation_error_type = ValidationError.jobError write_file_error(job_id, None, validation_error_type) raise ResponseException( 'Job ID {} not found in database'.format(job_id), StatusCode.CLIENT_ERROR, None, validation_error_type) mark_job_status(job_id, 'ready') # We can either validate or generate a file based on Job ID if job.job_type.name == 'file_upload': # Generate A, E, or F file file_generation_manager = FileGenerationManager(sess, g.is_local, job=job) file_generation_manager.generate_file(agency_code) else: # Run validations validation_manager = ValidationManager( g.is_local, CONFIG_SERVICES['error_report_path']) validation_manager.validate_job(job.job_id) except (ResponseException, csv.Error, UnicodeDecodeError, ValueError) as e: # Handle exceptions explicitly raised during validation error_data = { 'message': 'An exception occurred in the Validator', 'message_type': 'ValidatorInfo', 'job_id': job_id, 'traceback': traceback.format_exc() } if job: error_data.update({ 'submission_id': job.submission_id, 'file_type': job.file_type.name }) logger.error(error_data) sess.refresh(job) job.error_message = str(e) if job.filename is not None: error_type = ValidationError.unknownError if isinstance(e, UnicodeDecodeError): error_type = ValidationError.encodingError elif isinstance(e, ResponseException): error_type = e.errorType write_file_error(job.job_id, job.filename, error_type) mark_job_status(job.job_id, 'invalid') else: logger.error(error_data) raise e except Exception as e: # Log uncaught exceptions and fail the Job error_data = { 'message': 'An unhandled exception occurred in the Validator', 'message_type': 'ValidatorInfo', 'job_id': job_id, 'traceback': traceback.format_exc() } if job: error_data.update({ 'submission_id': job.submission_id, 'file_type': job.file_type.name }) logger.error(error_data) # Try to mark the Job as failed, but continue raising the original Exception if not possible try: mark_job_status(job_id, 'failed') sess.refresh(job) job.error_message = str(e) sess.commit() except: pass raise e
def validator_process_file_generation(file_gen_id, is_retry=False): """ Retrieves a FileGeneration object based on its ID, and kicks off a file generation. Handles errors by ensuring the FileGeneration (if exists) is no longer cached. Args: file_gen_id: ID of a FileGeneration object is_retry: If this is not the very first time handling execution of this job. If True, cleanup is performed before proceeding to retry the job Raises: Any Exceptions raised by the FileGenerationManager """ if is_retry: if cleanup_generation(file_gen_id): log_job_message( logger=logger, message= "Attempting a retry of {} after successful retry-cleanup.". format(inspect.stack()[0][3]), job_id=file_gen_id, is_debug=True) else: log_job_message( logger=logger, message="Retry of {} found to be not necessary after cleanup. " "Returning from job with success.".format( inspect.stack()[0][3]), job_id=file_gen_id, is_debug=True) return sess = GlobalDB.db().session file_generation = None try: file_generation = sess.query(FileGeneration).filter_by( file_generation_id=file_gen_id).one_or_none() if file_generation is None: raise ResponseException( 'FileGeneration ID {} not found in database'.format( file_gen_id), StatusCode.CLIENT_ERROR, None) file_generation_manager = FileGenerationManager( sess, g.is_local, file_generation=file_generation) file_generation_manager.generate_file() except Exception as e: # Log uncaught exceptions and fail all Jobs referencing this FileGeneration error_data = { 'message': 'An unhandled exception occurred in the Validator during file generation', 'message_type': 'ValidatorInfo', 'file_generation_id': file_gen_id, 'traceback': traceback.format_exc() } if file_generation: error_data.update({ 'agency_code': file_generation.agency_code, 'agency_type': file_generation.agency_type, 'start_date': file_generation.start_date, 'end_date': file_generation.end_date, 'file_type': file_generation.file_type, 'file_path': file_generation.file_path, }) logger.error(error_data) # Try to mark the Jobs as failed, but continue raising the original Exception if not possible try: if file_generation: # Uncache the FileGeneration sess.refresh(file_generation) file_generation.is_cached_file = False # Mark all Jobs waiting on this FileGeneration as failed generation_jobs = sess.query(Job).filter_by( file_generation_id=file_gen_id).all() for job in generation_jobs: if job.job_status in [ JOB_STATUS_DICT['waiting'], JOB_STATUS_DICT['ready'], JOB_STATUS_DICT['running'] ]: mark_job_status(job.job_id, 'failed') sess.refresh(job) job.file_generation_id = None job.error_message = str(e) sess.commit() except: pass # ResponseExceptions only occur at very specific times, and should not affect the Validator's future attempts # at handling messages from SQS if not isinstance(e, ResponseException): raise e
def test_generate_d2_file_query(mock_broker_config_paths, database): """ A CSV with fields in the right order should be written to the file system """ sess = database.session pafa = PublishedAwardFinancialAssistanceFactory pafa_1 = pafa(awarding_agency_code='123', action_date='20170101', afa_generated_unique='unique1', is_active=True) pafa_2 = pafa(awarding_agency_code='123', action_date='20170131', afa_generated_unique='unique2', is_active=True) pafa_3 = pafa(awarding_agency_code='123', action_date='20161231', afa_generated_unique='unique3', is_active=True) pafa_4 = pafa(awarding_agency_code='123', action_date='20170201', afa_generated_unique='unique4', is_active=True) pafa_5 = pafa(awarding_agency_code='123', action_date='20170115', afa_generated_unique='unique5', is_active=False) pafa_6 = pafa(awarding_agency_code='234', action_date='20170115', afa_generated_unique='unique6', is_active=True) sess.add_all([pafa_1, pafa_2, pafa_3, pafa_4, pafa_5, pafa_6]) file_path = str( mock_broker_config_paths['d_file_storage_path'].join('d2_test')) job = JobFactory(job_status_id=JOB_STATUS_DICT['running'], job_type_id=JOB_TYPE_DICT['file_upload'], file_type_id=FILE_TYPE_DICT['award'], filename=file_path, original_filename='d2_test', start_date='01/01/2017', end_date='01/31/2017') sess.add(job) sess.commit() file_gen_manager = FileGenerationManager(job, '123', 'awarding', CONFIG_BROKER['local']) file_gen_manager.generate_d_file() # check headers file_rows = read_file_rows(file_path) assert file_rows[0] == [ key for key in file_generation_manager.fileD2.mapping ] # check body pafa1 = sess.query(PublishedAwardFinancialAssistance).filter_by( afa_generated_unique='unique1').first() pafa2 = sess.query(PublishedAwardFinancialAssistance).filter_by( afa_generated_unique='unique2').first() expected1, expected2 = [], [] for value in file_generation_manager.fileD2.db_columns: # loop through all values and format date columns if value in [ 'period_of_performance_star', 'period_of_performance_curr', 'modified_at', 'action_date' ]: expected1.append( re.sub(r"[-]", r"", str(pafa1.__dict__[value]))[0:8]) expected2.append( re.sub(r"[-]", r"", str(pafa2.__dict__[value]))[0:8]) else: expected1.append(str(pafa1.__dict__[value])) expected2.append(str(pafa2.__dict__[value])) assert expected1 in file_rows assert expected2 in file_rows
def test_generate_d1_file_query(mock_broker_config_paths, database): """ A CSV with fields in the right order should be written to the file system """ sess = database.session dap_model = DetachedAwardProcurementFactory dap_1 = dap_model(awarding_agency_code='123', action_date='20170101', detached_award_proc_unique='unique1') dap_2 = dap_model(awarding_agency_code='123', action_date='20170131', detached_award_proc_unique='unique2') dap_3 = dap_model(awarding_agency_code='123', action_date='20170201', detached_award_proc_unique='unique3') dap_4 = dap_model(awarding_agency_code='123', action_date='20161231', detached_award_proc_unique='unique4') dap_5 = dap_model(awarding_agency_code='234', action_date='20170115', detached_award_proc_unique='unique5') sess.add_all([dap_1, dap_2, dap_3, dap_4, dap_5]) file_path = str( mock_broker_config_paths['d_file_storage_path'].join('d1_test')) job = JobFactory(job_status_id=JOB_STATUS_DICT['running'], job_type_id=JOB_TYPE_DICT['file_upload'], file_type_id=FILE_TYPE_DICT['award_procurement'], filename=file_path, original_filename='d1_test', start_date='01/01/2017', end_date='01/31/2017') sess.add(job) sess.commit() file_gen_manager = FileGenerationManager(job, '123', 'awarding', CONFIG_BROKER['local']) file_gen_manager.generate_d_file() # check headers file_rows = read_file_rows(file_path) assert file_rows[0] == [ key for key in file_generation_manager.fileD1.mapping ] # check body dap_one = sess.query(DetachedAwardProcurement).filter_by( detached_award_proc_unique='unique1').first() dap_two = sess.query(DetachedAwardProcurement).filter_by( detached_award_proc_unique='unique2').first() expected1, expected2 = [], [] for value in file_generation_manager.fileD1.db_columns: # loop through all values and format date columns if value in [ 'period_of_performance_star', 'period_of_performance_curr', 'period_of_perf_potential_e', 'ordering_period_end_date', 'action_date', 'last_modified' ]: expected1.append( re.sub(r"[-]", r"", str(dap_one.__dict__[value]))[0:8]) expected2.append( re.sub(r"[-]", r"", str(dap_two.__dict__[value]))[0:8]) else: expected1.append(str(dap_one.__dict__[value])) expected2.append(str(dap_two.__dict__[value])) assert expected1 in file_rows assert expected2 in file_rows
def test_generate_a(mock_broker_config_paths, database): sess = database.session agency_cgac = '097' year = 2017 tas1_dict = { 'allocation_transfer_agency': agency_cgac, 'agency_identifier': '000', 'beginning_period_of_availa': '2017', 'ending_period_of_availabil': '2017', 'availability_type_code': ' ', 'main_account_code': '0001', 'sub_account_code': '001' } tas1_str = concat_tas_dict(tas1_dict) tas2_dict = { 'allocation_transfer_agency': None, 'agency_identifier': '017', 'beginning_period_of_availa': '2017', 'ending_period_of_availabil': '2017', 'availability_type_code': ' ', 'main_account_code': '0001', 'sub_account_code': '001' } tas2_str = concat_tas_dict(tas2_dict) sf1 = SF133Factory(period=6, fiscal_year=year, tas=tas1_str, line=1160, amount='1.00', **tas1_dict) sf2 = SF133Factory(period=6, fiscal_year=year, tas=tas1_str, line=1180, amount='2.00', **tas1_dict) sf3 = SF133Factory(period=6, fiscal_year=year, tas=tas2_str, line=1000, amount='4.00', **tas2_dict) tas1 = TASFactory(financial_indicator2=' ', **tas1_dict) tas2 = TASFactory(financial_indicator2=' ', **tas2_dict) job = JobFactory(job_status_id=JOB_STATUS_DICT['running'], job_type_id=JOB_TYPE_DICT['file_upload'], file_type_id=FILE_TYPE_DICT['appropriations'], filename=None, start_date='01/01/2017', end_date='03/31/2017', submission_id=None) sess.add_all([sf1, sf2, sf3, tas1, tas2, job]) sess.commit() file_gen_manager = FileGenerationManager(sess, CONFIG_BROKER['local'], job=job) # providing agency code here as it will be passed via SQS and detached file jobs don't store agency code file_gen_manager.generate_file(agency_cgac) assert job.filename is not None # check headers file_rows = read_file_rows(job.filename) assert file_rows[0] == [key for key in file_generation_manager.fileA.mapping] # check body sf1 = sess.query(SF133).filter_by(tas=tas1_str).first() sf2 = sess.query(SF133).filter_by(tas=tas2_str).first() expected1 = [] expected2 = [] sum_cols = [ 'total_budgetary_resources_cpe', 'budget_authority_appropria_cpe', 'budget_authority_unobligat_fyb', 'adjustments_to_unobligated_cpe', 'other_budgetary_resources_cpe', 'contract_authority_amount_cpe', 'borrowing_authority_amount_cpe', 'spending_authority_from_of_cpe', 'status_of_budgetary_resour_cpe', 'obligations_incurred_total_cpe', 'gross_outlay_amount_by_tas_cpe', 'unobligated_balance_cpe', 'deobligations_recoveries_r_cpe' ] zero_sum_cols = {sum_col: '0' for sum_col in sum_cols} expected1_sum_cols = zero_sum_cols.copy() expected1_sum_cols['budget_authority_appropria_cpe'] = '3.00' expected2_sum_cols = zero_sum_cols.copy() expected2_sum_cols['budget_authority_unobligat_fyb'] = '4.00' for value in file_generation_manager.fileA.db_columns: # loop through all values and format date columns if value in sf1.__dict__: expected1.append(str(sf1.__dict__[value] or '')) expected2.append(str(sf2.__dict__[value] or '')) elif value in expected1_sum_cols: expected1.append(expected1_sum_cols[value]) expected2.append(expected2_sum_cols[value]) assert expected1 in file_rows assert expected2 in file_rows
def test_generate_noncached_d1_file_success(monkeypatch, mock_broker_config_paths, database, job_constants): """ Testing that a new D1 file is generated """ sess = database.session job1 = JobFactory( job_status=sess.query(JobStatus).filter_by(name='waiting').one(), job_type=sess.query(JobType).filter_by(name='file_upload').one(), file_type=sess.query(FileType).filter_by( name='award_procurement').one(), filename=str(mock_broker_config_paths['d_file_storage_path'].join( 'diff_agency')), start_date='01/01/2017', end_date='01/31/2017', original_filename='diff_agency', from_cached=False, ) job2 = JobFactory( job_status=sess.query(JobStatus).filter_by(name='waiting').one(), job_type=sess.query(JobType).filter_by(name='file_upload').one(), file_type=sess.query(FileType).filter_by(name='award').one(), filename=str(mock_broker_config_paths['d_file_storage_path'].join( 'diff_start_date')), start_date='01/02/2017', end_date='01/31/2017', original_filename='diff_start_date', from_cached=False, ) job3 = JobFactory( job_status=sess.query(JobStatus).filter_by(name='waiting').one(), job_type=sess.query(JobType).filter_by(name='file_upload').one(), file_type=sess.query(FileType).filter_by( name='award_procurement').one(), filename=str(mock_broker_config_paths['d_file_storage_path'].join( 'diff_end_date')), start_date='01/01/2017', end_date='01/30/2017', original_filename='diff_end_date', from_cached=False, ) sess.add_all([job1, job2, job3]) sess.commit() file_request1 = FileRequestFactory(job=job1, is_cached_file=True, agency_code='124', start_date='01/01/2017', end_date='01/31/2017', file_type='D1', request_date=datetime.now().date()) file_request2 = FileRequestFactory(job=job1, is_cached_file=True, agency_code='123', start_date='01/02/2017', end_date='01/31/2017', file_type='D1', request_date=datetime.now().date()) file_request3 = FileRequestFactory(job=job1, is_cached_file=True, agency_code='123', start_date='01/01/2017', end_date='01/30/2017', file_type='D1', request_date=datetime.now().date()) job = JobFactory( job_status=sess.query(JobStatus).filter_by(name='waiting').one(), job_type=sess.query(JobType).filter_by(name='file_upload').one(), file_type=sess.query(FileType).filter_by( name='award_procurement').one(), start_date='01/01/2017', end_date='01/31/2017', ) sess.add_all([job, file_request1, file_request2, file_request3]) sess.commit() monkeypatch.setattr(file_generation_handler, 'retrieve_job_context_data', Mock(return_value=(sess, job))) FileGenerationManager().generate_from_job(job.job_id, '123') sess.refresh(job) file_request = sess.query(FileRequest).filter( FileRequest.job_id == job.job_id).one_or_none() assert file_request is not None assert file_request.is_cached_file is True assert file_request.start_date == job.start_date assert file_request.end_date == job.end_date assert file_request.agency_code == '123' assert file_request.request_date == datetime.now().date() assert job.original_filename != job1.original_filename assert job.original_filename != job2.original_filename assert job.original_filename != job3.original_filename assert job.from_cached is False assert job.job_status_id == sess.query(JobStatus).filter_by( name='finished').one().job_status_id
def test_generate_noncached_d2_file_success(monkeypatch, mock_broker_config_paths, database): """ Testing that a new D2 file is generated """ sess = database.session job1 = JobFactory(job_status_id=JOB_STATUS_DICT['waiting'], job_type_id=JOB_TYPE_DICT['file_upload'], file_type_id=FILE_TYPE_DICT['award_procurement'], filename=str( mock_broker_config_paths['d_file_storage_path'].join( 'diff_agency')), start_date='01/01/2017', end_date='01/31/2017', original_filename='diff_agency', from_cached=False) job2 = JobFactory(job_status_id=JOB_STATUS_DICT['waiting'], job_type_id=JOB_TYPE_DICT['file_upload'], file_type_id=FILE_TYPE_DICT['award'], filename=str( mock_broker_config_paths['d_file_storage_path'].join( 'diff_start_date')), start_date='01/02/2017', end_date='01/31/2017', original_filename='diff_start_date', from_cached=False) job3 = JobFactory(job_status_id=JOB_STATUS_DICT['waiting'], job_type_id=JOB_TYPE_DICT['file_upload'], file_type_id=FILE_TYPE_DICT['award_procurement'], filename=str( mock_broker_config_paths['d_file_storage_path'].join( 'diff_end_date')), start_date='01/01/2017', end_date='01/30/2017', original_filename='diff_end_date', from_cached=False) sess.add_all([job1, job2, job3]) sess.commit() file_request1 = FileRequestFactory(job=job1, is_cached_file=True, agency_code='124', agency_type='awarding', start_date='01/01/2017', end_date='01/31/2017', file_type='D2', request_date=datetime.now().date()) file_request2 = FileRequestFactory(job=job1, is_cached_file=True, agency_code='123', agency_type='awarding', start_date='01/02/2017', end_date='01/31/2017', file_type='D2', request_date=datetime.now().date()) file_request3 = FileRequestFactory(job=job1, is_cached_file=True, agency_code='123', agency_type='awarding', start_date='01/01/2017', end_date='01/30/2017', file_type='D2', request_date=datetime.now().date()) job = JobFactory(job_status_id=JOB_STATUS_DICT['waiting'], job_type_id=JOB_TYPE_DICT['file_upload'], file_type_id=FILE_TYPE_DICT['award_procurement'], start_date='01/01/2017', end_date='01/31/2017') sess.add_all([job, file_request1, file_request2, file_request3]) sess.commit() file_gen_manager = FileGenerationManager(job, '123', 'awarding', CONFIG_BROKER['local']) file_gen_manager.generate_from_job() sess.refresh(job) file_request = sess.query(FileRequest).filter( FileRequest.job_id == job.job_id).one_or_none() assert file_request is not None assert file_request.is_cached_file is True assert file_request.start_date == job.start_date assert file_request.end_date == job.end_date assert file_request.agency_code == '123' assert file_request.request_date == datetime.now().date() assert job.original_filename != job1.original_filename assert job.original_filename != job2.original_filename assert job.original_filename != job3.original_filename assert job.from_cached is False assert job.job_status_id == JOB_STATUS_DICT['finished']
def test_generate_a(database): sess = database.session agency_cgac = '097' year = 2017 tas1_dict = { 'allocation_transfer_agency': agency_cgac, 'agency_identifier': '000', 'beginning_period_of_availa': '2017', 'ending_period_of_availabil': '2017', 'availability_type_code': ' ', 'main_account_code': '0001', 'sub_account_code': '001' } tas1_str = concat_tas_dict(tas1_dict) tas2_dict = { 'allocation_transfer_agency': None, 'agency_identifier': '017', 'beginning_period_of_availa': '2017', 'ending_period_of_availabil': '2017', 'availability_type_code': ' ', 'main_account_code': '0001', 'sub_account_code': '001' } tas2_str = concat_tas_dict(tas2_dict) sf1 = SF133Factory(period=6, fiscal_year=year, tas=tas1_str, line=1160, amount='1.00', **tas1_dict) sf2 = SF133Factory(period=6, fiscal_year=year, tas=tas1_str, line=1180, amount='2.00', **tas1_dict) sf3 = SF133Factory(period=6, fiscal_year=year, tas=tas2_str, line=1000, amount='4.00', **tas2_dict) tas1 = TASFactory(financial_indicator2=' ', **tas1_dict) tas2 = TASFactory(financial_indicator2=' ', **tas2_dict) job = JobFactory(job_status_id=JOB_STATUS_DICT['running'], job_type_id=JOB_TYPE_DICT['file_upload'], file_type_id=FILE_TYPE_DICT['appropriations'], filename=None, start_date='01/01/2017', end_date='03/31/2017', submission_id=None) sess.add_all([sf1, sf2, sf3, tas1, tas2, job]) sess.commit() file_gen_manager = FileGenerationManager(sess, CONFIG_BROKER['local'], job=job) # providing agency code here as it will be passed via SQS and detached file jobs don't store agency code file_gen_manager.generate_file(agency_cgac) assert job.filename is not None # check headers file_rows = read_file_rows(job.filename) assert file_rows[0] == [ key for key in file_generation_manager.fileA.mapping ] # check body sf1 = sess.query(SF133).filter_by(tas=tas1_str).first() sf2 = sess.query(SF133).filter_by(tas=tas2_str).first() expected1 = [] expected2 = [] sum_cols = [ 'total_budgetary_resources_cpe', 'budget_authority_appropria_cpe', 'budget_authority_unobligat_fyb', 'adjustments_to_unobligated_cpe', 'other_budgetary_resources_cpe', 'contract_authority_amount_cpe', 'borrowing_authority_amount_cpe', 'spending_authority_from_of_cpe', 'status_of_budgetary_resour_cpe', 'obligations_incurred_total_cpe', 'gross_outlay_amount_by_tas_cpe', 'unobligated_balance_cpe', 'deobligations_recoveries_r_cpe' ] zero_sum_cols = {sum_col: '0' for sum_col in sum_cols} expected1_sum_cols = zero_sum_cols.copy() expected1_sum_cols['budget_authority_appropria_cpe'] = '3.00' expected2_sum_cols = zero_sum_cols.copy() expected2_sum_cols['budget_authority_unobligat_fyb'] = '4.00' for value in file_generation_manager.fileA.db_columns: # loop through all values and format date columns if value in sf1.__dict__: expected1.append(str(sf1.__dict__[value] or '')) expected2.append(str(sf2.__dict__[value] or '')) elif value in expected1_sum_cols: expected1.append(expected1_sum_cols[value]) expected2.append(expected2_sum_cols[value]) assert expected1 in file_rows assert expected2 in file_rows
def run_app(): """Run the application.""" app = create_app() # This is for DataDog (Do Not Delete) if USE_DATADOG: TraceMiddleware(app, tracer, service="broker-dd", distributed_tracing=False) with app.app_context(): current_app.debug = CONFIG_SERVICES['debug'] local = CONFIG_BROKER['local'] g.is_local = local error_report_path = CONFIG_SERVICES['error_report_path'] current_app.config.from_object(__name__) # Create connection to job tracker database sess = GlobalDB.db().session # Future: Override config w/ environment variable, if set current_app.config.from_envvar('VALIDATOR_SETTINGS', silent=True) queue = sqs_queue() messages = [] logger.info("Starting SQS polling") while True: # Set current_message to None before every loop to ensure it's never set to the previous message current_message = None try: # Grabs one (or more) messages from the queue messages = queue.receive_messages( WaitTimeSeconds=10, MessageAttributeNames=['All']) for message in messages: logger.info("Message received: %s", message.body) # Retrieve the job_id from the message body current_message = message g.job_id = message.body mark_job_status(g.job_id, "ready") # Get the job job = sess.query(Job).filter_by( job_id=g.job_id).one_or_none() if job is None: validation_error_type = ValidationError.jobError write_file_error(g.job_id, None, validation_error_type) raise ResponseException( 'Job ID {} not found in database'.format(g.job_id), StatusCode.CLIENT_ERROR, None, validation_error_type) # We have two major functionalities in the Validator: validation and file generation if (not job.file_type or job.file_type.letter_name in ['A', 'B', 'C', 'FABS'] or job.job_type.name != 'file_upload') and job.submission_id: # Run validations validation_manager = ValidationManager( local, error_report_path) validation_manager.validate_job(job.job_id) else: # Retrieve the agency code data from the message attributes msg_attr = current_message.message_attributes agency_code = msg_attr['agency_code']['StringValue'] if msg_attr and \ msg_attr.get('agency_code') else None agency_type = msg_attr['agency_type']['StringValue'] if msg_attr and \ msg_attr.get('agency_type') else None file_generation_manager = FileGenerationManager( job, agency_code, agency_type, local) file_generation_manager.generate_from_job() sess.commit() sess.refresh(job) # Delete from SQS once processed message.delete() except ResponseException as e: # Handle exceptions explicitly raised during validation. logger.error(traceback.format_exc()) job = get_current_job() if job: if job.filename is not None: # Insert file-level error info to the database write_file_error(job.job_id, job.filename, e.errorType, e.extraInfo) if e.errorType != ValidationError.jobError: # Job passed prerequisites for validation but an error happened somewhere: mark job as 'invalid' mark_job_status(job.job_id, 'invalid') if current_message: if e.errorType in [ ValidationError.rowCountError, ValidationError.headerError, ValidationError.fileTypeError ]: current_message.delete() except Exception as e: # Handle uncaught exceptions in validation process. logger.error(traceback.format_exc()) # csv-specific errors get a different job status and response code if isinstance(e, ValueError) or isinstance( e, csv.Error) or isinstance(e, UnicodeDecodeError): job_status = 'invalid' else: job_status = 'failed' job = get_current_job() if job: if job.filename is not None: error_type = ValidationError.unknownError if isinstance(e, UnicodeDecodeError): error_type = ValidationError.encodingError # TODO Is this really the only case where the message should be deleted? if current_message: current_message.delete() write_file_error(job.job_id, job.filename, error_type) mark_job_status(job.job_id, job_status) finally: GlobalDB.close() # Set visibility to 0 so that another attempt can be made to process in SQS immediately, # instead of waiting for the timeout window to expire for message in messages: try: message.change_visibility(VisibilityTimeout=0) except ClientError: # Deleted messages will throw errors, which is fine because they are handled pass
def test_generate_e_file(mock_broker_config_paths, database): """ Verify that generate_e_file makes an appropriate query (matching both D1 and D2 entries) and creates a file matching the expected DUNS """ # Generate several file D1 entries, largely with the same submission_id, and with two overlapping DUNS. Generate # several D2 entries with the same submission_id as well sess = database.session sub = SubmissionFactory() sub_2 = SubmissionFactory() sess.add_all([sub, sub_2]) sess.commit() file_path = str(mock_broker_config_paths['broker_files'].join('e_test1')) job = JobFactory(job_status_id=JOB_STATUS_DICT['running'], job_type_id=JOB_TYPE_DICT['file_upload'], file_type_id=FILE_TYPE_DICT['executive_compensation'], filename=file_path, original_filename='e_test1', submission_id=sub.submission_id) database.session.add(job) database.session.commit() model = AwardProcurementFactory(submission_id=sub.submission_id) aps = [ AwardProcurementFactory(submission_id=sub.submission_id) for _ in range(4) ] afas = [ AwardFinancialAssistanceFactory(submission_id=sub.submission_id) for _ in range(5) ] same_duns = AwardProcurementFactory( submission_id=sub.submission_id, awardee_or_recipient_uniqu=model.awardee_or_recipient_uniqu) unrelated = AwardProcurementFactory(submission_id=sub_2.submission_id) duns_list = [ DunsFactory( awardee_or_recipient_uniqu=model.awardee_or_recipient_uniqu) ] duns_list.extend([ DunsFactory(awardee_or_recipient_uniqu=ap.awardee_or_recipient_uniqu) for ap in aps ]) duns_list.extend([ DunsFactory(awardee_or_recipient_uniqu=afa.awardee_or_recipient_uniqu) for afa in afas ]) sess.add_all(aps + afas + duns_list + [model, same_duns, unrelated]) sess.commit() file_gen_manager = FileGenerationManager(database.session, CONFIG_BROKER['local'], job=job) file_gen_manager.generate_file() # check headers file_rows = read_file_rows(file_path) assert file_rows[0] == [ 'AwardeeOrRecipientUniqueIdentifier', 'AwardeeOrRecipientLegalEntityName', 'UltimateParentUniqueIdentifier', 'UltimateParentLegalEntityName', 'HighCompOfficer1FullName', 'HighCompOfficer1Amount', 'HighCompOfficer2FullName', 'HighCompOfficer2Amount', 'HighCompOfficer3FullName', 'HighCompOfficer3Amount', 'HighCompOfficer4FullName', 'HighCompOfficer4Amount', 'HighCompOfficer5FullName', 'HighCompOfficer5Amount' ] # Check listed DUNS expected = [[ duns.awardee_or_recipient_uniqu, duns.legal_business_name, duns.ultimate_parent_unique_ide, duns.ultimate_parent_legal_enti, duns.high_comp_officer1_full_na, duns.high_comp_officer1_amount, duns.high_comp_officer2_full_na, duns.high_comp_officer2_amount, duns.high_comp_officer3_full_na, duns.high_comp_officer3_amount, duns.high_comp_officer4_full_na, duns.high_comp_officer4_amount, duns.high_comp_officer5_full_na, duns.high_comp_officer5_amount ] for duns in duns_list] received = [file_row for file_row in file_rows[1:]] assert sorted(received) == list(sorted(expected))