def test_start_d_generation_submission_change_request(database, monkeypatch): """ In-submission generations that change their requested start or end dates must actually generate files based on the new dates. """ sess = database.session original_filename = 'D1_test_gen.csv' file_path = gen_file_path_from_submission('None/', original_filename) submission = SubmissionFactory(submission_id=1000, reporting_start_date='2017-01-01', reporting_end_date='2017-01-31', cgac_code='123', frec_code=None, is_quarter_format=False, publishable=False, reporting_fiscal_year='2017') file_gen = FileGenerationFactory(request_date=datetime.now().date(), start_date='2017-01-01', end_date='2017-01-31', file_type='D1', agency_code='123', agency_type='awarding', is_cached_file=True, file_path=file_path, file_generation_id=1000, file_format='csv') up_job = JobFactory(job_status_id=JOB_STATUS_DICT['waiting'], error_message=None, job_type_id=JOB_TYPE_DICT['file_upload'], file_type_id=FILE_TYPE_DICT['award_procurement'], filename=None, submission_id=submission.submission_id, file_generation_id=file_gen.file_generation_id, original_filename=original_filename) val_job = JobFactory(job_status_id=JOB_STATUS_DICT['waiting'], error_message=None, file_type_id=FILE_TYPE_DICT['award_procurement'], job_type_id=JOB_TYPE_DICT['csv_record_validation'], filename=None, submission_id=submission.submission_id, original_filename=original_filename) sess.add_all([submission, file_gen, up_job, val_job]) sess.commit() monkeypatch.setattr( generation_helper, 'g', Mock(return_value={'is_local': CONFIG_BROKER['local']})) start_d_generation(up_job, '01/01/2017', '01/30/2017', 'funding') assert up_job.file_generation_id != file_gen.file_generation_id assert up_job.start_date == date(2017, 1, 1) assert up_job.end_date == date(2017, 1, 30) assert up_job.original_filename != original_filename assert up_job.filename != gen_file_path_from_submission( up_job.submission_id, original_filename) assert up_job.start_date == date(2017, 1, 1) assert up_job.end_date == date(2017, 1, 30) assert up_job.original_filename == up_job.original_filename assert up_job.filename == up_job.filename
def test_retrieve_cached_file_generation_not_cached(database): """ Should return no FileGeneration """ sess = database.session job = JobFactory(start_date='2017-01-01', end_date='2017-01-31', job_status_id=JOB_STATUS_DICT['waiting'], error_message=None, file_type_id=FILE_TYPE_DICT['award'], job_type_id=JOB_TYPE_DICT['file_upload'], filename=None, original_filename=None, file_generation_id=None) file_gen = FileGenerationFactory(request_date=datetime.now().date(), start_date='2017-01-01', end_date='2017-01-31', file_type='D2', agency_code='123', agency_type='awarding', is_cached_file=False, file_format='csv') sess.add_all([job, file_gen]) sess.commit() file_generation = retrieve_cached_file_generation(job, 'awarding', '123', 'csv') assert file_generation is None
def test_copy_file_generation_to_job(monkeypatch, database): sess = database.session original_filename = 'new_filename.csv' file_path = gen_file_path_from_submission('None', original_filename) job = JobFactory(job_status_id=JOB_STATUS_DICT['running'], job_type_id=JOB_TYPE_DICT['file_upload'], file_type_id=FILE_TYPE_DICT['award']) file_gen = FileGenerationFactory(file_type='D1', file_path=file_path, file_format='csv') sess.add_all([job, file_gen]) sess.commit() monkeypatch.setattr( generation_helper, 'g', Mock(return_value={'is_local': CONFIG_BROKER['local']})) copy_file_generation_to_job(job, file_gen, True) sess.refresh(job) sess.refresh(file_gen) assert job.job_status.name == 'finished' assert job.filename == gen_file_path_from_submission( job.submission_id, original_filename) assert job.original_filename == original_filename assert job.number_of_errors == 0 assert job.number_of_warnings == 0 assert job.file_generation_id == file_gen.file_generation_id
def test_start_d_generation_submission_cached(database, monkeypatch): """ Cached D files must update the upload Job with the FileGeneration data. """ sess = database.session original_filename = 'D1_test_gen.csv' file_path = gen_file_path_from_submission('None/', original_filename) submission = SubmissionFactory(submission_id=1000, reporting_start_date='2017-01-01', reporting_end_date='2017-01-31', frec_code='1234', cgac_code=None, is_quarter_format=False, publishable=False, reporting_fiscal_year='2017') file_gen = FileGenerationFactory(request_date=datetime.now().date(), start_date='2017-01-01', end_date='2017-01-31', file_type='D2', agency_code='1234', agency_type='awarding', is_cached_file=True, file_path=file_path) up_job = JobFactory(job_status_id=JOB_STATUS_DICT['waiting'], file_type_id=FILE_TYPE_DICT['award'], error_message=None, job_type_id=JOB_TYPE_DICT['file_upload'], filename=None, original_filename=None, submission_id=submission.submission_id) val_job = JobFactory(job_status_id=JOB_STATUS_DICT['waiting'], error_message=None, file_type_id=FILE_TYPE_DICT['award'], job_type_id=JOB_TYPE_DICT['csv_record_validation'], filename=None, original_filename=None, submission_id=submission.submission_id) sess.add_all([submission, file_gen, up_job, val_job]) sess.commit() monkeypatch.setattr( generation_helper, 'g', Mock(return_value={'is_local': CONFIG_BROKER['local']})) start_d_generation(up_job, '01/01/2017', '01/31/2017', 'awarding') assert up_job.file_generation_id == file_gen.file_generation_id assert up_job.start_date == date(2017, 1, 1) assert up_job.end_date == date(2017, 1, 31) assert up_job.original_filename == original_filename assert up_job.filename == gen_file_path_from_submission( up_job.submission_id, original_filename) assert up_job.job_status_id == JOB_STATUS_DICT['finished'] assert up_job.start_date == date(2017, 1, 1) assert up_job.end_date == date(2017, 1, 31) assert up_job.original_filename == original_filename assert up_job.filename == gen_file_path_from_submission( up_job.submission_id, original_filename) assert up_job.job_status_id != JOB_STATUS_DICT['waiting']
def test_copy_file_generation_to_job_attached(monkeypatch, database): sess = database.session original_filename = 'new_filename.csv' submission = SubmissionFactory(submission_id=1, reporting_fiscal_year='2022', reporting_fiscal_period='4', is_quarter_format=False) file_path = gen_file_path_from_submission(submission.submission_id, original_filename) up_job = JobFactory(submission=submission, job_status_id=JOB_STATUS_DICT['running'], job_type_id=JOB_TYPE_DICT['file_upload'], file_type_id=FILE_TYPE_DICT['award_procurement']) val_job = JobFactory(submission=submission, job_status_id=JOB_STATUS_DICT['running'], job_type_id=JOB_TYPE_DICT['csv_record_validation'], file_type_id=FILE_TYPE_DICT['award_procurement']) file_gen = FileGenerationFactory(file_type='D1', file_path=file_path, file_format='csv', agency_type='awarding', start_date='2022-01-01', end_date='2022-01-31') sess.add_all([submission, up_job, val_job, file_gen]) sess.commit() monkeypatch.setattr( generation_helper, 'g', Mock(return_value={'is_local': CONFIG_BROKER['local']})) monkeypatch.setattr(generation_helper, 'get_timestamp', Mock(return_value='123456789')) monkeypatch.setattr(shutil, 'copyfile', Mock()) copy_file_generation_to_job(up_job, file_gen, True) sess.refresh(up_job) sess.refresh(file_gen) expected_filename = 'SubID-1_File-D1_FY22P04_20220101_20220131_awarding_123456789.csv' assert up_job.job_status.name == 'finished' assert up_job.filename == gen_file_path_from_submission( up_job.submission_id, expected_filename) assert up_job.original_filename == expected_filename assert up_job.number_of_errors == 0 assert up_job.number_of_warnings == 0 assert up_job.file_generation_id == file_gen.file_generation_id assert val_job.filename == gen_file_path_from_submission( up_job.submission_id, expected_filename) assert val_job.original_filename == expected_filename
def test_generate_file_updates_jobs(monkeypatch, mock_broker_config_paths, database): sess = database.session job1 = JobFactory(job_status_id=JOB_STATUS_DICT['running'], job_type_id=JOB_TYPE_DICT['file_upload'], file_type_id=FILE_TYPE_DICT['award_procurement'], filename=None, original_filename=None, start_date='01/01/2017', end_date='01/31/2017') job2 = JobFactory(job_status_id=JOB_STATUS_DICT['running'], job_type_id=JOB_TYPE_DICT['file_upload'], file_type_id=FILE_TYPE_DICT['award_procurement'], filename=None, original_filename=None, start_date='01/01/2017', end_date='01/31/2017') job3 = JobFactory(job_status_id=JOB_STATUS_DICT['running'], job_type_id=JOB_TYPE_DICT['file_upload'], file_type_id=FILE_TYPE_DICT['award_procurement'], filename=None, original_filename=None, start_date='01/01/2017', end_date='01/31/2017') file_gen = FileGenerationFactory(request_date=datetime.now().date(), start_date='01/01/2017', end_date='01/31/2017', file_type='D1', agency_code='123', agency_type='awarding', is_cached_file=True, file_path=None) sess.add_all([job1, job2, job3, file_gen]) sess.commit() job1.file_generation_id = file_gen.file_generation_id job2.file_generation_id = file_gen.file_generation_id job3.file_generation_id = file_gen.file_generation_id sess.commit() monkeypatch.setattr(generation_helper, 'g', Mock(return_value={'is_local': CONFIG_BROKER['local']})) file_gen_manager = FileGenerationManager(sess, CONFIG_BROKER['local'], file_generation=file_gen) file_gen_manager.generate_file() sess.refresh(file_gen) original_filename = file_gen.file_path.split('/')[-1] assert job1.job_status_id == JOB_STATUS_DICT['finished'] assert job1.original_filename == original_filename assert job1.filename == '{}{}'.format( CONFIG_BROKER['broker_files'] if CONFIG_BROKER['local'] else job1.submission_id + '/', original_filename) assert job2.job_status_id == JOB_STATUS_DICT['finished'] assert job2.original_filename == original_filename assert job2.filename == '{}{}'.format( CONFIG_BROKER['broker_files'] if CONFIG_BROKER['local'] else job2.submission_id + '/', original_filename) assert job3.job_status_id == JOB_STATUS_DICT['finished'] assert job3.original_filename == original_filename assert job3.filename == '{}{}'.format( CONFIG_BROKER['broker_files'] if CONFIG_BROKER['local'] else job3.submission_id + '/', original_filename)
def test_generate_funding_d2(mock_broker_config_paths, database): sess = database.session pafa = PublishedAwardFinancialAssistanceFactory pafa_1 = pafa(funding_agency_code='123', action_date='20170101', afa_generated_unique='unique1', is_active=True) pafa_2 = pafa(funding_agency_code='123', action_date='20170131', afa_generated_unique='unique2', is_active=True) pafa_3 = pafa(funding_agency_code='123', action_date='20161231', afa_generated_unique='unique3', is_active=True) pafa_4 = pafa(funding_agency_code='123', action_date='20170201', afa_generated_unique='unique4', is_active=True) pafa_5 = pafa(funding_agency_code='123', action_date='20170115', afa_generated_unique='unique5', is_active=False) pafa_6 = pafa(funding_agency_code='234', action_date='20170115', afa_generated_unique='unique6', is_active=True) file_gen = FileGenerationFactory(request_date=datetime.now().date(), start_date='01/01/2017', end_date='01/31/2017', file_type='D2', agency_code='123', agency_type='funding', is_cached_file=True, file_path=None) sess.add_all([pafa_1, pafa_2, pafa_3, pafa_4, pafa_5, pafa_6, file_gen]) sess.commit() file_gen_manager = FileGenerationManager(sess, CONFIG_BROKER['local'], file_generation=file_gen) file_gen_manager.generate_file() assert file_gen.file_path is not None # check headers file_rows = read_file_rows(file_gen.file_path) assert file_rows[0] == [key for key in file_generation_manager.fileD2.mapping] # check body pafa1 = sess.query(PublishedAwardFinancialAssistance).filter_by(afa_generated_unique='unique1').first() pafa2 = sess.query(PublishedAwardFinancialAssistance).filter_by(afa_generated_unique='unique2').first() expected1, expected2 = [], [] for value in file_generation_manager.fileD2.db_columns: # loop through all values and format date columns if value in ['period_of_performance_star', 'period_of_performance_curr', 'modified_at', 'action_date']: expected1.append(re.sub(r"[-]", r"", str(pafa1.__dict__[value]))[0:8]) expected2.append(re.sub(r"[-]", r"", str(pafa2.__dict__[value]))[0:8]) else: expected1.append(str(pafa1.__dict__[value])) expected2.append(str(pafa2.__dict__[value])) assert expected1 in file_rows assert expected2 in file_rows
def test_generate_funding_d1(mock_broker_config_paths, database): sess = database.session dap_model = DetachedAwardProcurementFactory dap_1 = dap_model(funding_agency_code='123', action_date='20170101', detached_award_proc_unique='unique1') dap_2 = dap_model(funding_agency_code='123', action_date='20170131', detached_award_proc_unique='unique2') dap_3 = dap_model(funding_agency_code='123', action_date='20170201', detached_award_proc_unique='unique3') dap_4 = dap_model(funding_agency_code='123', action_date='20161231', detached_award_proc_unique='unique4') dap_5 = dap_model(funding_agency_code='234', action_date='20170115', detached_award_proc_unique='unique5') file_gen = FileGenerationFactory(request_date=datetime.now().date(), start_date='01/01/2017', end_date='01/31/2017', file_type='D1', agency_code='123', agency_type='funding', is_cached_file=True, file_path=None) sess.add_all([dap_1, dap_2, dap_3, dap_4, dap_5, file_gen]) sess.commit() file_gen_manager = FileGenerationManager(sess, CONFIG_BROKER['local'], file_generation=file_gen) file_gen_manager.generate_file() assert file_gen.file_path is not None # check headers file_rows = read_file_rows(file_gen.file_path) assert file_rows[0] == [key for key in file_generation_manager.fileD1.mapping] # check body dap_one = sess.query(DetachedAwardProcurement).filter_by(detached_award_proc_unique='unique1').first() dap_two = sess.query(DetachedAwardProcurement).filter_by(detached_award_proc_unique='unique2').first() expected1, expected2 = [], [] for value in file_generation_manager.fileD1.db_columns: # loop through all values and format date columns if value in ['period_of_performance_star', 'period_of_performance_curr', 'period_of_perf_potential_e', 'ordering_period_end_date', 'action_date', 'last_modified', 'solicitation_date']: expected1.append(re.sub(r"[-]", r"", str(dap_one.__dict__[value]))[0:8]) expected2.append(re.sub(r"[-]", r"", str(dap_two.__dict__[value]))[0:8]) else: expected1.append(str(dap_one.__dict__[value])) expected2.append(str(dap_two.__dict__[value])) assert expected1 in file_rows assert expected2 in file_rows