def test_list_submissions_failure(database, monkeypatch):
user = UserFactory(user_id=1)
sub = SubmissionFactory(user_id=1, submission_id=1, number_of_errors=1, publish_status_id=1)
add_models(database, [user, sub])
monkeypatch.setattr(fileHandler, 'g', Mock(user=user))
result = list_submissions_result()
assert result['total'] == 1
assert result['submissions'][0]['status'] == "validation_errors"
delete_models(database, [user, sub])
user = UserFactory(user_id=1)
sub = SubmissionFactory(user_id=1, submission_id=1, publish_status_id=1)
job = JobFactory(submission_id=1, job_status_id=JOB_STATUS_DICT['failed'],
job_type_id=JOB_TYPE_DICT['csv_record_validation'], file_type_id=FILE_TYPE_DICT['award'])
add_models(database, [user, sub, job])
result = list_submissions_result()
assert result['total'] == 1
assert result['submissions'][0]['status'] == "failed"
delete_models(database, [user, sub, job])
user = UserFactory(user_id=1)
sub = SubmissionFactory(user_id=1, submission_id=1, publish_status_id=1)
job = JobFactory(submission_id=1, job_status_id=JOB_STATUS_DICT['invalid'],
job_type_id=JOB_TYPE_DICT['csv_record_validation'], file_type_id=FILE_TYPE_DICT['award'])
add_models(database, [user, sub, job])
result = list_submissions_result()
assert result['total'] == 1
assert result['submissions'][0]['status'] == "file_errors"
delete_models(database, [user, sub, job])
def test_check_job_dependencies_has_unfinished_dependencies(database):
""" Tests check_job_dependencies with a job that isn't finished """
sess = database.session
sub = SubmissionFactory(submission_id=1)
job = JobFactory(submission_id=sub.submission_id,
job_status_id=JOB_STATUS_DICT['finished'],
job_type_id=JOB_TYPE_DICT['csv_record_validation'],
file_type_id=FILE_TYPE_DICT['award'],
number_of_errors=0)
job_2 = JobFactory(submission_id=sub.submission_id,
job_status_id=JOB_STATUS_DICT['waiting'],
job_type_id=JOB_TYPE_DICT['csv_record_validation'],
file_type_id=FILE_TYPE_DICT['award'])
job_3 = JobFactory(submission_id=sub.submission_id,
job_status_id=JOB_STATUS_DICT['waiting'],
job_type_id=JOB_TYPE_DICT['csv_record_validation'],
file_type_id=FILE_TYPE_DICT['award'],
number_of_errors=0)
sess.add_all([sub, job, job_2, job_3])
sess.commit()
# Job 1 finished, it is a prerequisite for job 2 (waiting)
job_dep = JobDependency(job_id=job_2.job_id, prerequisite_id=job.job_id)
# Job 3 is also a prerequisite of job 2, it's not done, job 2 should stay in "waiting"
job_dep_2 = JobDependency(job_id=job_2.job_id,
prerequisite_id=job_3.job_id)
sess.add_all([job_dep, job_dep_2])
sess.commit()
check_job_dependencies(job.job_id)
assert job_2.job_status_id == JOB_STATUS_DICT['waiting']
def test_get_status_fabs(database):
""" Test get status function for a fabs submission """
sess = database.session
sub = SubmissionFactory(submission_id=1, d2_submission=True)
job_up = JobFactory(
submission_id=sub.submission_id,
job_type=sess.query(JobType).filter_by(name='file_upload').one(),
file_type=sess.query(FileType).filter_by(name='fabs').one(),
job_status=sess.query(JobStatus).filter_by(name='finished').one(),
number_of_errors=0,
number_of_warnings=0)
job_val = JobFactory(
submission_id=sub.submission_id,
job_type=sess.query(JobType).filter_by(
name='csv_record_validation').one(),
file_type=sess.query(FileType).filter_by(name='fabs').one(),
job_status=sess.query(JobStatus).filter_by(name='finished').one(),
number_of_errors=0,
number_of_warnings=4)
sess.add_all([sub, job_up, job_val])
sess.commit()
json_response = fileHandler.get_status(sub)
assert json_response.status_code == 200
json_content = json.loads(json_response.get_data().decode('UTF-8'))
assert json_content['fabs'] == {
'status': 'finished',
'has_errors': False,
'has_warnings': True,
'message': ''
}
def test_copy_parent_file_request_data(database):
sess = database.session
job_one = JobFactory(
job_status=sess.query(JobStatus).filter_by(name='finished').one(),
job_type=sess.query(JobType).filter_by(name='file_upload').one(),
file_type=sess.query(FileType).filter_by(name='award').one(),
)
job_two = JobFactory(
job_status=sess.query(JobStatus).filter_by(name='running').one(),
job_type=sess.query(JobType).filter_by(name='file_upload').one(),
file_type=sess.query(FileType).filter_by(name='award').one(),
filename='job_id/new_filename'
)
sess.add_all([job_one, job_two])
sess.commit()
file_generation_handler.copy_parent_file_request_data(sess, job_two, job_one, True)
sess.refresh(job_one)
sess.refresh(job_two)
assert job_two.job_status_id == job_one.job_status_id
assert job_two.filename == 'job_id/{}'.format(job_one.original_filename)
assert job_two.original_filename == job_one.original_filename
assert job_two.number_of_errors == job_one.number_of_errors
assert job_two.number_of_warnings == job_one.number_of_warnings
assert job_two.from_cached is True
def test_check_job_dependencies_prior_dependency_has_errors(database):
""" Tests check_job_dependencies with a job that is finished but has errors """
sess = database.session
sub = SubmissionFactory(submission_id=1)
job = JobFactory(
submission_id=sub.submission_id,
job_status=sess.query(JobStatus).filter_by(name='finished').one(),
job_type=sess.query(JobType).filter_by(
name='csv_record_validation').one(),
file_type=sess.query(FileType).filter_by(name='award').one(),
number_of_errors=3)
job_2 = JobFactory(
submission_id=sub.submission_id,
job_status=sess.query(JobStatus).filter_by(name='waiting').one(),
job_type=sess.query(JobType).filter_by(
name='csv_record_validation').one(),
file_type=sess.query(FileType).filter_by(name='award').one())
sess.add_all([sub, job, job_2])
sess.commit()
# Job 1 finished, it is a prerequisite for job 2 (waiting) but it has errors
job_dep = JobDependency(job_id=job_2.job_id, prerequisite_id=job.job_id)
sess.add(job_dep)
sess.commit()
check_job_dependencies(job.job_id)
assert job_2.job_status_id == JOB_STATUS_DICT['waiting']
def test_check_generation_prereqs_d_has_errors(database):
""" Tests a set of conditions that has an error in one of the A,B,C files, prevent D file generation. """
sess = database.session
sub = SubmissionFactory(submission_id=1, d2_submission=False)
job_1 = JobFactory(submission_id=sub.submission_id,
job_type_id=JOB_TYPE_DICT['csv_record_validation'],
file_type_id=FILE_TYPE_DICT['appropriations'],
job_status_id=JOB_STATUS_DICT['finished'],
number_of_errors=1,
number_of_warnings=0,
error_message=None)
job_2 = JobFactory(submission_id=sub.submission_id,
job_type_id=JOB_TYPE_DICT['csv_record_validation'],
file_type_id=FILE_TYPE_DICT['program_activity'],
job_status_id=JOB_STATUS_DICT['finished'],
number_of_errors=0,
number_of_warnings=0,
error_message=None)
job_3 = JobFactory(submission_id=sub.submission_id,
job_type_id=JOB_TYPE_DICT['csv_record_validation'],
file_type_id=FILE_TYPE_DICT['award_financial'],
job_status_id=JOB_STATUS_DICT['finished'],
number_of_errors=0,
number_of_warnings=0,
error_message=None)
sess.add_all([sub, job_1, job_2, job_3])
sess.commit()
can_generate = check_generation_prereqs(sub.submission_id, 'D1')
assert can_generate is False
def test_copy_parent_file_request_data(database):
sess = database.session
job_one = JobFactory(job_status_id=JOB_STATUS_DICT['finished'],
job_type_id=JOB_TYPE_DICT['file_upload'],
file_type_id=FILE_TYPE_DICT['award'])
job_two = JobFactory(job_status_id=JOB_STATUS_DICT['running'],
job_type_id=JOB_TYPE_DICT['file_upload'],
file_type_id=FILE_TYPE_DICT['award'],
filename='None/new_filename.csv')
sess.add_all([job_one, job_two])
sess.commit()
copy_parent_file_request_data(job_two, job_one, True)
sess.refresh(job_one)
sess.refresh(job_two)
assert job_two.job_status_id == job_one.job_status_id
filepath = CONFIG_BROKER['broker_files'] if CONFIG_BROKER[
'local'] else "{}/".format(str(job_two.submission_id))
assert job_two.filename == '{}{}'.format(filepath,
job_one.original_filename)
assert job_two.original_filename == job_one.original_filename
assert job_two.number_of_errors == job_one.number_of_errors
assert job_two.number_of_warnings == job_one.number_of_warnings
assert job_two.from_cached is True
def test_check_generation_prereqs_d_valid(database):
""" Tests a set of conditions that passes the prerequisite checks to allow D files to be generated. Show that
warnings do not prevent generation.
"""
sess = database.session
sub = SubmissionFactory(submission_id=1, is_fabs=False)
job_1 = JobFactory(submission_id=sub.submission_id,
job_type_id=JOB_TYPE_DICT['csv_record_validation'],
file_type_id=FILE_TYPE_DICT['appropriations'],
job_status_id=JOB_STATUS_DICT['finished'],
number_of_errors=0,
number_of_warnings=0,
error_message=None)
job_2 = JobFactory(submission_id=sub.submission_id,
job_type_id=JOB_TYPE_DICT['csv_record_validation'],
file_type_id=FILE_TYPE_DICT['program_activity'],
job_status_id=JOB_STATUS_DICT['finished'],
number_of_errors=0,
number_of_warnings=0,
error_message=None)
job_3 = JobFactory(submission_id=sub.submission_id,
job_type_id=JOB_TYPE_DICT['csv_record_validation'],
file_type_id=FILE_TYPE_DICT['award_financial'],
job_status_id=JOB_STATUS_DICT['finished'],
number_of_errors=0,
number_of_warnings=1,
error_message=None)
sess.add_all([sub, job_1, job_2, job_3])
sess.commit()
can_generate = check_generation_prereqs(sub.submission_id, 'D1')
assert can_generate is True
def test_get_status_fabs(database):
""" Test get status function for a fabs submission """
sess = database.session
sub = SubmissionFactory(submission_id=1, d2_submission=True)
job_up = JobFactory(submission_id=sub.submission_id,
job_type_id=JOB_TYPE_DICT['file_upload'],
file_type_id=FILE_TYPE_DICT['fabs'],
job_status_id=JOB_STATUS_DICT['finished'],
number_of_errors=0,
number_of_warnings=0)
job_val = JobFactory(submission_id=sub.submission_id,
job_type_id=JOB_TYPE_DICT['csv_record_validation'],
file_type_id=FILE_TYPE_DICT['fabs'],
job_status_id=JOB_STATUS_DICT['finished'],
number_of_errors=0,
number_of_warnings=4)
sess.add_all([sub, job_up, job_val])
sess.commit()
json_response = fileHandler.get_status(sub)
assert json_response.status_code == 200
json_content = json.loads(json_response.get_data().decode('UTF-8'))
assert json_content['fabs'] == {
'status': 'finished',
'has_errors': False,
'has_warnings': True,
'message': ''
}
def test_check_job_dependencies_ready(mock_sqs_queue, database):
""" Tests check_job_dependencies with a job that can be set to ready """
# Mock so it always returns the mock queue for the test
mock_sqs_queue.return_value = SQSMockQueue
sess = database.session
sub = SubmissionFactory(submission_id=1)
job = JobFactory(submission_id=sub.submission_id,
job_status_id=JOB_STATUS_DICT['finished'],
job_type_id=JOB_TYPE_DICT['csv_record_validation'],
file_type_id=FILE_TYPE_DICT['award'],
number_of_errors=0)
job_2 = JobFactory(submission_id=sub.submission_id,
job_status_id=JOB_STATUS_DICT['waiting'],
job_type_id=JOB_TYPE_DICT['csv_record_validation'],
file_type_id=FILE_TYPE_DICT['award'])
sess.add_all([sub, job, job_2])
sess.commit()
# Job 1 finished, it is a prerequisite for job 2 (waiting) but it has errors
job_dep = JobDependency(job_id=job_2.job_id, prerequisite_id=job.job_id)
sess.add(job_dep)
sess.commit()
check_job_dependencies(job.job_id)
assert job_2.job_status_id == JOB_STATUS_DICT['ready']
def test_start_d_generation_submission_change_request(database, monkeypatch):
""" In-submission generations that change their requested start or end dates must actually generate files based on
the new dates.
"""
sess = database.session
original_filename = 'D1_test_gen.csv'
file_path = gen_file_path_from_submission('None/', original_filename)
submission = SubmissionFactory(submission_id=1000,
reporting_start_date='2017-01-01',
reporting_end_date='2017-01-31',
cgac_code='123',
frec_code=None,
is_quarter_format=False,
publishable=False,
reporting_fiscal_year='2017')
file_gen = FileGenerationFactory(request_date=datetime.now().date(),
start_date='2017-01-01',
end_date='2017-01-31',
file_type='D1',
agency_code='123',
agency_type='awarding',
is_cached_file=True,
file_path=file_path,
file_generation_id=1000,
file_format='csv')
up_job = JobFactory(job_status_id=JOB_STATUS_DICT['waiting'],
error_message=None,
job_type_id=JOB_TYPE_DICT['file_upload'],
file_type_id=FILE_TYPE_DICT['award_procurement'],
filename=None,
submission_id=submission.submission_id,
file_generation_id=file_gen.file_generation_id,
original_filename=original_filename)
val_job = JobFactory(job_status_id=JOB_STATUS_DICT['waiting'],
error_message=None,
file_type_id=FILE_TYPE_DICT['award_procurement'],
job_type_id=JOB_TYPE_DICT['csv_record_validation'],
filename=None,
submission_id=submission.submission_id,
original_filename=original_filename)
sess.add_all([submission, file_gen, up_job, val_job])
sess.commit()
monkeypatch.setattr(
generation_helper, 'g',
Mock(return_value={'is_local': CONFIG_BROKER['local']}))
start_d_generation(up_job, '01/01/2017', '01/30/2017', 'funding')
assert up_job.file_generation_id != file_gen.file_generation_id
assert up_job.start_date == date(2017, 1, 1)
assert up_job.end_date == date(2017, 1, 30)
assert up_job.original_filename != original_filename
assert up_job.filename != gen_file_path_from_submission(
up_job.submission_id, original_filename)
assert up_job.start_date == date(2017, 1, 1)
assert up_job.end_date == date(2017, 1, 30)
assert up_job.original_filename == up_job.original_filename
assert up_job.filename == up_job.filename
def test_start_d_generation_submission_cached(database, monkeypatch):
""" Cached D files must update the upload Job with the FileGeneration data. """
sess = database.session
original_filename = 'D1_test_gen.csv'
file_path = gen_file_path_from_submission('None/', original_filename)
submission = SubmissionFactory(submission_id=1000,
reporting_start_date='2017-01-01',
reporting_end_date='2017-01-31',
frec_code='1234',
cgac_code=None,
is_quarter_format=False,
publishable=False,
reporting_fiscal_year='2017')
file_gen = FileGenerationFactory(request_date=datetime.now().date(),
start_date='2017-01-01',
end_date='2017-01-31',
file_type='D2',
agency_code='1234',
agency_type='awarding',
is_cached_file=True,
file_path=file_path)
up_job = JobFactory(job_status_id=JOB_STATUS_DICT['waiting'],
file_type_id=FILE_TYPE_DICT['award'],
error_message=None,
job_type_id=JOB_TYPE_DICT['file_upload'],
filename=None,
original_filename=None,
submission_id=submission.submission_id)
val_job = JobFactory(job_status_id=JOB_STATUS_DICT['waiting'],
error_message=None,
file_type_id=FILE_TYPE_DICT['award'],
job_type_id=JOB_TYPE_DICT['csv_record_validation'],
filename=None,
original_filename=None,
submission_id=submission.submission_id)
sess.add_all([submission, file_gen, up_job, val_job])
sess.commit()
monkeypatch.setattr(
generation_helper, 'g',
Mock(return_value={'is_local': CONFIG_BROKER['local']}))
start_d_generation(up_job, '01/01/2017', '01/31/2017', 'awarding')
assert up_job.file_generation_id == file_gen.file_generation_id
assert up_job.start_date == date(2017, 1, 1)
assert up_job.end_date == date(2017, 1, 31)
assert up_job.original_filename == original_filename
assert up_job.filename == gen_file_path_from_submission(
up_job.submission_id, original_filename)
assert up_job.job_status_id == JOB_STATUS_DICT['finished']
assert up_job.start_date == date(2017, 1, 1)
assert up_job.end_date == date(2017, 1, 31)
assert up_job.original_filename == original_filename
assert up_job.filename == gen_file_path_from_submission(
up_job.submission_id, original_filename)
assert up_job.job_status_id != JOB_STATUS_DICT['waiting']
def test_generate_new_d1_file_keep_old_job_files_success(
monkeypatch, mock_broker_config_paths, database):
""" Testing that when a new file is generated by a child job, the parent job's files stay the same """
sess = database.session
original_job = JobFactory(
job_status_id=JOB_STATUS_DICT['waiting'],
job_type_id=JOB_TYPE_DICT['file_upload'],
file_type_id=FILE_TYPE_DICT['award_procurement'],
filename=str(
mock_broker_config_paths['d_file_storage_path'].join('original')),
start_date='01/01/2017',
end_date='01/31/2017',
original_filename='original',
from_cached=False)
new_job = JobFactory(
job_status_id=JOB_STATUS_DICT['finished'],
job_type_id=JOB_TYPE_DICT['file_upload'],
file_type_id=FILE_TYPE_DICT['award_procurement'],
filename=str(
mock_broker_config_paths['d_file_storage_path'].join('original')),
start_date='01/01/2017',
end_date='01/31/2017',
original_filename='original',
from_cached=False)
sess.add_all([original_job, new_job])
sess.commit()
fr = FileRequestFactory(job=original_job,
parent_job_id=None,
is_cached_file=False,
agency_code='123',
agency_type='awarding',
start_date=original_job.start_date,
end_date=original_job.end_date,
file_type='D1',
request_date=datetime.now().date())
fr_2 = FileRequestFactory(job=new_job,
parent_job_id=original_job.job_id,
is_cached_file=False,
agency_code='123',
agency_type='awarding',
start_date=new_job.start_date,
end_date=new_job.end_date,
file_type='D1',
request_date=datetime.now().date())
sess.add_all([fr, fr_2])
sess.commit()
file_gen_manager = FileGenerationManager(original_job, '123', 'awarding',
CONFIG_BROKER['local'])
file_gen_manager.generate_from_job()
sess.refresh(original_job)
sess.refresh(new_job)
assert original_job.original_filename != 'original'
assert new_job.original_filename == 'original'
def test_uncache_new_d1_file_fpds_success(monkeypatch,
mock_broker_config_paths, database,
job_constants):
"""Testing that a new file is not generated if another job has already has a successfully generated file"""
sess = database.session
original_job = JobFactory(
job_status=sess.query(JobStatus).filter_by(name='finished').one(),
job_type=sess.query(JobType).filter_by(name='file_upload').one(),
file_type=sess.query(FileType).filter_by(
name='award_procurement').one(),
filename=str(
mock_broker_config_paths['d_file_storage_path'].join('original')),
start_date='01/01/2017',
end_date='01/31/2017',
original_filename='original',
from_cached=True,
)
sess.add(original_job)
sess.commit()
file_request = FileRequestFactory(
job=original_job,
is_cached_file=True,
agency_code='123',
start_date='01/01/2017',
end_date='01/31/2017',
file_type='D1',
request_date=(datetime.now().date() - timedelta(1)),
)
new_job = JobFactory(
job_status=sess.query(JobStatus).filter_by(name='waiting').one(),
job_type=sess.query(JobType).filter_by(name='file_upload').one(),
file_type=sess.query(FileType).filter_by(
name='award_procurement').one(),
start_date='01/01/2017',
end_date='01/31/2017',
)
sess.add_all([file_request, new_job])
sess.commit()
monkeypatch.setattr(file_generation_handler, 'retrieve_job_context_data',
Mock(return_value=(sess, new_job)))
FileGenerationManager().generate_from_job(new_job.job_id, '123')
sess.refresh(new_job)
file_request = sess.query(FileRequest).filter(
FileRequest.job_id == new_job.job_id).one_or_none()
assert file_request is not None
assert file_request.is_cached_file is True
assert file_request.start_date == new_job.start_date
assert file_request.end_date == new_job.end_date
assert file_request.agency_code == '123'
assert file_request.request_date == datetime.now().date()
assert new_job.original_filename != 'original'
assert new_job.from_cached is False
assert new_job.job_status_id == sess.query(JobStatus).filter_by(
name='finished').one().job_status_id
def test_generate_new_d1_file_different_dates_success(monkeypatch,
mock_broker_config_paths,
database):
""" Testing that a new D1 file is generated using the same data except for the dates """
sess = database.session
job = JobFactory(
job_status_id=JOB_STATUS_DICT['waiting'],
job_type_id=JOB_TYPE_DICT['file_upload'],
file_type_id=FILE_TYPE_DICT['award_procurement'],
filename=str(
mock_broker_config_paths['d_file_storage_path'].join('original')),
start_date='01/01/2017',
end_date='01/31/2017',
original_filename='original',
from_cached=True)
sess.add(job)
sess.commit()
fr = FileRequestFactory(job=job,
is_cached_file=True,
agency_code='123',
agency_type='awarding',
start_date=job.start_date,
end_date=job.end_date,
file_type='D1',
request_date=datetime.now().date())
sess.add(fr)
sess.commit()
# Change the job start date
old_start_date = job.start_date
job.start_date = '01/02/2017'
sess.commit()
file_gen_manager = FileGenerationManager(job, '123', 'awarding',
CONFIG_BROKER['local'])
file_gen_manager.generate_from_job()
sess.refresh(job)
new_file_request = sess.query(FileRequest).filter(
FileRequest.job_id == job.job_id,
FileRequest.is_cached_file.is_(True)).one_or_none()
assert new_file_request is not None
assert new_file_request.is_cached_file is True
assert new_file_request.start_date == job.start_date
assert new_file_request.end_date == job.end_date
assert new_file_request.agency_code == '123'
assert new_file_request.agency_type == 'awarding'
old_file_request = sess.query(FileRequest).filter(
FileRequest.job_id == job.job_id,
FileRequest.is_cached_file.is_(False)).one_or_none()
assert old_file_request is not None
assert old_file_request.is_cached_file is False
assert old_file_request.start_date == old_start_date
assert old_file_request.end_date == job.end_date
assert old_file_request.agency_code == '123'
assert old_file_request.agency_type == 'awarding'
def test_start_d_generation_submission_new(database, monkeypatch):
""" A new file generation must update the upload Job and create a new FileGeneration object. """
sess = database.session
original_filename = 'D2_test_gen.csv'
submission = SubmissionFactory(submission_id=1000,
reporting_start_date='2017-01-01',
reporting_end_date='2017-01-31',
cgac_code='123',
frec_code=None,
is_quarter_format=False,
publishable=False,
reporting_fiscal_year='2017')
up_job = JobFactory(job_status_id=JOB_STATUS_DICT['waiting'],
error_message=None,
file_type_id=FILE_TYPE_DICT['award'],
job_type_id=JOB_TYPE_DICT['file_upload'],
filename=None,
submission_id=submission.submission_id,
original_filename=original_filename,
file_generation_id=None)
val_job = JobFactory(job_status_id=JOB_STATUS_DICT['waiting'],
error_message=None,
file_type_id=FILE_TYPE_DICT['award'],
job_type_id=JOB_TYPE_DICT['csv_record_validation'],
filename=None,
submission_id=submission.submission_id,
original_filename=original_filename)
sess.add_all([submission, up_job, val_job])
sess.commit()
monkeypatch.setattr(
generation_helper, 'g',
Mock(return_value={'is_local': CONFIG_BROKER['local']}))
start_d_generation(up_job, '01/01/2017', '01/31/2017', 'awarding')
assert up_job.file_generation_id is not None
assert up_job.start_date == date(2017, 1, 1)
assert up_job.end_date == date(2017, 1, 31)
assert up_job.original_filename != original_filename
assert up_job.filename != gen_file_path_from_submission(
up_job.submission_id, original_filename)
assert up_job.start_date == date(2017, 1, 1)
assert up_job.end_date == date(2017, 1, 31)
assert up_job.original_filename == up_job.original_filename
assert up_job.filename == up_job.filename
file_gen = sess.query(FileGeneration).filter_by(
file_generation_id=up_job.file_generation_id).one_or_none()
assert file_gen is not None
assert file_gen.request_date == datetime.now().date()
assert file_gen.start_date == date(2017, 1, 1)
assert file_gen.end_date == date(2017, 1, 31)
assert file_gen.file_type == 'D2'
assert file_gen.file_path != gen_file_path_from_submission(
'None', original_filename)
def test_move_certified_data(database):
""" Tests the move_certified_data function """
with Flask('test-app').app_context():
sess = database.session
# Create 2 submissions
sub_1 = SubmissionFactory()
sub_2 = SubmissionFactory()
sess.add_all([sub_1, sub_2])
sess.commit()
# Create jobs so we can put a job ID into the tables
job_1 = JobFactory(submission_id=sub_1.submission_id)
job_2 = JobFactory(submission_id=sub_2.submission_id)
sess.add_all([job_1, job_2])
sess.commit()
# Create Appropriation entries, 1 per submission, and one of each other kind
approp_1 = Appropriation(submission_id=sub_1.submission_id, job_id=job_1.job_id, row_number=1,
spending_authority_from_of_cpe=2)
approp_2 = Appropriation(submission_id=sub_2.submission_id, job_id=job_2.job_id, row_number=1,
spending_authority_from_of_cpe=2)
ocpa = ObjectClassProgramActivity(submission_id=sub_1.submission_id, job_id=job_1.job_id, row_number=1)
award_fin = AwardFinancial(submission_id=sub_1.submission_id, job_id=job_1.job_id, row_number=1)
error_1 = ErrorMetadata(job_id=job_1.job_id)
error_2 = ErrorMetadata(job_id=job_2.job_id)
sess.add_all([approp_1, approp_2, ocpa, award_fin, error_1, error_2])
sess.commit()
move_certified_data(sess, sub_1.submission_id)
# There are 2 entries, we only want to move the 1 with the submission ID that matches
approp_query = sess.query(CertifiedAppropriation).filter_by(submission_id=sub_1.submission_id).all()
assert len(approp_query) == 1
assert approp_query[0].spending_authority_from_of_cpe == 2
# Make sure the others got moved as well
ocpa_query = sess.query(CertifiedObjectClassProgramActivity).filter_by(submission_id=sub_1.submission_id).all()
award_query = sess.query(CertifiedAwardFinancial).filter_by(submission_id=sub_1.submission_id).all()
# Query all job IDs but only one result should show up
error_query = sess.query(CertifiedErrorMetadata).\
filter(CertifiedErrorMetadata.job_id.in_([job_1.job_id, job_2.job_id])).all()
assert len(ocpa_query) == 1
assert len(award_query) == 1
assert len(error_query) == 1
# Change the Appropriation data
approp_1.spending_authority_from_of_cpe = 5
sess.refresh(approp_1)
# Move the data again (recertify) and make sure we didn't add extras, just adjusted the one we had
move_certified_data(sess, sub_1.submission_id)
approp_query = sess.query(CertifiedAppropriation).filter_by(submission_id=sub_1.submission_id).all()
assert len(approp_query) == 1
assert approp_query[0].spending_authority_from_of_cpe == 2
def test_list_submissions_failure(database, job_constants, monkeypatch):
fh = fileHandler.FileHandler(Mock())
mock_value = Mock()
mock_value.getName.return_value = 1
monkeypatch.setattr(fileHandler, 'LoginSession', mock_value)
user = UserFactory(user_id=1, cgac_code='cgac')
sub = SubmissionFactory(user_id=1,
submission_id=1,
number_of_errors=1,
cgac_code='cgac')
add_models(database, [user, sub])
json_response = fh.list_submissions(PAGE, LIMIT, CERTIFIED)
assert json.loads(json_response.get_data().decode("utf-8"))['total'] == 1
assert json.loads(json_response.get_data().decode(
"utf-8"))['submissions'][0]['status'] == "validation_errors"
delete_models(database, [user, sub])
sess = database.session
user = UserFactory(user_id=1, cgac_code='cgac')
sub = SubmissionFactory(user_id=1, submission_id=1, cgac_code='cgac')
job = JobFactory(
submission_id=1,
job_status=sess.query(JobStatus).filter_by(name='failed').one(),
job_type=sess.query(JobType).filter_by(
name='csv_record_validation').one(),
file_type=sess.query(FileType).filter_by(name='award').one())
add_models(database, [user, sub, job])
json_response = fh.list_submissions(PAGE, LIMIT, CERTIFIED)
assert json.loads(json_response.get_data().decode("utf-8"))['total'] == 1
assert json.loads(json_response.get_data().decode(
"utf-8"))['submissions'][0]['status'] == "failed"
delete_models(database, [user, sub, job])
sess = database.session
user = UserFactory(user_id=1, cgac_code='cgac')
sub = SubmissionFactory(user_id=1, submission_id=1, cgac_code='cgac')
job = JobFactory(
submission_id=1,
job_status=sess.query(JobStatus).filter_by(name='invalid').one(),
job_type=sess.query(JobType).filter_by(
name='csv_record_validation').one(),
file_type=sess.query(FileType).filter_by(name='award').one())
add_models(database, [user, sub, job])
json_response = fh.list_submissions(PAGE, LIMIT, CERTIFIED)
assert json.loads(json_response.get_data().decode("utf-8"))['total'] == 1
assert json.loads(json_response.get_data().decode(
"utf-8"))['submissions'][0]['status'] == "file_errors"
delete_models(database, [user, sub, job])
def test_get_submission_metadata_quarterly_dabs_cgac(database):
""" Tests the get_submission_metadata function for quarterly dabs submissions """
sess = database.session
now = datetime.datetime.utcnow()
now_plus_10 = now + datetime.timedelta(minutes=10)
cgac = CGACFactory(cgac_code='001', agency_name='CGAC Agency')
frec_cgac = CGACFactory(cgac_code='999', agency_name='FREC CGAC')
frec = FRECFactory(frec_code='0001', agency_name='FREC Agency', cgac=frec_cgac)
sub = SubmissionFactory(submission_id=1, created_at=now, updated_at=now_plus_10, cgac_code=cgac.cgac_code,
reporting_fiscal_period=3, reporting_fiscal_year=2017, is_quarter_format=True,
publish_status_id=PUBLISH_STATUS_DICT['updated'], d2_submission=False, number_of_errors=40,
number_of_warnings=200)
# Job for submission
job = JobFactory(submission_id=sub.submission_id, last_validated=now_plus_10,
job_type=sess.query(JobType).filter_by(name='csv_record_validation').one(),
job_status=sess.query(JobStatus).filter_by(name='finished').one(),
file_type=sess.query(FileType).filter_by(name='appropriations').one(),
number_of_rows=3,
file_size=7655)
job_2 = JobFactory(submission_id=sub.submission_id, last_validated=now_plus_10,
job_type=sess.query(JobType).filter_by(name='csv_record_validation').one(),
job_status=sess.query(JobStatus).filter_by(name='finished').one(),
file_type=sess.query(FileType).filter_by(name='program_activity').one(),
number_of_rows=7,
file_size=12345)
sess.add_all([cgac, frec_cgac, frec, sub, job, job_2])
sess.commit()
# Test for Quarterly, updated DABS cgac submission
expected_results = {
'cgac_code': cgac.cgac_code,
'frec_code': None,
'agency_name': cgac.agency_name,
'number_of_errors': 40,
'number_of_warnings': 200,
'number_of_rows': 10,
'total_size': 20000,
'created_on': now.strftime('%m/%d/%Y'),
'last_updated': now_plus_10.strftime("%Y-%m-%dT%H:%M:%S"),
'last_validated': now_plus_10.strftime('%m/%d/%Y'),
'reporting_period': 'Q1/2017',
'publish_status': 'updated',
'quarterly_submission': True,
'fabs_submission': False,
'fabs_meta': None
}
results = get_submission_metadata(sub)
assert results == expected_results
def test_certify_dabs_submission(database, monkeypatch):
""" Tests the certify_dabs_submission function """
with Flask('test-app').app_context():
now = datetime.datetime.utcnow()
sess = database.session
user = UserFactory()
cgac = CGACFactory(cgac_code='001', agency_name='CGAC Agency')
submission = SubmissionFactory(created_at=now, updated_at=now, cgac_code=cgac.cgac_code,
reporting_fiscal_period=3, reporting_fiscal_year=2017, is_quarter_format=True,
publishable=True, publish_status_id=PUBLISH_STATUS_DICT['unpublished'],
d2_submission=False, number_of_errors=0, number_of_warnings=200,
certifying_user_id=None)
quarter_reval = QuarterlyRevalidationThresholdFactory(year=2017, quarter=1,
window_start=now - datetime.timedelta(days=1))
sess.add_all([user, cgac, submission, quarter_reval])
sess.commit()
comment = CommentFactory(file_type_id=FILE_TYPE_DICT['appropriations'], comment='Test',
submission_id=submission.submission_id)
job_1 = JobFactory(submission_id=submission.submission_id, last_validated=now,
job_type_id=JOB_TYPE_DICT['csv_record_validation'])
job_2 = JobFactory(submission_id=submission.submission_id, last_validated=now + datetime.timedelta(days=1),
job_type_id=JOB_TYPE_DICT['csv_record_validation'])
sess.add_all([job_1, job_2, comment])
sess.commit()
flex_field = FlexField(file_type_id=FILE_TYPE_DICT['appropriations'], header='flex_test', job_id=job_1.job_id,
submission_id=submission.submission_id, row_number=2, cell=None)
sess.add(flex_field)
sess.commit()
g.user = user
file_handler = fileHandler.FileHandler({}, is_local=True)
monkeypatch.setattr(file_handler, 'move_certified_files', Mock(return_value=True))
monkeypatch.setattr(fileHandler.GlobalDB, 'db', Mock(return_value=database))
certify_dabs_submission(submission, file_handler)
sess.refresh(submission)
certify_history = sess.query(CertifyHistory).filter_by(submission_id=submission.submission_id).one_or_none()
assert certify_history is not None
assert submission.certifying_user_id == user.user_id
assert submission.publish_status_id == PUBLISH_STATUS_DICT['published']
# Make sure certified comments are created
certified_comment = sess.query(CertifiedComment).filter_by(submission_id=submission.submission_id).one_or_none()
assert certified_comment is not None
# Make sure certified flex fields are created
certified_flex = sess.query(CertifiedFlexField).filter_by(submission_id=submission.submission_id).one_or_none()
assert certified_flex is not None
def test_uncache_new_d1_file_fpds_success(monkeypatch,
mock_broker_config_paths, database):
"""Testing that a new file is not generated if another job has already has a successfully generated file"""
sess = database.session
original_job = JobFactory(
job_status_id=JOB_STATUS_DICT['finished'],
job_type_id=JOB_TYPE_DICT['file_upload'],
file_type_id=FILE_TYPE_DICT['award_procurement'],
filename=str(
mock_broker_config_paths['d_file_storage_path'].join('original')),
start_date='01/01/2017',
end_date='01/31/2017',
original_filename='original',
from_cached=True)
sess.add(original_job)
sess.commit()
file_request = FileRequestFactory(job=original_job,
is_cached_file=True,
agency_code='123',
agency_type='awarding',
start_date='01/01/2017',
end_date='01/31/2017',
file_type='D1',
request_date=(datetime.now().date() -
timedelta(1)))
new_job = JobFactory(job_status_id=JOB_STATUS_DICT['waiting'],
job_type_id=JOB_TYPE_DICT['file_upload'],
file_type_id=FILE_TYPE_DICT['award_procurement'],
start_date='01/01/2017',
end_date='01/31/2017')
sess.add_all([file_request, new_job])
sess.commit()
file_gen_manager = FileGenerationManager(new_job, '123', 'awarding',
CONFIG_BROKER['local'])
file_gen_manager.generate_from_job()
sess.refresh(new_job)
file_request = sess.query(FileRequest).filter(
FileRequest.job_id == new_job.job_id).one_or_none()
assert file_request is not None
assert file_request.is_cached_file is True
assert file_request.start_date == new_job.start_date
assert file_request.end_date == new_job.end_date
assert file_request.agency_code == '123'
assert file_request.request_date == datetime.now().date()
assert new_job.original_filename != 'original'
assert new_job.from_cached is False
assert new_job.job_status_id == JOB_STATUS_DICT['finished']
def test_copy_file_generation_to_job_attached(monkeypatch, database):
sess = database.session
original_filename = 'new_filename.csv'
submission = SubmissionFactory(submission_id=1,
reporting_fiscal_year='2022',
reporting_fiscal_period='4',
is_quarter_format=False)
file_path = gen_file_path_from_submission(submission.submission_id,
original_filename)
up_job = JobFactory(submission=submission,
job_status_id=JOB_STATUS_DICT['running'],
job_type_id=JOB_TYPE_DICT['file_upload'],
file_type_id=FILE_TYPE_DICT['award_procurement'])
val_job = JobFactory(submission=submission,
job_status_id=JOB_STATUS_DICT['running'],
job_type_id=JOB_TYPE_DICT['csv_record_validation'],
file_type_id=FILE_TYPE_DICT['award_procurement'])
file_gen = FileGenerationFactory(file_type='D1',
file_path=file_path,
file_format='csv',
agency_type='awarding',
start_date='2022-01-01',
end_date='2022-01-31')
sess.add_all([submission, up_job, val_job, file_gen])
sess.commit()
monkeypatch.setattr(
generation_helper, 'g',
Mock(return_value={'is_local': CONFIG_BROKER['local']}))
monkeypatch.setattr(generation_helper, 'get_timestamp',
Mock(return_value='123456789'))
monkeypatch.setattr(shutil, 'copyfile', Mock())
copy_file_generation_to_job(up_job, file_gen, True)
sess.refresh(up_job)
sess.refresh(file_gen)
expected_filename = 'SubID-1_File-D1_FY22P04_20220101_20220131_awarding_123456789.csv'
assert up_job.job_status.name == 'finished'
assert up_job.filename == gen_file_path_from_submission(
up_job.submission_id, expected_filename)
assert up_job.original_filename == expected_filename
assert up_job.number_of_errors == 0
assert up_job.number_of_warnings == 0
assert up_job.file_generation_id == file_gen.file_generation_id
assert val_job.filename == gen_file_path_from_submission(
up_job.submission_id, expected_filename)
assert val_job.original_filename == expected_filename
def test_retrieve_cached_file_generation_not_cached(database):
""" Should return no FileGeneration """
sess = database.session
job = JobFactory(start_date='2017-01-01',
end_date='2017-01-31',
job_status_id=JOB_STATUS_DICT['waiting'],
error_message=None,
file_type_id=FILE_TYPE_DICT['award'],
job_type_id=JOB_TYPE_DICT['file_upload'],
filename=None,
original_filename=None,
file_generation_id=None)
file_gen = FileGenerationFactory(request_date=datetime.now().date(),
start_date='2017-01-01',
end_date='2017-01-31',
file_type='D2',
agency_code='123',
agency_type='awarding',
is_cached_file=False,
file_format='csv')
sess.add_all([job, file_gen])
sess.commit()
file_generation = retrieve_cached_file_generation(job, 'awarding', '123',
'csv')
assert file_generation is None
def test_certify_dabs_submission_quarterly_revalidation_not_in_db(database):
""" Tests that a DABS submission that doesnt have its year/quarter in the system won't be able to certify. """
with Flask('test-app').app_context():
now = datetime.datetime.utcnow()
sess = database.session
user = UserFactory()
cgac = CGACFactory(cgac_code='001', agency_name='CGAC Agency')
submission = SubmissionFactory(created_at=now, updated_at=now, cgac_code=cgac.cgac_code,
reporting_fiscal_period=3, reporting_fiscal_year=2017, is_quarter_format=True,
publishable=True, publish_status_id=PUBLISH_STATUS_DICT['unpublished'],
d2_submission=False, number_of_errors=0, number_of_warnings=200,
certifying_user_id=None)
sess.add_all([user, cgac, submission])
sess.commit()
job = JobFactory(submission_id=submission.submission_id, last_validated=now,
job_type_id=JOB_TYPE_DICT['csv_record_validation'])
sess.add(job)
sess.commit()
g.user = user
file_handler = fileHandler.FileHandler({}, is_local=True)
response = certify_dabs_submission(submission, file_handler)
response_json = json.loads(response.data.decode('UTF-8'))
assert response.status_code == 400
assert response_json['message'] == "No submission window for this year and quarter was found. If this is an " \
"error, please contact the Service Desk."
def test_certify_dabs_submission_quarterly_revalidation_too_early(database):
""" Tests that a DABS submission that was last validated before the window start cannot be certified. """
with Flask('test-app').app_context():
now = datetime.datetime.utcnow()
earlier = now - datetime.timedelta(days=1)
sess = database.session
user = UserFactory()
cgac = CGACFactory(cgac_code='001', agency_name='CGAC Agency')
submission = SubmissionFactory(created_at=earlier, updated_at=earlier, cgac_code=cgac.cgac_code,
reporting_fiscal_period=3, reporting_fiscal_year=2017, is_quarter_format=True,
publishable=True, publish_status_id=PUBLISH_STATUS_DICT['unpublished'],
d2_submission=False, number_of_errors=0, number_of_warnings=200,
certifying_user_id=None)
quarter_reval = QuarterlyRevalidationThresholdFactory(year=2017, quarter=1, window_start=now)
sess.add_all([user, cgac, submission, quarter_reval])
sess.commit()
job = JobFactory(submission_id=submission.submission_id, last_validated=earlier,
job_type_id=JOB_TYPE_DICT['csv_record_validation'])
sess.add(job)
sess.commit()
g.user = user
file_handler = fileHandler.FileHandler({}, is_local=True)
response = certify_dabs_submission(submission, file_handler)
response_json = json.loads(response.data.decode('UTF-8'))
assert response.status_code == 400
assert response_json['message'] == "This submission was last validated or its D files generated before the " \
"start of the submission window ({}). Please revalidate before " \
"certifying.".\
format(quarter_reval.window_start.strftime('%m/%d/%Y'))
def test_generate_new_d1_file_success(monkeypatch, mock_broker_config_paths,
database):
""" Testing that a new D1 file is generated """
sess = database.session
job = JobFactory(
job_status_id=JOB_STATUS_DICT['waiting'],
job_type_id=JOB_TYPE_DICT['file_upload'],
file_type_id=FILE_TYPE_DICT['award_procurement'],
filename=str(
mock_broker_config_paths['d_file_storage_path'].join('original')),
start_date='01/01/2017',
end_date='01/31/2017',
original_filename='original',
from_cached=True)
sess.add(job)
sess.commit()
file_gen_manager = FileGenerationManager(job, '123', 'awarding',
CONFIG_BROKER['local'])
file_gen_manager.generate_from_job()
sess.refresh(job)
file_request = sess.query(FileRequest).filter(
FileRequest.job_id == job.job_id).one_or_none()
assert file_request is not None
assert file_request.is_cached_file is True
assert file_request.start_date == job.start_date
assert file_request.end_date == job.end_date
assert file_request.agency_code == '123'
assert file_request.request_date == datetime.now().date()
assert job.original_filename != 'original'
assert job.from_cached is False
assert job.job_status_id == JOB_STATUS_DICT['finished']
def test_get_upload_file_url_s3(database, monkeypatch):
""" Test getting the url of the uploaded file non-locally. """
monkeypatch.setattr(fileHandler, 'CONFIG_BROKER', {'local': False})
s3_url_handler = Mock()
s3_url_handler.return_value.get_signed_url.return_value = 'some/url/here.csv'
monkeypatch.setattr(fileHandler, 'S3Handler', s3_url_handler)
# create and insert submission/job
sess = database.session
sub = SubmissionFactory(submission_id=1, d2_submission=False)
job = JobFactory(
submission_id=1,
job_status=sess.query(JobStatus).filter_by(name='finished').one(),
job_type=sess.query(JobType).filter_by(name='file_upload').one(),
file_type=sess.query(FileType).filter_by(name='appropriations').one(),
filename='1/some_file.csv')
add_models(database, [sub, job])
json_response = fileHandler.get_upload_file_url(sub, 'A')
url = json.loads(json_response.get_data().decode('utf-8'))['url']
assert url == 'some/url/here.csv'
assert s3_url_handler.return_value.get_signed_url.call_args == ((
'1', 'some_file.csv'), {
'method':
'GET'
})
def test_relevant_flex_data(database):
"""Verify that we can retrieve multiple flex fields from our data"""
sess = database.session
subs = [SubmissionFactory() for _ in range(3)]
sess.add_all(subs)
sess.commit()
# Three jobs per submission
jobs = [JobFactory(submission_id=sub.submission_id) for sub in subs for _ in range(3)]
sess.add_all(jobs)
sess.commit()
# Set up ten rows of three fields per job
sess.add_all([
FlexField(submission_id=job.submission_id, job_id=job.job_id, row_number=row_number, header=str(idx),
cell="cell"*row_number)
for job in jobs for idx in range(3) for row_number in range(1, 11)
])
sess.commit()
failures = [{'row_number': 3}, {'row_number': 7}]
result = validator.relevant_flex_data(failures, jobs[0].job_id)
assert {3, 7} == set(result.keys())
assert len(result[3]) == 3
# spot check some of the values
assert result[3][0].header == '0'
assert result[3][1].cell == 'cell' * 3
assert result[3][2].job_id == jobs[0].job_id
assert result[7][1].header == '1'
assert result[7][0].cell == 'cell' * 7
def test_certify_dabs_submission_quarterly_revalidation_multiple_thresholds(database):
""" Tests that a DABS submission is not affected by a different quarterly revalidation threshold than the one that
matches its reporting_start_date.
"""
with Flask('test-app').app_context():
now = datetime.datetime.utcnow()
earlier = now - datetime.timedelta(days=1)
sess = database.session
user = UserFactory()
cgac = CGACFactory(cgac_code='001', agency_name='CGAC Agency')
submission = SubmissionFactory(created_at=earlier, updated_at=earlier, cgac_code=cgac.cgac_code,
reporting_fiscal_period=3, reporting_fiscal_year=2017,
reporting_start_date='2016-10-01', is_quarter_format=True, publishable=True,
publish_status_id=PUBLISH_STATUS_DICT['unpublished'], d2_submission=False,
number_of_errors=0, number_of_warnings=200, certifying_user_id=None)
quarter_reval = QuarterlyRevalidationThresholdFactory(year=2017, quarter=1, window_start=earlier)
quarter_reval_2 = QuarterlyRevalidationThresholdFactory(year=2017, quarter=2,
window_start=now + datetime.timedelta(days=10))
sess.add_all([user, cgac, submission, quarter_reval, quarter_reval_2])
sess.commit()
job = JobFactory(submission_id=submission.submission_id, last_validated=now,
job_type_id=JOB_TYPE_DICT['csv_record_validation'])
sess.add(job)
sess.commit()
g.user = user
file_handler = fileHandler.FileHandler({}, is_local=True)
response = certify_dabs_submission(submission, file_handler)
assert response.status_code == 200
def test_copy_file_generation_to_job(monkeypatch, database):
sess = database.session
original_filename = 'new_filename.csv'
file_path = gen_file_path_from_submission('None', original_filename)
job = JobFactory(job_status_id=JOB_STATUS_DICT['running'],
job_type_id=JOB_TYPE_DICT['file_upload'],
file_type_id=FILE_TYPE_DICT['award'])
file_gen = FileGenerationFactory(file_type='D1',
file_path=file_path,
file_format='csv')
sess.add_all([job, file_gen])
sess.commit()
monkeypatch.setattr(
generation_helper, 'g',
Mock(return_value={'is_local': CONFIG_BROKER['local']}))
copy_file_generation_to_job(job, file_gen, True)
sess.refresh(job)
sess.refresh(file_gen)
assert job.job_status.name == 'finished'
assert job.filename == gen_file_path_from_submission(
job.submission_id, original_filename)
assert job.original_filename == original_filename
assert job.number_of_errors == 0
assert job.number_of_warnings == 0
assert job.file_generation_id == file_gen.file_generation_id
def test_check_detached_d_file_generation(database):
""" Job statuses should return the correct status and error message to the user """
sess = database.session
# Detached D2 generation waiting to be picked up by the Validator
job = JobFactory(job_status_id=JOB_STATUS_DICT['waiting'], job_type_id=JOB_TYPE_DICT['file_upload'],
file_type_id=FILE_TYPE_DICT['award'], error_message='', filename='job_id/file.csv',
original_filename='file.csv')
sess.add(job)
sess.commit()
response_dict = check_file_generation(job.job_id)
assert response_dict['status'] == 'waiting'
# Detached D2 generation running in the Validator
job.job_status_id = JOB_STATUS_DICT['running']
sess.commit()
response_dict = check_file_generation(job.job_id)
assert response_dict['status'] == 'waiting'
# Detached D2 generation completed by the Validator
job.job_status_id = JOB_STATUS_DICT['finished']
sess.commit()
response_dict = check_file_generation(job.job_id)
assert response_dict['status'] == 'finished'
assert response_dict['message'] == ''
# Detached D2 generation with an unknown error
job.job_status_id = JOB_STATUS_DICT['failed']
sess.commit()
response_dict = check_file_generation(job.job_id)
assert response_dict['status'] == 'failed'
assert response_dict['message'] == 'Upload job failed without error message'
# Detached D2 generation with a known error
job.error_message = 'FABS upload error message'
sess.commit()
response_dict = check_file_generation(job.job_id)
assert response_dict['status'] == 'failed'
assert response_dict['message'] == 'FABS upload error message'
def test_generate_file_updates_jobs(monkeypatch, mock_broker_config_paths, database):
sess = database.session
job1 = JobFactory(job_status_id=JOB_STATUS_DICT['running'], job_type_id=JOB_TYPE_DICT['file_upload'],
file_type_id=FILE_TYPE_DICT['award_procurement'], filename=None, original_filename=None,
start_date='01/01/2017', end_date='01/31/2017')
job2 = JobFactory(job_status_id=JOB_STATUS_DICT['running'], job_type_id=JOB_TYPE_DICT['file_upload'],
file_type_id=FILE_TYPE_DICT['award_procurement'], filename=None, original_filename=None,
start_date='01/01/2017', end_date='01/31/2017')
job3 = JobFactory(job_status_id=JOB_STATUS_DICT['running'], job_type_id=JOB_TYPE_DICT['file_upload'],
file_type_id=FILE_TYPE_DICT['award_procurement'], filename=None, original_filename=None,
start_date='01/01/2017', end_date='01/31/2017')
file_gen = FileGenerationFactory(request_date=datetime.now().date(), start_date='01/01/2017',
end_date='01/31/2017', file_type='D1', agency_code='123',
agency_type='awarding', is_cached_file=True, file_path=None)
sess.add_all([job1, job2, job3, file_gen])
sess.commit()
job1.file_generation_id = file_gen.file_generation_id
job2.file_generation_id = file_gen.file_generation_id
job3.file_generation_id = file_gen.file_generation_id
sess.commit()
monkeypatch.setattr(generation_helper, 'g', Mock(return_value={'is_local': CONFIG_BROKER['local']}))
file_gen_manager = FileGenerationManager(sess, CONFIG_BROKER['local'], file_generation=file_gen)
file_gen_manager.generate_file()
sess.refresh(file_gen)
original_filename = file_gen.file_path.split('/')[-1]
assert job1.job_status_id == JOB_STATUS_DICT['finished']
assert job1.original_filename == original_filename
assert job1.filename == '{}{}'.format(
CONFIG_BROKER['broker_files'] if CONFIG_BROKER['local'] else job1.submission_id + '/', original_filename)
assert job2.job_status_id == JOB_STATUS_DICT['finished']
assert job2.original_filename == original_filename
assert job2.filename == '{}{}'.format(
CONFIG_BROKER['broker_files'] if CONFIG_BROKER['local'] else job2.submission_id + '/', original_filename)
assert job3.job_status_id == JOB_STATUS_DICT['finished']
assert job3.original_filename == original_filename
assert job3.filename == '{}{}'.format(
CONFIG_BROKER['broker_files'] if CONFIG_BROKER['local'] else job3.submission_id + '/', original_filename)
def test_check_submission_d_file_generation(database):
""" Job statuses should return the correct status and error message to the user """
sess = database.session
sub = SubmissionFactory()
sess.add(sub)
# D1 generation waiting to be picked up by the Validator
job = JobFactory(job_status_id=JOB_STATUS_DICT['waiting'], job_type_id=JOB_TYPE_DICT['file_upload'],
file_type_id=FILE_TYPE_DICT['award_procurement'], submission=sub, error_message='',
filename='job_id/file.csv', original_filename='file.csv')
val_job = JobFactory(job_status_id=JOB_STATUS_DICT['waiting'], job_type_id=JOB_TYPE_DICT['csv_record_validation'],
file_type_id=FILE_TYPE_DICT['award_procurement'], submission=sub, error_message='',
number_of_errors=0)
sess.add_all([job, val_job])
sess.commit()
response_dict = check_file_generation(job.job_id)
assert response_dict['status'] == 'waiting'
# D1 generation running in the Validator
job.job_status_id = JOB_STATUS_DICT['running']
sess.commit()
response_dict = check_file_generation(job.job_id)
assert response_dict['status'] == 'waiting'
# D1 generation with an unknown error
job.job_status_id = JOB_STATUS_DICT['failed']
sess.commit()
response_dict = check_file_generation(job.job_id)
assert response_dict['status'] == 'failed'
assert response_dict['message'] == 'Upload job failed without error message'
# D1 generation with a known error
job.error_message = 'D1 upload error message'
sess.commit()
response_dict = check_file_generation(job.job_id)
assert response_dict['status'] == 'failed'
assert response_dict['message'] == 'D1 upload error message'
# D1 generation completed by the Validator; validation waiting to be picked up
job.error_message = ''
job.job_status_id = JOB_STATUS_DICT['finished']
sess.commit()
response_dict = check_file_generation(job.job_id)
assert response_dict['status'] == 'waiting'
# D1 generation completed; validation running in the Validator
val_job.job_status_id = JOB_STATUS_DICT['running']
sess.commit()
response_dict = check_file_generation(job.job_id)
assert response_dict['status'] == 'waiting'
# D1 generation completed; validation completed by the Validator
val_job.job_status_id = JOB_STATUS_DICT['finished']
sess.commit()
response_dict = check_file_generation(job.job_id)
assert response_dict['status'] == 'finished'
# D1 generation completed; validation completed by the Validator
val_job.number_of_errors = 10
sess.commit()
response_dict = check_file_generation(job.job_id)
assert response_dict['status'] == 'failed'
assert response_dict['message'] == 'Validation completed but row-level errors were found'
# D1 generation completed; validation with an unknown error
job.error_message = ''
val_job.error_message = ''
val_job.job_status_id = JOB_STATUS_DICT['failed']
val_job.number_of_errors = 0
sess.commit()
response_dict = check_file_generation(job.job_id)
assert response_dict['status'] == 'failed'
assert response_dict['message'] == 'Validation job had an internal error'
# D1 generation completed; validation with a known error
job.error_message = ''
val_job.error_message = ''
val_job.error_message = 'D1 upload error message'
sess.commit()
response_dict = check_file_generation(job.job_id)
assert response_dict['status'] == 'failed'
assert response_dict['message'] == 'D1 upload error message'
# D1 generation completed; validation with an unknown error
job.error_message = ''
val_job.error_message = ''
val_job.job_status_id = JOB_STATUS_DICT['invalid']
sess.commit()
response_dict = check_file_generation(job.job_id)
assert response_dict['status'] == 'failed'
assert response_dict['message'] == 'Generated file had file-level errors'
def test_current_page(file_app, database):
"""Test the route to check what the current progress of the submission is at
the correct page
"""
cgac = CGACFactory()
user = UserFactory.with_cgacs(cgac)
user.user_id = 1
user.name = 'Oliver Queen'
user.website_admin = True
database.session.add(user)
database.session.commit()
g.user = user
sub = SubmissionFactory(user_id=1, cgac_code=cgac.cgac_code)
database.session.add(sub)
database.session.commit()
csv_validation = JOB_TYPE_DICT['csv_record_validation']
upload = JOB_TYPE_DICT['file_upload']
validation = JOB_TYPE_DICT['validation']
finished_job = JOB_STATUS_DICT['finished']
waiting = JOB_STATUS_DICT['waiting']
job_a = JobFactory(submission_id=sub.submission_id, file_type_id=FILE_TYPE_DICT['appropriations'],
job_type_id=csv_validation, number_of_errors=0, file_size=123, job_status_id=finished_job)
job_b = JobFactory(submission_id=sub.submission_id, file_type_id=FILE_TYPE_DICT['program_activity'],
job_type_id=csv_validation, number_of_errors=0, file_size=123, job_status_id=finished_job)
job_c = JobFactory(submission_id=sub.submission_id, file_type_id=FILE_TYPE_DICT['award_financial'],
job_type_id=csv_validation, number_of_errors=0, file_size=123, job_status_id=finished_job)
job_d1 = JobFactory(submission_id=sub.submission_id, file_type_id=FILE_TYPE_DICT['award_procurement'],
job_type_id=csv_validation, number_of_errors=0, file_size=123, job_status_id=finished_job)
job_d2 = JobFactory(submission_id=sub.submission_id, file_type_id=FILE_TYPE_DICT['award'],
job_type_id=csv_validation, number_of_errors=0, file_size=123, job_status_id=finished_job)
job_e = JobFactory(submission_id=sub.submission_id, file_type_id=FILE_TYPE_DICT['executive_compensation'],
job_type_id=upload, number_of_errors=0, file_size=123, job_status_id=finished_job)
job_f = JobFactory(submission_id=sub.submission_id, file_type_id=FILE_TYPE_DICT['sub_award'], job_type_id=upload,
number_of_errors=0, file_size=123, job_status_id=finished_job)
job_cross_file = JobFactory(submission_id=sub.submission_id, file_type_id=None, job_type_id=validation,
number_of_errors=0, file_size=123, job_status_id=finished_job)
database.session.add_all([job_a, job_b, job_c, job_d1, job_d2, job_e, job_f, job_cross_file])
database.session.commit()
# Everything ok
response = file_app.get("/v1/check_current_page/?submission_id=" + str(sub.submission_id))
response_json = json.loads(response.data.decode('UTF-8'))
assert response_json['step'] == '5'
job_e.job_status_id = 6
database.session.commit()
# E or F failed
response = file_app.get("/v1/check_current_page/?submission_id=" + str(sub.submission_id))
response_json = json.loads(response.data.decode('UTF-8'))
assert response_json['step'] == '4'
job_e.job_status_id = 4
job_cross_file.number_of_errors = 6
database.session.commit()
# Restore job_e and create errors for cross_file
response = file_app.get("/v1/check_current_page/?submission_id=" + str(sub.submission_id))
response_json = json.loads(response.data.decode('UTF-8'))
assert response_json['step'] == '3'
job_d1.number_of_errors = 6
database.session.commit()
# D file has errors
response = file_app.get("/v1/check_current_page/?submission_id=" + str(sub.submission_id))
response_json = json.loads(response.data.decode('UTF-8'))
assert response_json['step'] == '2'
job_c.number_of_errors = 6
database.session.commit()
# Fail C file validation
response = file_app.get("/v1/check_current_page/?submission_id=" + str(sub.submission_id))
response_json = json.loads(response.data.decode('UTF-8'))
assert response_json['step'] == '1'
job_cross_file.job_status_id = waiting
job_d1.number_of_errors = 0
database.session.commit()
# E and F generated with C file errors
response = file_app.get("/v1/check_current_page/?submission_id=" + str(sub.submission_id))
response_json = json.loads(response.data.decode('UTF-8'))
assert response_json['step'] == '1'
