def test_check_generation_prereqs_bad_type(database):
""" Tests that check_generation_prereqs raises an error if an invalid type is provided. """
sess = database.session
sub = SubmissionFactory()
sess.add(sub)
sess.commit()
with pytest.raises(ResponseException):
check_generation_prereqs(sub.submission_id, 'A')
def test_check_generation_prereqs_bad_type(database):
""" Tests that check_generation_prereqs raises an error if an invalid type is provided. """
sess = database.session
sub = SubmissionFactory()
sess.add(sub)
sess.commit()
with pytest.raises(ResponseException):
check_generation_prereqs(sub.submission_id, 'A')
def test_check_generation_prereqs_d_has_errors(database):
""" Tests a set of conditions that has an error in one of the A,B,C files, prevent D file generation. """
sess = database.session
sub = SubmissionFactory(submission_id=1, d2_submission=False)
job_1 = JobFactory(submission_id=sub.submission_id,
job_type_id=JOB_TYPE_DICT['csv_record_validation'],
file_type_id=FILE_TYPE_DICT['appropriations'],
job_status_id=JOB_STATUS_DICT['finished'],
number_of_errors=1,
number_of_warnings=0,
error_message=None)
job_2 = JobFactory(submission_id=sub.submission_id,
job_type_id=JOB_TYPE_DICT['csv_record_validation'],
file_type_id=FILE_TYPE_DICT['program_activity'],
job_status_id=JOB_STATUS_DICT['finished'],
number_of_errors=0,
number_of_warnings=0,
error_message=None)
job_3 = JobFactory(submission_id=sub.submission_id,
job_type_id=JOB_TYPE_DICT['csv_record_validation'],
file_type_id=FILE_TYPE_DICT['award_financial'],
job_status_id=JOB_STATUS_DICT['finished'],
number_of_errors=0,
number_of_warnings=0,
error_message=None)
sess.add_all([sub, job_1, job_2, job_3])
sess.commit()
can_generate = check_generation_prereqs(sub.submission_id, 'D1')
assert can_generate is False
def test_check_generation_prereqs_d_valid(database):
""" Tests a set of conditions that passes the prerequisite checks to allow D files to be generated. Show that
warnings do not prevent generation.
"""
sess = database.session
sub = SubmissionFactory(submission_id=1, is_fabs=False)
job_1 = JobFactory(submission_id=sub.submission_id,
job_type_id=JOB_TYPE_DICT['csv_record_validation'],
file_type_id=FILE_TYPE_DICT['appropriations'],
job_status_id=JOB_STATUS_DICT['finished'],
number_of_errors=0,
number_of_warnings=0,
error_message=None)
job_2 = JobFactory(submission_id=sub.submission_id,
job_type_id=JOB_TYPE_DICT['csv_record_validation'],
file_type_id=FILE_TYPE_DICT['program_activity'],
job_status_id=JOB_STATUS_DICT['finished'],
number_of_errors=0,
number_of_warnings=0,
error_message=None)
job_3 = JobFactory(submission_id=sub.submission_id,
job_type_id=JOB_TYPE_DICT['csv_record_validation'],
file_type_id=FILE_TYPE_DICT['award_financial'],
job_status_id=JOB_STATUS_DICT['finished'],
number_of_errors=0,
number_of_warnings=1,
error_message=None)
sess.add_all([sub, job_1, job_2, job_3])
sess.commit()
can_generate = check_generation_prereqs(sub.submission_id, 'D1')
assert can_generate is True
def test_check_generation_prereqs_ef_has_errors(database):
""" Tests a set of conditions that has an error in cross-file, fail the generation check for E/F files. """
sess = database.session
sub = SubmissionFactory(submission_id=1, d2_submission=False)
cross_val = JobFactory(submission_id=sub.submission_id, job_type_id=JOB_TYPE_DICT['validation'], file_type_id=None,
job_status_id=JOB_STATUS_DICT['finished'], number_of_errors=1, number_of_warnings=0,
error_message=None)
sess.add_all([sub, cross_val])
sess.commit()
can_generate = check_generation_prereqs(sub.submission_id, 'E')
assert can_generate is False
def test_check_generation_prereqs_ef_valid(database):
""" Tests a set of conditions that passes the prerequisite checks to allow E/F files to be generated. Show that
warnings do not prevent generation.
"""
sess = database.session
sub = SubmissionFactory(submission_id=1, d2_submission=False)
cross_val = JobFactory(submission_id=sub.submission_id, job_type_id=JOB_TYPE_DICT['validation'],
file_type_id=None, job_status_id=JOB_STATUS_DICT['finished'], number_of_errors=0,
number_of_warnings=1, error_message=None)
sess.add_all([sub, cross_val])
sess.commit()
can_generate = check_generation_prereqs(sub.submission_id, 'E')
assert can_generate is True
def test_check_generation_prereqs_ef_has_errors(database):
""" Tests a set of conditions that has an error in cross-file, fail the generation check for E/F files. """
sess = database.session
sub = SubmissionFactory(submission_id=1, d2_submission=False)
cross_val = JobFactory(submission_id=sub.submission_id,
job_type_id=JOB_TYPE_DICT['validation'],
file_type_id=None,
job_status_id=JOB_STATUS_DICT['finished'],
number_of_errors=1,
number_of_warnings=0,
error_message=None)
sess.add_all([sub, cross_val])
sess.commit()
can_generate = check_generation_prereqs(sub.submission_id, 'E')
assert can_generate is False
def test_check_generation_prereqs_d_has_errors(database):
""" Tests a set of conditions that has an error in one of the A,B,C files, prevent D file generation. """
sess = database.session
sub = SubmissionFactory(submission_id=1, d2_submission=False)
job_1 = JobFactory(submission_id=sub.submission_id, job_type_id=JOB_TYPE_DICT['csv_record_validation'],
file_type_id=FILE_TYPE_DICT['appropriations'], job_status_id=JOB_STATUS_DICT['finished'],
number_of_errors=1, number_of_warnings=0, error_message=None)
job_2 = JobFactory(submission_id=sub.submission_id, job_type_id=JOB_TYPE_DICT['csv_record_validation'],
file_type_id=FILE_TYPE_DICT['program_activity'], job_status_id=JOB_STATUS_DICT['finished'],
number_of_errors=0, number_of_warnings=0, error_message=None)
job_3 = JobFactory(submission_id=sub.submission_id, job_type_id=JOB_TYPE_DICT['csv_record_validation'],
file_type_id=FILE_TYPE_DICT['award_financial'], job_status_id=JOB_STATUS_DICT['finished'],
number_of_errors=0, number_of_warnings=0, error_message=None)
sess.add_all([sub, job_1, job_2, job_3])
sess.commit()
can_generate = check_generation_prereqs(sub.submission_id, 'D1')
assert can_generate is False
def test_check_generation_prereqs_ef_valid(database):
""" Tests a set of conditions that passes the prerequisite checks to allow E/F files to be generated. Show that
warnings do not prevent generation.
"""
sess = database.session
sub = SubmissionFactory(submission_id=1, d2_submission=False)
cross_val = JobFactory(submission_id=sub.submission_id,
job_type_id=JOB_TYPE_DICT['validation'],
file_type_id=None,
job_status_id=JOB_STATUS_DICT['finished'],
number_of_errors=0,
number_of_warnings=1,
error_message=None)
sess.add_all([sub, cross_val])
sess.commit()
can_generate = check_generation_prereqs(sub.submission_id, 'E')
assert can_generate is True
def generate_file(submission, file_type, start, end, agency_type):
""" Start a file generation job for the specified file type within a submission
Args:
submission: submission for which we're generating the file
file_type: type of file to generate the job for
start: the start date for the file to generate
end: the end date for the file to generate
agency_type: The type of agency (awarding or funding) to generate the file for (only used for D file
generation)
Returns:
Results of check_generation or JsonResponse object containing an error if the prerequisite job isn't
complete.
"""
error_message = None
# submission is a FABS submission
if submission.d2_submission:
error_message = "Cannot generate files for FABS submissions."
elif file_type in ['D1', 'D2']:
# D file generation requires start and end date
if not start or not end:
error_message = "Must have a start and end date for D file generation."
# D files can only be generated by awarding or funding agency
elif agency_type not in ['awarding', 'funding']:
error_message = "agency_type must be either awarding or funding for D file generation."
# Only D1, D2, E, and F files can be generated
elif file_type not in ['E', 'F']:
error_message = "File type must be either D1, D2, E, or F"
# Return any client errors
if error_message:
return JsonResponse.error(ValueError(error_message),
StatusCode.CLIENT_ERROR)
sess = GlobalDB.db().session
job = sess.query(Job).filter(
Job.submission_id == submission.submission_id,
Job.file_type_id == lookups.FILE_TYPE_DICT_LETTER_ID[file_type],
Job.job_type_id == lookups.JOB_TYPE_DICT['file_upload']).one()
log_data = {
'message':
'Starting {} file generation within submission {}'.format(
file_type, submission.submission_id),
'message_type':
'BrokerInfo',
'submission_id':
submission.submission_id,
'job_id':
job.job_id,
'file_type':
file_type
}
logger.info(log_data)
# Check prerequisites on upload job
if not generation_helper.check_generation_prereqs(submission.submission_id,
file_type):
return JsonResponse.error(
ResponseException(
"Must wait for completion of prerequisite validation job",
StatusCode.CLIENT_ERROR), StatusCode.CLIENT_ERROR)
try:
if file_type in ['D1', 'D2']:
generation_helper.start_d_generation(job, start, end, agency_type)
else:
generation_helper.start_e_f_generation(job)
except Exception as e:
mark_job_status(job.job_id, 'failed')
job.error_message = str(e)
sess.commit()
return JsonResponse.error(e, StatusCode.INTERNAL_ERROR)
# Return same response as check generation route
return check_generation(submission, file_type)
def generate_file(submission, file_type, start, end, agency_type):
""" Start a file generation job for the specified file type within a submission
Args:
submission: submission for which we're generating the file
file_type: type of file to generate the job for
start: the start date for the file to generate
end: the end date for the file to generate
agency_type: The type of agency (awarding or funding) to generate the file for (only used for D file
generation)
Returns:
Results of check_generation or JsonResponse object containing an error if the prerequisite job isn't
complete.
"""
error_message = None
# submission is a FABS submission
if submission.d2_submission:
error_message = "Cannot generate files for FABS submissions."
elif file_type in ['D1', 'D2']:
# D file generation requires start and end date
if not start or not end:
error_message = "Must have a start and end date for D file generation."
# D files can only be generated by awarding or funding agency
elif agency_type not in ['awarding', 'funding']:
error_message = "agency_type must be either awarding or funding for D file generation."
# Only D1, D2, E, and F files can be generated
elif file_type not in ['E', 'F']:
error_message = "File type must be either D1, D2, E, or F"
# Return any client errors
if error_message:
return JsonResponse.error(ValueError(error_message), StatusCode.CLIENT_ERROR)
sess = GlobalDB.db().session
job = sess.query(Job).filter(Job.submission_id == submission.submission_id,
Job.file_type_id == lookups.FILE_TYPE_DICT_LETTER_ID[file_type],
Job.job_type_id == lookups.JOB_TYPE_DICT['file_upload']).one()
logger.info({
'message': 'Starting {} file generation within submission {}'.format(file_type, submission.submission_id),
'message_type': 'BrokerInfo',
'submission_id': submission.submission_id,
'job_id': job.job_id,
'file_type': file_type
})
# Check prerequisites on upload job
if not generation_helper.check_generation_prereqs(submission.submission_id, file_type):
return JsonResponse.error(ResponseException("Must wait for completion of prerequisite validation job",
StatusCode.CLIENT_ERROR), StatusCode.CLIENT_ERROR)
try:
if file_type in ['D1', 'D2']:
generation_helper.start_d_generation(job, start, end, agency_type)
else:
generation_helper.start_e_f_generation(job)
except Exception as e:
mark_job_status(job.job_id, 'failed')
job.error_message = str(e)
sess.commit()
return JsonResponse.error(e, StatusCode.INTERNAL_ERROR)
# Return same response as check generation route
return check_generation(submission, file_type)
