def test_check_generation_prereqs_bad_type(database): """ Tests that check_generation_prereqs raises an error if an invalid type is provided. """ sess = database.session sub = SubmissionFactory() sess.add(sub) sess.commit() with pytest.raises(ResponseException): check_generation_prereqs(sub.submission_id, 'A')
def test_check_generation_prereqs_bad_type(database): """ Tests that check_generation_prereqs raises an error if an invalid type is provided. """ sess = database.session sub = SubmissionFactory() sess.add(sub) sess.commit() with pytest.raises(ResponseException): check_generation_prereqs(sub.submission_id, 'A')
def test_check_generation_prereqs_d_has_errors(database): """ Tests a set of conditions that has an error in one of the A,B,C files, prevent D file generation. """ sess = database.session sub = SubmissionFactory(submission_id=1, d2_submission=False) job_1 = JobFactory(submission_id=sub.submission_id, job_type_id=JOB_TYPE_DICT['csv_record_validation'], file_type_id=FILE_TYPE_DICT['appropriations'], job_status_id=JOB_STATUS_DICT['finished'], number_of_errors=1, number_of_warnings=0, error_message=None) job_2 = JobFactory(submission_id=sub.submission_id, job_type_id=JOB_TYPE_DICT['csv_record_validation'], file_type_id=FILE_TYPE_DICT['program_activity'], job_status_id=JOB_STATUS_DICT['finished'], number_of_errors=0, number_of_warnings=0, error_message=None) job_3 = JobFactory(submission_id=sub.submission_id, job_type_id=JOB_TYPE_DICT['csv_record_validation'], file_type_id=FILE_TYPE_DICT['award_financial'], job_status_id=JOB_STATUS_DICT['finished'], number_of_errors=0, number_of_warnings=0, error_message=None) sess.add_all([sub, job_1, job_2, job_3]) sess.commit() can_generate = check_generation_prereqs(sub.submission_id, 'D1') assert can_generate is False
def test_check_generation_prereqs_d_valid(database): """ Tests a set of conditions that passes the prerequisite checks to allow D files to be generated. Show that warnings do not prevent generation. """ sess = database.session sub = SubmissionFactory(submission_id=1, is_fabs=False) job_1 = JobFactory(submission_id=sub.submission_id, job_type_id=JOB_TYPE_DICT['csv_record_validation'], file_type_id=FILE_TYPE_DICT['appropriations'], job_status_id=JOB_STATUS_DICT['finished'], number_of_errors=0, number_of_warnings=0, error_message=None) job_2 = JobFactory(submission_id=sub.submission_id, job_type_id=JOB_TYPE_DICT['csv_record_validation'], file_type_id=FILE_TYPE_DICT['program_activity'], job_status_id=JOB_STATUS_DICT['finished'], number_of_errors=0, number_of_warnings=0, error_message=None) job_3 = JobFactory(submission_id=sub.submission_id, job_type_id=JOB_TYPE_DICT['csv_record_validation'], file_type_id=FILE_TYPE_DICT['award_financial'], job_status_id=JOB_STATUS_DICT['finished'], number_of_errors=0, number_of_warnings=1, error_message=None) sess.add_all([sub, job_1, job_2, job_3]) sess.commit() can_generate = check_generation_prereqs(sub.submission_id, 'D1') assert can_generate is True
def test_check_generation_prereqs_ef_has_errors(database): """ Tests a set of conditions that has an error in cross-file, fail the generation check for E/F files. """ sess = database.session sub = SubmissionFactory(submission_id=1, d2_submission=False) cross_val = JobFactory(submission_id=sub.submission_id, job_type_id=JOB_TYPE_DICT['validation'], file_type_id=None, job_status_id=JOB_STATUS_DICT['finished'], number_of_errors=1, number_of_warnings=0, error_message=None) sess.add_all([sub, cross_val]) sess.commit() can_generate = check_generation_prereqs(sub.submission_id, 'E') assert can_generate is False
def test_check_generation_prereqs_ef_valid(database): """ Tests a set of conditions that passes the prerequisite checks to allow E/F files to be generated. Show that warnings do not prevent generation. """ sess = database.session sub = SubmissionFactory(submission_id=1, d2_submission=False) cross_val = JobFactory(submission_id=sub.submission_id, job_type_id=JOB_TYPE_DICT['validation'], file_type_id=None, job_status_id=JOB_STATUS_DICT['finished'], number_of_errors=0, number_of_warnings=1, error_message=None) sess.add_all([sub, cross_val]) sess.commit() can_generate = check_generation_prereqs(sub.submission_id, 'E') assert can_generate is True
def test_check_generation_prereqs_ef_has_errors(database): """ Tests a set of conditions that has an error in cross-file, fail the generation check for E/F files. """ sess = database.session sub = SubmissionFactory(submission_id=1, d2_submission=False) cross_val = JobFactory(submission_id=sub.submission_id, job_type_id=JOB_TYPE_DICT['validation'], file_type_id=None, job_status_id=JOB_STATUS_DICT['finished'], number_of_errors=1, number_of_warnings=0, error_message=None) sess.add_all([sub, cross_val]) sess.commit() can_generate = check_generation_prereqs(sub.submission_id, 'E') assert can_generate is False
def test_check_generation_prereqs_d_has_errors(database): """ Tests a set of conditions that has an error in one of the A,B,C files, prevent D file generation. """ sess = database.session sub = SubmissionFactory(submission_id=1, d2_submission=False) job_1 = JobFactory(submission_id=sub.submission_id, job_type_id=JOB_TYPE_DICT['csv_record_validation'], file_type_id=FILE_TYPE_DICT['appropriations'], job_status_id=JOB_STATUS_DICT['finished'], number_of_errors=1, number_of_warnings=0, error_message=None) job_2 = JobFactory(submission_id=sub.submission_id, job_type_id=JOB_TYPE_DICT['csv_record_validation'], file_type_id=FILE_TYPE_DICT['program_activity'], job_status_id=JOB_STATUS_DICT['finished'], number_of_errors=0, number_of_warnings=0, error_message=None) job_3 = JobFactory(submission_id=sub.submission_id, job_type_id=JOB_TYPE_DICT['csv_record_validation'], file_type_id=FILE_TYPE_DICT['award_financial'], job_status_id=JOB_STATUS_DICT['finished'], number_of_errors=0, number_of_warnings=0, error_message=None) sess.add_all([sub, job_1, job_2, job_3]) sess.commit() can_generate = check_generation_prereqs(sub.submission_id, 'D1') assert can_generate is False
def test_check_generation_prereqs_ef_valid(database): """ Tests a set of conditions that passes the prerequisite checks to allow E/F files to be generated. Show that warnings do not prevent generation. """ sess = database.session sub = SubmissionFactory(submission_id=1, d2_submission=False) cross_val = JobFactory(submission_id=sub.submission_id, job_type_id=JOB_TYPE_DICT['validation'], file_type_id=None, job_status_id=JOB_STATUS_DICT['finished'], number_of_errors=0, number_of_warnings=1, error_message=None) sess.add_all([sub, cross_val]) sess.commit() can_generate = check_generation_prereqs(sub.submission_id, 'E') assert can_generate is True
def generate_file(submission, file_type, start, end, agency_type): """ Start a file generation job for the specified file type within a submission Args: submission: submission for which we're generating the file file_type: type of file to generate the job for start: the start date for the file to generate end: the end date for the file to generate agency_type: The type of agency (awarding or funding) to generate the file for (only used for D file generation) Returns: Results of check_generation or JsonResponse object containing an error if the prerequisite job isn't complete. """ error_message = None # submission is a FABS submission if submission.d2_submission: error_message = "Cannot generate files for FABS submissions." elif file_type in ['D1', 'D2']: # D file generation requires start and end date if not start or not end: error_message = "Must have a start and end date for D file generation." # D files can only be generated by awarding or funding agency elif agency_type not in ['awarding', 'funding']: error_message = "agency_type must be either awarding or funding for D file generation." # Only D1, D2, E, and F files can be generated elif file_type not in ['E', 'F']: error_message = "File type must be either D1, D2, E, or F" # Return any client errors if error_message: return JsonResponse.error(ValueError(error_message), StatusCode.CLIENT_ERROR) sess = GlobalDB.db().session job = sess.query(Job).filter( Job.submission_id == submission.submission_id, Job.file_type_id == lookups.FILE_TYPE_DICT_LETTER_ID[file_type], Job.job_type_id == lookups.JOB_TYPE_DICT['file_upload']).one() log_data = { 'message': 'Starting {} file generation within submission {}'.format( file_type, submission.submission_id), 'message_type': 'BrokerInfo', 'submission_id': submission.submission_id, 'job_id': job.job_id, 'file_type': file_type } logger.info(log_data) # Check prerequisites on upload job if not generation_helper.check_generation_prereqs(submission.submission_id, file_type): return JsonResponse.error( ResponseException( "Must wait for completion of prerequisite validation job", StatusCode.CLIENT_ERROR), StatusCode.CLIENT_ERROR) try: if file_type in ['D1', 'D2']: generation_helper.start_d_generation(job, start, end, agency_type) else: generation_helper.start_e_f_generation(job) except Exception as e: mark_job_status(job.job_id, 'failed') job.error_message = str(e) sess.commit() return JsonResponse.error(e, StatusCode.INTERNAL_ERROR) # Return same response as check generation route return check_generation(submission, file_type)
def generate_file(submission, file_type, start, end, agency_type): """ Start a file generation job for the specified file type within a submission Args: submission: submission for which we're generating the file file_type: type of file to generate the job for start: the start date for the file to generate end: the end date for the file to generate agency_type: The type of agency (awarding or funding) to generate the file for (only used for D file generation) Returns: Results of check_generation or JsonResponse object containing an error if the prerequisite job isn't complete. """ error_message = None # submission is a FABS submission if submission.d2_submission: error_message = "Cannot generate files for FABS submissions." elif file_type in ['D1', 'D2']: # D file generation requires start and end date if not start or not end: error_message = "Must have a start and end date for D file generation." # D files can only be generated by awarding or funding agency elif agency_type not in ['awarding', 'funding']: error_message = "agency_type must be either awarding or funding for D file generation." # Only D1, D2, E, and F files can be generated elif file_type not in ['E', 'F']: error_message = "File type must be either D1, D2, E, or F" # Return any client errors if error_message: return JsonResponse.error(ValueError(error_message), StatusCode.CLIENT_ERROR) sess = GlobalDB.db().session job = sess.query(Job).filter(Job.submission_id == submission.submission_id, Job.file_type_id == lookups.FILE_TYPE_DICT_LETTER_ID[file_type], Job.job_type_id == lookups.JOB_TYPE_DICT['file_upload']).one() logger.info({ 'message': 'Starting {} file generation within submission {}'.format(file_type, submission.submission_id), 'message_type': 'BrokerInfo', 'submission_id': submission.submission_id, 'job_id': job.job_id, 'file_type': file_type }) # Check prerequisites on upload job if not generation_helper.check_generation_prereqs(submission.submission_id, file_type): return JsonResponse.error(ResponseException("Must wait for completion of prerequisite validation job", StatusCode.CLIENT_ERROR), StatusCode.CLIENT_ERROR) try: if file_type in ['D1', 'D2']: generation_helper.start_d_generation(job, start, end, agency_type) else: generation_helper.start_e_f_generation(job) except Exception as e: mark_job_status(job.job_id, 'failed') job.error_message = str(e) sess.commit() return JsonResponse.error(e, StatusCode.INTERNAL_ERROR) # Return same response as check generation route return check_generation(submission, file_type)