def test_check_detached_d_file_generation(database): """ Job statuses should return the correct status and error message to the user """ sess = database.session # Detached D2 generation waiting to be picked up by the Validator job = JobFactory(job_status_id=JOB_STATUS_DICT['waiting'], job_type_id=JOB_TYPE_DICT['file_upload'], file_type_id=FILE_TYPE_DICT['award'], error_message='', filename='job_id/file.csv', original_filename='file.csv') sess.add(job) sess.commit() response_dict = check_file_generation(job.job_id) assert response_dict['status'] == 'waiting' # Detached D2 generation running in the Validator job.job_status_id = JOB_STATUS_DICT['running'] sess.commit() response_dict = check_file_generation(job.job_id) assert response_dict['status'] == 'waiting' # Detached D2 generation completed by the Validator job.job_status_id = JOB_STATUS_DICT['finished'] sess.commit() response_dict = check_file_generation(job.job_id) assert response_dict['status'] == 'finished' assert response_dict['message'] == '' # Detached D2 generation with an unknown error job.job_status_id = JOB_STATUS_DICT['failed'] sess.commit() response_dict = check_file_generation(job.job_id) assert response_dict['status'] == 'failed' assert response_dict[ 'message'] == 'Upload job failed without error message' # Detached D2 generation with a known error job.error_message = 'FABS upload error message' sess.commit() response_dict = check_file_generation(job.job_id) assert response_dict['status'] == 'failed' assert response_dict['message'] == 'FABS upload error message'
def test_check_detached_d_file_generation(database): """ Job statuses should return the correct status and error message to the user """ sess = database.session # Detached D2 generation waiting to be picked up by the Validator job = JobFactory(job_status_id=JOB_STATUS_DICT['waiting'], job_type_id=JOB_TYPE_DICT['file_upload'], file_type_id=FILE_TYPE_DICT['award'], error_message='', filename='job_id/file.csv', original_filename='file.csv') sess.add(job) sess.commit() response_dict = check_file_generation(job.job_id) assert response_dict['status'] == 'waiting' # Detached D2 generation running in the Validator job.job_status_id = JOB_STATUS_DICT['running'] sess.commit() response_dict = check_file_generation(job.job_id) assert response_dict['status'] == 'waiting' # Detached D2 generation completed by the Validator job.job_status_id = JOB_STATUS_DICT['finished'] sess.commit() response_dict = check_file_generation(job.job_id) assert response_dict['status'] == 'finished' assert response_dict['message'] == '' # Detached D2 generation with an unknown error job.job_status_id = JOB_STATUS_DICT['failed'] sess.commit() response_dict = check_file_generation(job.job_id) assert response_dict['status'] == 'failed' assert response_dict['message'] == 'Upload job failed without error message' # Detached D2 generation with a known error job.error_message = 'FABS upload error message' sess.commit() response_dict = check_file_generation(job.job_id) assert response_dict['status'] == 'failed' assert response_dict['message'] == 'FABS upload error message'
def test_check_submission_d_file_generation(database): """ Job statuses should return the correct status and error message to the user """ sess = database.session sub = SubmissionFactory() sess.add(sub) # D1 generation waiting to be picked up by the Validator job = JobFactory(job_status_id=JOB_STATUS_DICT['waiting'], job_type_id=JOB_TYPE_DICT['file_upload'], file_type_id=FILE_TYPE_DICT['award_procurement'], submission=sub, error_message='', filename='job_id/file.csv', original_filename='file.csv') val_job = JobFactory(job_status_id=JOB_STATUS_DICT['waiting'], job_type_id=JOB_TYPE_DICT['csv_record_validation'], file_type_id=FILE_TYPE_DICT['award_procurement'], submission=sub, error_message='', number_of_errors=0) sess.add_all([job, val_job]) sess.commit() response_dict = check_file_generation(job.job_id) assert response_dict['status'] == 'waiting' # D1 generation running in the Validator job.job_status_id = JOB_STATUS_DICT['running'] sess.commit() response_dict = check_file_generation(job.job_id) assert response_dict['status'] == 'waiting' # D1 generation with an unknown error job.job_status_id = JOB_STATUS_DICT['failed'] sess.commit() response_dict = check_file_generation(job.job_id) assert response_dict['status'] == 'failed' assert response_dict['message'] == 'Upload job failed without error message' # D1 generation with a known error job.error_message = 'D1 upload error message' sess.commit() response_dict = check_file_generation(job.job_id) assert response_dict['status'] == 'failed' assert response_dict['message'] == 'D1 upload error message' # D1 generation completed by the Validator; validation waiting to be picked up job.error_message = '' job.job_status_id = JOB_STATUS_DICT['finished'] sess.commit() response_dict = check_file_generation(job.job_id) assert response_dict['status'] == 'waiting' # D1 generation completed; validation running in the Validator val_job.job_status_id = JOB_STATUS_DICT['running'] sess.commit() response_dict = check_file_generation(job.job_id) assert response_dict['status'] == 'waiting' # D1 generation completed; validation completed by the Validator val_job.job_status_id = JOB_STATUS_DICT['finished'] sess.commit() response_dict = check_file_generation(job.job_id) assert response_dict['status'] == 'finished' # D1 generation completed; validation completed by the Validator val_job.number_of_errors = 10 sess.commit() response_dict = check_file_generation(job.job_id) assert response_dict['status'] == 'failed' assert response_dict['message'] == 'Validation completed but row-level errors were found' # D1 generation completed; validation with an unknown error job.error_message = '' val_job.error_message = '' val_job.job_status_id = JOB_STATUS_DICT['failed'] val_job.number_of_errors = 0 sess.commit() response_dict = check_file_generation(job.job_id) assert response_dict['status'] == 'failed' assert response_dict['message'] == 'Validation job had an internal error' # D1 generation completed; validation with a known error job.error_message = '' val_job.error_message = '' val_job.error_message = 'D1 upload error message' sess.commit() response_dict = check_file_generation(job.job_id) assert response_dict['status'] == 'failed' assert response_dict['message'] == 'D1 upload error message' # D1 generation completed; validation with an unknown error job.error_message = '' val_job.error_message = '' val_job.job_status_id = JOB_STATUS_DICT['invalid'] sess.commit() response_dict = check_file_generation(job.job_id) assert response_dict['status'] == 'failed' assert response_dict['message'] == 'Generated file had file-level errors'
def test_current_page(file_app, database, user_constants, job_constants, monkeypatch): """Test the route to check what the current progress of the submission is at the correct page """ cgac = CGACFactory() user = UserFactory.with_cgacs(cgac) user.user_id = 1 user.name = 'Oliver Queen' user.website_admin = True database.session.add(user) database.session.commit() g.user = user sub = SubmissionFactory(user_id=1, cgac_code=cgac.cgac_code) database.session.add(sub) csv_validation = database.session.query(JobType).filter_by( name='csv_record_validation').one() upload = database.session.query(JobType).filter_by( name='file_upload').one() validation = database.session.query(JobType).filter_by( name='validation').one() finished_job = database.session.query(JobStatus).filter_by( name='finished').one() waiting = database.session.query(JobStatus).filter_by(name='waiting').one() job_a = JobFactory(submission_id=sub.submission_id, file_type=database.session.query(FileType).filter_by( name='appropriations').one(), job_type=csv_validation, number_of_errors=0, file_size=123, job_status=finished_job) job_b = JobFactory(submission_id=sub.submission_id, file_type=database.session.query(FileType).filter_by( name='program_activity').one(), job_type=csv_validation, number_of_errors=0, file_size=123, job_status=finished_job) job_c = JobFactory(submission_id=sub.submission_id, file_type=database.session.query(FileType).filter_by( name='award_financial').one(), job_type=csv_validation, number_of_errors=0, file_size=123, job_status=finished_job) job_d1 = JobFactory(submission_id=sub.submission_id, file_type=database.session.query(FileType).filter_by( name='award_procurement').one(), job_type=csv_validation, number_of_errors=0, file_size=123, job_status=finished_job) job_d2 = JobFactory(submission_id=sub.submission_id, file_type=database.session.query(FileType).filter_by( name='award').one(), job_type=csv_validation, number_of_errors=0, file_size=123, job_status=finished_job) job_e = JobFactory(submission_id=sub.submission_id, file_type=database.session.query(FileType).filter_by( name='executive_compensation').one(), job_type=upload, number_of_errors=0, file_size=123, job_status=finished_job) job_f = JobFactory(submission_id=sub.submission_id, file_type=database.session.query(FileType).filter_by( name='sub_award').one(), job_type=upload, number_of_errors=0, file_size=123, job_status=finished_job) job_cross_file = JobFactory(submission_id=sub.submission_id, file_type=None, job_type=validation, number_of_errors=0, file_size=123, job_status=finished_job) database.session.add_all( [job_a, job_b, job_c, job_d1, job_d2, job_e, job_f, job_cross_file]) database.session.commit() # Everything ok response = file_app.get("/v1/check_current_page/?submission_id=" + str(sub.submission_id)) response_json = json.loads(response.data.decode('UTF-8')) assert response_json['step'] == '5' job_e.job_status_id = 6 database.session.commit() # E or F failed response = file_app.get("/v1/check_current_page/?submission_id=" + str(sub.submission_id)) response_json = json.loads(response.data.decode('UTF-8')) assert response_json['step'] == '4' job_e.job_status_id = 4 job_cross_file.number_of_errors = 6 database.session.commit() # Restore job_e and create errors for cross_file response = file_app.get("/v1/check_current_page/?submission_id=" + str(sub.submission_id)) response_json = json.loads(response.data.decode('UTF-8')) assert response_json['step'] == '3' job_d1.number_of_errors = 6 database.session.commit() # D file has errors response = file_app.get("/v1/check_current_page/?submission_id=" + str(sub.submission_id)) response_json = json.loads(response.data.decode('UTF-8')) assert response_json['step'] == '2' job_c.number_of_errors = 6 database.session.commit() # Fail C file validation response = file_app.get("/v1/check_current_page/?submission_id=" + str(sub.submission_id)) response_json = json.loads(response.data.decode('UTF-8')) assert response_json['step'] == '1' job_cross_file.job_status = waiting job_d1.number_of_errors = 0 database.session.commit() # E and F generated with C file errors response = file_app.get("/v1/check_current_page/?submission_id=" + str(sub.submission_id)) response_json = json.loads(response.data.decode('UTF-8')) assert response_json['step'] == '1'
def test_current_page(file_app, database): """Test the route to check what the current progress of the submission is at the correct page """ cgac = CGACFactory() user = UserFactory.with_cgacs(cgac) user.user_id = 1 user.name = 'Oliver Queen' user.website_admin = True database.session.add(user) database.session.commit() g.user = user sub = SubmissionFactory(user_id=1, cgac_code=cgac.cgac_code) database.session.add(sub) database.session.commit() csv_validation = JOB_TYPE_DICT['csv_record_validation'] upload = JOB_TYPE_DICT['file_upload'] validation = JOB_TYPE_DICT['validation'] finished_job = JOB_STATUS_DICT['finished'] waiting = JOB_STATUS_DICT['waiting'] job_a = JobFactory(submission_id=sub.submission_id, file_type_id=FILE_TYPE_DICT['appropriations'], job_type_id=csv_validation, number_of_errors=0, file_size=123, job_status_id=finished_job) job_b = JobFactory(submission_id=sub.submission_id, file_type_id=FILE_TYPE_DICT['program_activity'], job_type_id=csv_validation, number_of_errors=0, file_size=123, job_status_id=finished_job) job_c = JobFactory(submission_id=sub.submission_id, file_type_id=FILE_TYPE_DICT['award_financial'], job_type_id=csv_validation, number_of_errors=0, file_size=123, job_status_id=finished_job) job_d1 = JobFactory(submission_id=sub.submission_id, file_type_id=FILE_TYPE_DICT['award_procurement'], job_type_id=csv_validation, number_of_errors=0, file_size=123, job_status_id=finished_job) job_d2 = JobFactory(submission_id=sub.submission_id, file_type_id=FILE_TYPE_DICT['award'], job_type_id=csv_validation, number_of_errors=0, file_size=123, job_status_id=finished_job) job_e = JobFactory(submission_id=sub.submission_id, file_type_id=FILE_TYPE_DICT['executive_compensation'], job_type_id=upload, number_of_errors=0, file_size=123, job_status_id=finished_job) job_f = JobFactory(submission_id=sub.submission_id, file_type_id=FILE_TYPE_DICT['sub_award'], job_type_id=upload, number_of_errors=0, file_size=123, job_status_id=finished_job) job_cross_file = JobFactory(submission_id=sub.submission_id, file_type_id=None, job_type_id=validation, number_of_errors=0, file_size=123, job_status_id=finished_job) database.session.add_all([job_a, job_b, job_c, job_d1, job_d2, job_e, job_f, job_cross_file]) database.session.commit() # Everything ok response = file_app.get("/v1/check_current_page/?submission_id=" + str(sub.submission_id)) response_json = json.loads(response.data.decode('UTF-8')) assert response_json['step'] == '5' job_e.job_status_id = 6 database.session.commit() # E or F failed response = file_app.get("/v1/check_current_page/?submission_id=" + str(sub.submission_id)) response_json = json.loads(response.data.decode('UTF-8')) assert response_json['step'] == '4' job_e.job_status_id = 4 job_cross_file.number_of_errors = 6 database.session.commit() # Restore job_e and create errors for cross_file response = file_app.get("/v1/check_current_page/?submission_id=" + str(sub.submission_id)) response_json = json.loads(response.data.decode('UTF-8')) assert response_json['step'] == '3' job_d1.number_of_errors = 6 database.session.commit() # D file has errors response = file_app.get("/v1/check_current_page/?submission_id=" + str(sub.submission_id)) response_json = json.loads(response.data.decode('UTF-8')) assert response_json['step'] == '2' job_c.number_of_errors = 6 database.session.commit() # Fail C file validation response = file_app.get("/v1/check_current_page/?submission_id=" + str(sub.submission_id)) response_json = json.loads(response.data.decode('UTF-8')) assert response_json['step'] == '1' job_cross_file.job_status_id = waiting job_d1.number_of_errors = 0 database.session.commit() # E and F generated with C file errors response = file_app.get("/v1/check_current_page/?submission_id=" + str(sub.submission_id)) response_json = json.loads(response.data.decode('UTF-8')) assert response_json['step'] == '1'
def test_check_submission_d_file_generation(database): """ Job statuses should return the correct status and error message to the user """ sess = database.session sub = SubmissionFactory() sess.add(sub) # D1 generation waiting to be picked up by the Validator job = JobFactory(job_status_id=JOB_STATUS_DICT['waiting'], job_type_id=JOB_TYPE_DICT['file_upload'], file_type_id=FILE_TYPE_DICT['award_procurement'], submission=sub, error_message='', filename='job_id/file.csv', original_filename='file.csv') val_job = JobFactory(job_status_id=JOB_STATUS_DICT['waiting'], job_type_id=JOB_TYPE_DICT['csv_record_validation'], file_type_id=FILE_TYPE_DICT['award_procurement'], submission=sub, error_message='', number_of_errors=0) sess.add_all([job, val_job]) sess.commit() response_dict = check_file_generation(job.job_id) assert response_dict['status'] == 'waiting' # D1 generation running in the Validator job.job_status_id = JOB_STATUS_DICT['running'] sess.commit() response_dict = check_file_generation(job.job_id) assert response_dict['status'] == 'waiting' # D1 generation with an unknown error job.job_status_id = JOB_STATUS_DICT['failed'] sess.commit() response_dict = check_file_generation(job.job_id) assert response_dict['status'] == 'failed' assert response_dict[ 'message'] == 'Upload job failed without error message' # D1 generation with a known error job.error_message = 'D1 upload error message' sess.commit() response_dict = check_file_generation(job.job_id) assert response_dict['status'] == 'failed' assert response_dict['message'] == 'D1 upload error message' # D1 generation completed by the Validator; validation waiting to be picked up job.error_message = '' job.job_status_id = JOB_STATUS_DICT['finished'] sess.commit() response_dict = check_file_generation(job.job_id) assert response_dict['status'] == 'waiting' # D1 generation completed; validation running in the Validator val_job.job_status_id = JOB_STATUS_DICT['running'] sess.commit() response_dict = check_file_generation(job.job_id) assert response_dict['status'] == 'waiting' # D1 generation completed; validation completed by the Validator val_job.job_status_id = JOB_STATUS_DICT['finished'] sess.commit() response_dict = check_file_generation(job.job_id) assert response_dict['status'] == 'finished' # D1 generation completed; validation completed by the Validator val_job.number_of_errors = 10 sess.commit() response_dict = check_file_generation(job.job_id) assert response_dict['status'] == 'failed' assert response_dict[ 'message'] == 'Validation completed but row-level errors were found' # D1 generation completed; validation with an unknown error job.error_message = '' val_job.error_message = '' val_job.job_status_id = JOB_STATUS_DICT['failed'] val_job.number_of_errors = 0 sess.commit() response_dict = check_file_generation(job.job_id) assert response_dict['status'] == 'failed' assert response_dict['message'] == 'Validation job had an internal error' # D1 generation completed; validation with a known error job.error_message = '' val_job.error_message = '' val_job.error_message = 'D1 upload error message' sess.commit() response_dict = check_file_generation(job.job_id) assert response_dict['status'] == 'failed' assert response_dict['message'] == 'D1 upload error message' # D1 generation completed; validation with an unknown error job.error_message = '' val_job.error_message = '' val_job.job_status_id = JOB_STATUS_DICT['invalid'] sess.commit() response_dict = check_file_generation(job.job_id) assert response_dict['status'] == 'failed' assert response_dict['message'] == 'Generated file had file-level errors'