def test_check_submission_d_file_generation(database):
""" Job statuses should return the correct status and error message to the user """
sess = database.session
sub = SubmissionFactory()
sess.add(sub)
# D1 generation waiting to be picked up by the Validator
job = JobFactory(job_status_id=JOB_STATUS_DICT['waiting'], job_type_id=JOB_TYPE_DICT['file_upload'],
file_type_id=FILE_TYPE_DICT['award_procurement'], submission=sub, error_message='',
filename='job_id/file.csv', original_filename='file.csv')
val_job = JobFactory(job_status_id=JOB_STATUS_DICT['waiting'], job_type_id=JOB_TYPE_DICT['csv_record_validation'],
file_type_id=FILE_TYPE_DICT['award_procurement'], submission=sub, error_message='',
number_of_errors=0)
sess.add_all([job, val_job])
sess.commit()
response_dict = check_file_generation(job.job_id)
assert response_dict['status'] == 'waiting'
# D1 generation running in the Validator
job.job_status_id = JOB_STATUS_DICT['running']
sess.commit()
response_dict = check_file_generation(job.job_id)
assert response_dict['status'] == 'waiting'
# D1 generation with an unknown error
job.job_status_id = JOB_STATUS_DICT['failed']
sess.commit()
response_dict = check_file_generation(job.job_id)
assert response_dict['status'] == 'failed'
assert response_dict['message'] == 'Upload job failed without error message'
# D1 generation with a known error
job.error_message = 'D1 upload error message'
sess.commit()
response_dict = check_file_generation(job.job_id)
assert response_dict['status'] == 'failed'
assert response_dict['message'] == 'D1 upload error message'
# D1 generation completed by the Validator; validation waiting to be picked up
job.error_message = ''
job.job_status_id = JOB_STATUS_DICT['finished']
sess.commit()
response_dict = check_file_generation(job.job_id)
assert response_dict['status'] == 'waiting'
# D1 generation completed; validation running in the Validator
val_job.job_status_id = JOB_STATUS_DICT['running']
sess.commit()
response_dict = check_file_generation(job.job_id)
assert response_dict['status'] == 'waiting'
# D1 generation completed; validation completed by the Validator
val_job.job_status_id = JOB_STATUS_DICT['finished']
sess.commit()
response_dict = check_file_generation(job.job_id)
assert response_dict['status'] == 'finished'
# D1 generation completed; validation completed by the Validator
val_job.number_of_errors = 10
sess.commit()
response_dict = check_file_generation(job.job_id)
assert response_dict['status'] == 'failed'
assert response_dict['message'] == 'Validation completed but row-level errors were found'
# D1 generation completed; validation with an unknown error
job.error_message = ''
val_job.error_message = ''
val_job.job_status_id = JOB_STATUS_DICT['failed']
val_job.number_of_errors = 0
sess.commit()
response_dict = check_file_generation(job.job_id)
assert response_dict['status'] == 'failed'
assert response_dict['message'] == 'Validation job had an internal error'
# D1 generation completed; validation with a known error
job.error_message = ''
val_job.error_message = ''
val_job.error_message = 'D1 upload error message'
sess.commit()
response_dict = check_file_generation(job.job_id)
assert response_dict['status'] == 'failed'
assert response_dict['message'] == 'D1 upload error message'
# D1 generation completed; validation with an unknown error
job.error_message = ''
val_job.error_message = ''
val_job.job_status_id = JOB_STATUS_DICT['invalid']
sess.commit()
response_dict = check_file_generation(job.job_id)
assert response_dict['status'] == 'failed'
assert response_dict['message'] == 'Generated file had file-level errors'
def test_current_page(file_app, database):
"""Test the route to check what the current progress of the submission is at
the correct page
"""
cgac = CGACFactory()
user = UserFactory.with_cgacs(cgac)
user.user_id = 1
user.name = 'Oliver Queen'
user.website_admin = True
database.session.add(user)
database.session.commit()
g.user = user
sub = SubmissionFactory(user_id=1, cgac_code=cgac.cgac_code)
database.session.add(sub)
database.session.commit()
csv_validation = JOB_TYPE_DICT['csv_record_validation']
upload = JOB_TYPE_DICT['file_upload']
validation = JOB_TYPE_DICT['validation']
finished_job = JOB_STATUS_DICT['finished']
waiting = JOB_STATUS_DICT['waiting']
job_a = JobFactory(submission_id=sub.submission_id, file_type_id=FILE_TYPE_DICT['appropriations'],
job_type_id=csv_validation, number_of_errors=0, file_size=123, job_status_id=finished_job)
job_b = JobFactory(submission_id=sub.submission_id, file_type_id=FILE_TYPE_DICT['program_activity'],
job_type_id=csv_validation, number_of_errors=0, file_size=123, job_status_id=finished_job)
job_c = JobFactory(submission_id=sub.submission_id, file_type_id=FILE_TYPE_DICT['award_financial'],
job_type_id=csv_validation, number_of_errors=0, file_size=123, job_status_id=finished_job)
job_d1 = JobFactory(submission_id=sub.submission_id, file_type_id=FILE_TYPE_DICT['award_procurement'],
job_type_id=csv_validation, number_of_errors=0, file_size=123, job_status_id=finished_job)
job_d2 = JobFactory(submission_id=sub.submission_id, file_type_id=FILE_TYPE_DICT['award'],
job_type_id=csv_validation, number_of_errors=0, file_size=123, job_status_id=finished_job)
job_e = JobFactory(submission_id=sub.submission_id, file_type_id=FILE_TYPE_DICT['executive_compensation'],
job_type_id=upload, number_of_errors=0, file_size=123, job_status_id=finished_job)
job_f = JobFactory(submission_id=sub.submission_id, file_type_id=FILE_TYPE_DICT['sub_award'], job_type_id=upload,
number_of_errors=0, file_size=123, job_status_id=finished_job)
job_cross_file = JobFactory(submission_id=sub.submission_id, file_type_id=None, job_type_id=validation,
number_of_errors=0, file_size=123, job_status_id=finished_job)
database.session.add_all([job_a, job_b, job_c, job_d1, job_d2, job_e, job_f, job_cross_file])
database.session.commit()
# Everything ok
response = file_app.get("/v1/check_current_page/?submission_id=" + str(sub.submission_id))
response_json = json.loads(response.data.decode('UTF-8'))
assert response_json['step'] == '5'
job_e.job_status_id = 6
database.session.commit()
# E or F failed
response = file_app.get("/v1/check_current_page/?submission_id=" + str(sub.submission_id))
response_json = json.loads(response.data.decode('UTF-8'))
assert response_json['step'] == '4'
job_e.job_status_id = 4
job_cross_file.number_of_errors = 6
database.session.commit()
# Restore job_e and create errors for cross_file
response = file_app.get("/v1/check_current_page/?submission_id=" + str(sub.submission_id))
response_json = json.loads(response.data.decode('UTF-8'))
assert response_json['step'] == '3'
job_d1.number_of_errors = 6
database.session.commit()
# D file has errors
response = file_app.get("/v1/check_current_page/?submission_id=" + str(sub.submission_id))
response_json = json.loads(response.data.decode('UTF-8'))
assert response_json['step'] == '2'
job_c.number_of_errors = 6
database.session.commit()
# Fail C file validation
response = file_app.get("/v1/check_current_page/?submission_id=" + str(sub.submission_id))
response_json = json.loads(response.data.decode('UTF-8'))
assert response_json['step'] == '1'
job_cross_file.job_status_id = waiting
job_d1.number_of_errors = 0
database.session.commit()
# E and F generated with C file errors
response = file_app.get("/v1/check_current_page/?submission_id=" + str(sub.submission_id))
response_json = json.loads(response.data.decode('UTF-8'))
assert response_json['step'] == '1'
def test_current_page(file_app, database, user_constants, job_constants,
monkeypatch):
"""Test the route to check what the current progress of the submission is at
the correct page
"""
cgac = CGACFactory()
user = UserFactory.with_cgacs(cgac)
user.user_id = 1
user.name = 'Oliver Queen'
user.website_admin = True
database.session.add(user)
database.session.commit()
g.user = user
sub = SubmissionFactory(user_id=1, cgac_code=cgac.cgac_code)
database.session.add(sub)
csv_validation = database.session.query(JobType).filter_by(
name='csv_record_validation').one()
upload = database.session.query(JobType).filter_by(
name='file_upload').one()
validation = database.session.query(JobType).filter_by(
name='validation').one()
finished_job = database.session.query(JobStatus).filter_by(
name='finished').one()
waiting = database.session.query(JobStatus).filter_by(name='waiting').one()
job_a = JobFactory(submission_id=sub.submission_id,
file_type=database.session.query(FileType).filter_by(
name='appropriations').one(),
job_type=csv_validation,
number_of_errors=0,
file_size=123,
job_status=finished_job)
job_b = JobFactory(submission_id=sub.submission_id,
file_type=database.session.query(FileType).filter_by(
name='program_activity').one(),
job_type=csv_validation,
number_of_errors=0,
file_size=123,
job_status=finished_job)
job_c = JobFactory(submission_id=sub.submission_id,
file_type=database.session.query(FileType).filter_by(
name='award_financial').one(),
job_type=csv_validation,
number_of_errors=0,
file_size=123,
job_status=finished_job)
job_d1 = JobFactory(submission_id=sub.submission_id,
file_type=database.session.query(FileType).filter_by(
name='award_procurement').one(),
job_type=csv_validation,
number_of_errors=0,
file_size=123,
job_status=finished_job)
job_d2 = JobFactory(submission_id=sub.submission_id,
file_type=database.session.query(FileType).filter_by(
name='award').one(),
job_type=csv_validation,
number_of_errors=0,
file_size=123,
job_status=finished_job)
job_e = JobFactory(submission_id=sub.submission_id,
file_type=database.session.query(FileType).filter_by(
name='executive_compensation').one(),
job_type=upload,
number_of_errors=0,
file_size=123,
job_status=finished_job)
job_f = JobFactory(submission_id=sub.submission_id,
file_type=database.session.query(FileType).filter_by(
name='sub_award').one(),
job_type=upload,
number_of_errors=0,
file_size=123,
job_status=finished_job)
job_cross_file = JobFactory(submission_id=sub.submission_id,
file_type=None,
job_type=validation,
number_of_errors=0,
file_size=123,
job_status=finished_job)
database.session.add_all(
[job_a, job_b, job_c, job_d1, job_d2, job_e, job_f, job_cross_file])
database.session.commit()
# Everything ok
response = file_app.get("/v1/check_current_page/?submission_id=" +
str(sub.submission_id))
response_json = json.loads(response.data.decode('UTF-8'))
assert response_json['step'] == '5'
job_e.job_status_id = 6
database.session.commit()
# E or F failed
response = file_app.get("/v1/check_current_page/?submission_id=" +
str(sub.submission_id))
response_json = json.loads(response.data.decode('UTF-8'))
assert response_json['step'] == '4'
job_e.job_status_id = 4
job_cross_file.number_of_errors = 6
database.session.commit()
# Restore job_e and create errors for cross_file
response = file_app.get("/v1/check_current_page/?submission_id=" +
str(sub.submission_id))
response_json = json.loads(response.data.decode('UTF-8'))
assert response_json['step'] == '3'
job_d1.number_of_errors = 6
database.session.commit()
# D file has errors
response = file_app.get("/v1/check_current_page/?submission_id=" +
str(sub.submission_id))
response_json = json.loads(response.data.decode('UTF-8'))
assert response_json['step'] == '2'
job_c.number_of_errors = 6
database.session.commit()
# Fail C file validation
response = file_app.get("/v1/check_current_page/?submission_id=" +
str(sub.submission_id))
response_json = json.loads(response.data.decode('UTF-8'))
assert response_json['step'] == '1'
job_cross_file.job_status = waiting
job_d1.number_of_errors = 0
database.session.commit()
# E and F generated with C file errors
response = file_app.get("/v1/check_current_page/?submission_id=" +
str(sub.submission_id))
response_json = json.loads(response.data.decode('UTF-8'))
assert response_json['step'] == '1'
def test_check_submission_d_file_generation(database):
""" Job statuses should return the correct status and error message to the user """
sess = database.session
sub = SubmissionFactory()
sess.add(sub)
# D1 generation waiting to be picked up by the Validator
job = JobFactory(
job_status=sess.query(JobStatus).filter_by(name='waiting').one(),
job_type=sess.query(JobType).filter_by(name='file_upload').one(),
file_type=sess.query(FileType).filter_by(name='award_procurement').one(),
submission=sub, error_message='', filename='job_id/file.csv', original_filename='file.csv'
)
val_job = JobFactory(
job_status=sess.query(JobStatus).filter_by(name='waiting').one(),
job_type=sess.query(JobType).filter_by(name='csv_record_validation').one(),
file_type=sess.query(FileType).filter_by(name='award_procurement').one(),
submission=sub, error_message='', number_of_errors=0,
)
sess.add_all([job, val_job])
sess.commit()
response_dict = file_generation_handler.check_file_generation(job.job_id)
assert response_dict['status'] == 'waiting'
# D1 generation running in the Validator
job.job_status = sess.query(JobStatus).filter_by(name='running').one()
sess.commit()
response_dict = file_generation_handler.check_file_generation(job.job_id)
assert response_dict['status'] == 'waiting'
# D1 generation with an unknown error
job.job_status = sess.query(JobStatus).filter_by(name='failed').one()
sess.commit()
response_dict = file_generation_handler.check_file_generation(job.job_id)
assert response_dict['status'] == 'failed'
assert response_dict['message'] == 'Upload job failed without error message'
# D1 generation with a known error
job.error_message = 'D1 upload error message'
sess.commit()
response_dict = file_generation_handler.check_file_generation(job.job_id)
assert response_dict['status'] == 'failed'
assert response_dict['message'] == 'D1 upload error message'
# D1 generation completed by the Validator; validation waiting to be picked up
job.error_message = ''
job.job_status = sess.query(JobStatus).filter_by(name='finished').one()
sess.commit()
response_dict = file_generation_handler.check_file_generation(job.job_id)
assert response_dict['status'] == 'waiting'
# D1 generation completed; validation running in the Validator
val_job.job_status = sess.query(JobStatus).filter_by(name='running').one()
sess.commit()
response_dict = file_generation_handler.check_file_generation(job.job_id)
assert response_dict['status'] == 'waiting'
# D1 generation completed; validation completed by the Validator
val_job.job_status = sess.query(JobStatus).filter_by(name='finished').one()
sess.commit()
response_dict = file_generation_handler.check_file_generation(job.job_id)
assert response_dict['status'] == 'finished'
# D1 generation completed; validation completed by the Validator
val_job.number_of_errors = 10
sess.commit()
response_dict = file_generation_handler.check_file_generation(job.job_id)
assert response_dict['status'] == 'failed'
assert response_dict['message'] == 'Validation completed but row-level errors were found'
# D1 generation completed; validation with an unknown error
job.error_message = ''
val_job.error_message = ''
val_job.job_status = sess.query(JobStatus).filter_by(name='failed').one()
val_job.number_of_errors = 0
sess.commit()
response_dict = file_generation_handler.check_file_generation(job.job_id)
assert response_dict['status'] == 'failed'
assert response_dict['message'] == 'Validation job had an internal error'
# D1 generation completed; validation with a known error
job.error_message = ''
val_job.error_message = ''
val_job.error_message = 'D1 upload error message'
sess.commit()
response_dict = file_generation_handler.check_file_generation(job.job_id)
assert response_dict['status'] == 'failed'
assert response_dict['message'] == 'D1 upload error message'
# D1 generation completed; validation with an unknown error
job.error_message = ''
val_job.error_message = ''
val_job.job_status = sess.query(JobStatus).filter_by(name='invalid').one()
sess.commit()
response_dict = file_generation_handler.check_file_generation(job.job_id)
assert response_dict['status'] == 'failed'
assert response_dict['message'] == 'Generated file had file-level errors'
