def test_check_detached_d_file_generation(database):
""" Job statuses should return the correct status and error message to the user """
sess = database.session
# Detached D2 generation waiting to be picked up by the Validator
job = JobFactory(
job_status=sess.query(JobStatus).filter_by(name='waiting').one(),
job_type=sess.query(JobType).filter_by(name='file_upload').one(),
file_type=sess.query(FileType).filter_by(name='award').one(),
error_message='', filename='job_id/file.csv', original_filename='file.csv'
)
sess.add(job)
sess.commit()
response_dict = file_generation_handler.check_file_generation(job.job_id)
assert response_dict['status'] == 'waiting'
# Detached D2 generation running in the Validator
job.job_status = sess.query(JobStatus).filter_by(name='running').one()
sess.commit()
response_dict = file_generation_handler.check_file_generation(job.job_id)
assert response_dict['status'] == 'waiting'
# Detached D2 generation completed by the Validator
job.job_status = sess.query(JobStatus).filter_by(name='finished').one()
sess.commit()
response_dict = file_generation_handler.check_file_generation(job.job_id)
assert response_dict['status'] == 'finished'
assert response_dict['message'] == ''
# Detached D2 generation with an unknown error
job.job_status = sess.query(JobStatus).filter_by(name='failed').one()
sess.commit()
response_dict = file_generation_handler.check_file_generation(job.job_id)
assert response_dict['status'] == 'failed'
assert response_dict['message'] == 'Upload job failed without error message'
# Detached D2 generation with a known error
job.error_message = 'FABS upload error message'
sess.commit()
response_dict = file_generation_handler.check_file_generation(job.job_id)
assert response_dict['status'] == 'failed'
assert response_dict['message'] == 'FABS upload error message'
def test_current_page(file_app, database, user_constants, job_constants,
monkeypatch):
"""Test the route to check what the current progress of the submission is at
the correct page
"""
cgac = CGACFactory()
user = UserFactory.with_cgacs(cgac)
user.user_id = 1
user.name = 'Oliver Queen'
user.website_admin = True
database.session.add(user)
database.session.commit()
g.user = user
sub = SubmissionFactory(user_id=1, cgac_code=cgac.cgac_code)
database.session.add(sub)
csv_validation = database.session.query(JobType).filter_by(
name='csv_record_validation').one()
upload = database.session.query(JobType).filter_by(
name='file_upload').one()
validation = database.session.query(JobType).filter_by(
name='validation').one()
finished_job = database.session.query(JobStatus).filter_by(
name='finished').one()
waiting = database.session.query(JobStatus).filter_by(name='waiting').one()
job_a = JobFactory(submission_id=sub.submission_id,
file_type=database.session.query(FileType).filter_by(
name='appropriations').one(),
job_type=csv_validation,
number_of_errors=0,
file_size=123,
job_status=finished_job)
job_b = JobFactory(submission_id=sub.submission_id,
file_type=database.session.query(FileType).filter_by(
name='program_activity').one(),
job_type=csv_validation,
number_of_errors=0,
file_size=123,
job_status=finished_job)
job_c = JobFactory(submission_id=sub.submission_id,
file_type=database.session.query(FileType).filter_by(
name='award_financial').one(),
job_type=csv_validation,
number_of_errors=0,
file_size=123,
job_status=finished_job)
job_d1 = JobFactory(submission_id=sub.submission_id,
file_type=database.session.query(FileType).filter_by(
name='award_procurement').one(),
job_type=csv_validation,
number_of_errors=0,
file_size=123,
job_status=finished_job)
job_d2 = JobFactory(submission_id=sub.submission_id,
file_type=database.session.query(FileType).filter_by(
name='award').one(),
job_type=csv_validation,
number_of_errors=0,
file_size=123,
job_status=finished_job)
job_e = JobFactory(submission_id=sub.submission_id,
file_type=database.session.query(FileType).filter_by(
name='executive_compensation').one(),
job_type=upload,
number_of_errors=0,
file_size=123,
job_status=finished_job)
job_f = JobFactory(submission_id=sub.submission_id,
file_type=database.session.query(FileType).filter_by(
name='sub_award').one(),
job_type=upload,
number_of_errors=0,
file_size=123,
job_status=finished_job)
job_cross_file = JobFactory(submission_id=sub.submission_id,
file_type=None,
job_type=validation,
number_of_errors=0,
file_size=123,
job_status=finished_job)
database.session.add_all(
[job_a, job_b, job_c, job_d1, job_d2, job_e, job_f, job_cross_file])
database.session.commit()
# Everything ok
response = file_app.get("/v1/check_current_page/?submission_id=" +
str(sub.submission_id))
response_json = json.loads(response.data.decode('UTF-8'))
assert response_json['step'] == '5'
job_e.job_status_id = 6
database.session.commit()
# E or F failed
response = file_app.get("/v1/check_current_page/?submission_id=" +
str(sub.submission_id))
response_json = json.loads(response.data.decode('UTF-8'))
assert response_json['step'] == '4'
job_e.job_status_id = 4
job_cross_file.number_of_errors = 6
database.session.commit()
# Restore job_e and create errors for cross_file
response = file_app.get("/v1/check_current_page/?submission_id=" +
str(sub.submission_id))
response_json = json.loads(response.data.decode('UTF-8'))
assert response_json['step'] == '3'
job_d1.number_of_errors = 6
database.session.commit()
# D file has errors
response = file_app.get("/v1/check_current_page/?submission_id=" +
str(sub.submission_id))
response_json = json.loads(response.data.decode('UTF-8'))
assert response_json['step'] == '2'
job_c.number_of_errors = 6
database.session.commit()
# Fail C file validation
response = file_app.get("/v1/check_current_page/?submission_id=" +
str(sub.submission_id))
response_json = json.loads(response.data.decode('UTF-8'))
assert response_json['step'] == '1'
job_cross_file.job_status = waiting
job_d1.number_of_errors = 0
database.session.commit()
# E and F generated with C file errors
response = file_app.get("/v1/check_current_page/?submission_id=" +
str(sub.submission_id))
response_json = json.loads(response.data.decode('UTF-8'))
assert response_json['step'] == '1'
def test_check_submission_d_file_generation(database):
""" Job statuses should return the correct status and error message to the user """
sess = database.session
sub = SubmissionFactory()
sess.add(sub)
# D1 generation waiting to be picked up by the Validator
job = JobFactory(
job_status=sess.query(JobStatus).filter_by(name='waiting').one(),
job_type=sess.query(JobType).filter_by(name='file_upload').one(),
file_type=sess.query(FileType).filter_by(name='award_procurement').one(),
submission=sub, error_message='', filename='job_id/file.csv', original_filename='file.csv'
)
val_job = JobFactory(
job_status=sess.query(JobStatus).filter_by(name='waiting').one(),
job_type=sess.query(JobType).filter_by(name='csv_record_validation').one(),
file_type=sess.query(FileType).filter_by(name='award_procurement').one(),
submission=sub, error_message='', number_of_errors=0,
)
sess.add_all([job, val_job])
sess.commit()
response_dict = file_generation_handler.check_file_generation(job.job_id)
assert response_dict['status'] == 'waiting'
# D1 generation running in the Validator
job.job_status = sess.query(JobStatus).filter_by(name='running').one()
sess.commit()
response_dict = file_generation_handler.check_file_generation(job.job_id)
assert response_dict['status'] == 'waiting'
# D1 generation with an unknown error
job.job_status = sess.query(JobStatus).filter_by(name='failed').one()
sess.commit()
response_dict = file_generation_handler.check_file_generation(job.job_id)
assert response_dict['status'] == 'failed'
assert response_dict['message'] == 'Upload job failed without error message'
# D1 generation with a known error
job.error_message = 'D1 upload error message'
sess.commit()
response_dict = file_generation_handler.check_file_generation(job.job_id)
assert response_dict['status'] == 'failed'
assert response_dict['message'] == 'D1 upload error message'
# D1 generation completed by the Validator; validation waiting to be picked up
job.error_message = ''
job.job_status = sess.query(JobStatus).filter_by(name='finished').one()
sess.commit()
response_dict = file_generation_handler.check_file_generation(job.job_id)
assert response_dict['status'] == 'waiting'
# D1 generation completed; validation running in the Validator
val_job.job_status = sess.query(JobStatus).filter_by(name='running').one()
sess.commit()
response_dict = file_generation_handler.check_file_generation(job.job_id)
assert response_dict['status'] == 'waiting'
# D1 generation completed; validation completed by the Validator
val_job.job_status = sess.query(JobStatus).filter_by(name='finished').one()
sess.commit()
response_dict = file_generation_handler.check_file_generation(job.job_id)
assert response_dict['status'] == 'finished'
# D1 generation completed; validation completed by the Validator
val_job.number_of_errors = 10
sess.commit()
response_dict = file_generation_handler.check_file_generation(job.job_id)
assert response_dict['status'] == 'failed'
assert response_dict['message'] == 'Validation completed but row-level errors were found'
# D1 generation completed; validation with an unknown error
job.error_message = ''
val_job.error_message = ''
val_job.job_status = sess.query(JobStatus).filter_by(name='failed').one()
val_job.number_of_errors = 0
sess.commit()
response_dict = file_generation_handler.check_file_generation(job.job_id)
assert response_dict['status'] == 'failed'
assert response_dict['message'] == 'Validation job had an internal error'
# D1 generation completed; validation with a known error
job.error_message = ''
val_job.error_message = ''
val_job.error_message = 'D1 upload error message'
sess.commit()
response_dict = file_generation_handler.check_file_generation(job.job_id)
assert response_dict['status'] == 'failed'
assert response_dict['message'] == 'D1 upload error message'
# D1 generation completed; validation with an unknown error
job.error_message = ''
val_job.error_message = ''
val_job.job_status = sess.query(JobStatus).filter_by(name='invalid').one()
sess.commit()
response_dict = file_generation_handler.check_file_generation(job.job_id)
assert response_dict['status'] == 'failed'
assert response_dict['message'] == 'Generated file had file-level errors'
def test_current_page(file_app, database, user_constants, job_constants, monkeypatch):
"""Test the route to check what the current progress of the submission is at
the correct page
"""
cgac = CGACFactory()
user = UserFactory.with_cgacs(cgac)
user.user_id = 1
user.name = 'Oliver Queen'
user.website_admin = True
database.session.add(user)
database.session.commit()
g.user = user
sub = SubmissionFactory(user_id=1, cgac_code=cgac.cgac_code)
database.session.add(sub)
csv_validation = database.session.query(JobType).filter_by(name='csv_record_validation').one()
upload = database.session.query(JobType).filter_by(name='file_upload').one()
validation = database.session.query(JobType).filter_by(name='validation').one()
finished_job = database.session.query(JobStatus).filter_by(name='finished').one()
waiting = database.session.query(JobStatus).filter_by(name='waiting').one()
job_a = JobFactory(submission_id=sub.submission_id, file_type=database.session.query(FileType)
.filter_by(name='appropriations').one(), job_type=csv_validation, number_of_errors=0,
file_size=123, job_status=finished_job)
job_b = JobFactory(submission_id=sub.submission_id, file_type=database.session.query(FileType)
.filter_by(name='program_activity').one(), job_type=csv_validation, number_of_errors=0,
file_size=123, job_status=finished_job)
job_c = JobFactory(submission_id=sub.submission_id, file_type=database.session.query(FileType)
.filter_by(name='award_financial').one(), job_type=csv_validation, number_of_errors=0,
file_size=123, job_status=finished_job)
job_d1 = JobFactory(submission_id=sub.submission_id, file_type=database.session.query(FileType)
.filter_by(name='award_procurement').one(), job_type=csv_validation, number_of_errors=0,
file_size=123, job_status=finished_job)
job_d2 = JobFactory(submission_id=sub.submission_id, file_type=database.session.query(FileType)
.filter_by(name='award').one(), job_type=csv_validation, number_of_errors=0, file_size=123,
job_status=finished_job)
job_e = JobFactory(submission_id=sub.submission_id, file_type=database.session.query(FileType)
.filter_by(name='executive_compensation').one(), job_type=upload, number_of_errors=0,
file_size=123, job_status=finished_job)
job_f = JobFactory(submission_id=sub.submission_id, file_type=database.session.query(FileType)
.filter_by(name='sub_award').one(), job_type=upload, number_of_errors=0, file_size=123,
job_status=finished_job)
job_cross_file = JobFactory(submission_id=sub.submission_id, file_type=None, job_type=validation,
number_of_errors=0, file_size=123, job_status=finished_job)
database.session.add_all([job_a, job_b, job_c, job_d1, job_d2, job_e, job_f, job_cross_file])
database.session.commit()
# Everything ok
response = file_app.get("/v1/check_current_page/?submission_id=" + str(sub.submission_id))
response_json = json.loads(response.data.decode('UTF-8'))
assert response_json['step'] == '5'
job_e.job_status_id = 6
database.session.commit()
# E or F failed
response = file_app.get("/v1/check_current_page/?submission_id=" + str(sub.submission_id))
response_json = json.loads(response.data.decode('UTF-8'))
assert response_json['step'] == '4'
job_e.job_status_id = 4
job_cross_file.number_of_errors = 6
database.session.commit()
# Restore job_e and create errors for cross_file
response = file_app.get("/v1/check_current_page/?submission_id=" + str(sub.submission_id))
response_json = json.loads(response.data.decode('UTF-8'))
assert response_json['step'] == '3'
job_d1.number_of_errors = 6
database.session.commit()
# D file has errors
response = file_app.get("/v1/check_current_page/?submission_id=" + str(sub.submission_id))
response_json = json.loads(response.data.decode('UTF-8'))
assert response_json['step'] == '2'
job_c.number_of_errors = 6
database.session.commit()
# Fail C file validation
response = file_app.get("/v1/check_current_page/?submission_id=" + str(sub.submission_id))
response_json = json.loads(response.data.decode('UTF-8'))
assert response_json['step'] == '1'
job_cross_file.job_status = waiting
job_d1.number_of_errors = 0
database.session.commit()
# E and F generated with C file errors
response = file_app.get("/v1/check_current_page/?submission_id=" + str(sub.submission_id))
response_json = json.loads(response.data.decode('UTF-8'))
assert response_json['step'] == '1'
