def setup_file_generation_submission(cls, sess, submission_id=None):
"""Create jobs for D, E, and F files."""
submission_id = cls.generation_submission_id if not submission_id else submission_id
submission = sess.query(Submission).filter(Submission.submission_id == submission_id).one()
# Create D1 jobs ready for generation route to be called
insert_job(
sess,
FILE_TYPE_DICT['award_procurement'],
JOB_STATUS_DICT['ready'],
JOB_TYPE_DICT['file_upload'],
submission.submission_id
)
award_roc_val_job = insert_job(
sess,
FILE_TYPE_DICT['award_procurement'],
JOB_STATUS_DICT['waiting'],
JOB_TYPE_DICT['csv_record_validation'],
submission.submission_id
)
# Create E and F jobs ready for check route
exec_comp_job = insert_job(
sess,
FILE_TYPE_DICT['executive_compensation'],
JOB_STATUS_DICT['finished'],
JOB_TYPE_DICT['file_upload'],
submission.submission_id
)
sub_award_job = insert_job(
sess,
FILE_TYPE_DICT['sub_award'],
JOB_STATUS_DICT['invalid'],
JOB_TYPE_DICT['file_upload'],
submission.submission_id
)
sub_award_job.error_message = "File was invalid"
# Create D2 jobs
insert_job(
sess,
FILE_TYPE_DICT['award'],
JOB_STATUS_DICT['finished'],
JOB_TYPE_DICT['file_upload'],
submission.submission_id
)
insert_job(
sess,
FILE_TYPE_DICT['award'],
JOB_STATUS_DICT['invalid'],
JOB_TYPE_DICT['csv_record_validation'],
submission.submission_id
)
# Create dependency
exec_comp_dep = JobDependency(
job_id=exec_comp_job.job_id,
prerequisite_id=award_roc_val_job.job_id
)
sess.add(exec_comp_dep)
sess.commit()
def setup_file_generation_submission(cls, sess, submission_id=None):
"""Create jobs for D, E, and F files."""
submission_id = cls.generation_submission_id if not submission_id else submission_id
submission = sess.query(Submission).filter(Submission.submission_id == submission_id).one()
# Create D1 jobs ready for generation route to be called
insert_job(
sess,
FILE_TYPE_DICT['award_procurement'],
JOB_STATUS_DICT['ready'],
JOB_TYPE_DICT['file_upload'],
submission.submission_id
)
award_roc_val_job = insert_job(
sess,
FILE_TYPE_DICT['award_procurement'],
JOB_STATUS_DICT['waiting'],
JOB_TYPE_DICT['csv_record_validation'],
submission.submission_id
)
# Create E and F jobs ready for check route
exec_comp_job = insert_job(
sess,
FILE_TYPE_DICT['executive_compensation'],
JOB_STATUS_DICT['finished'],
JOB_TYPE_DICT['file_upload'],
submission.submission_id
)
sub_award_job = insert_job(
sess,
FILE_TYPE_DICT['sub_award'],
JOB_STATUS_DICT['invalid'],
JOB_TYPE_DICT['file_upload'],
submission.submission_id
)
sub_award_job.error_message = "File was invalid"
# Create D2 jobs
insert_job(
sess,
FILE_TYPE_DICT['award'],
JOB_STATUS_DICT['finished'],
JOB_TYPE_DICT['file_upload'],
submission.submission_id
)
insert_job(
sess,
FILE_TYPE_DICT['award'],
JOB_STATUS_DICT['invalid'],
JOB_TYPE_DICT['csv_record_validation'],
submission.submission_id
)
# Create dependency
exec_comp_dep = JobDependency(
job_id=exec_comp_job.job_id,
prerequisite_id=award_roc_val_job.job_id
)
sess.add(exec_comp_dep)
sess.commit()
def setUpClass(cls):
"""Set up class-wide resources (test data)"""
super(ListSubmissionTests, cls).setUpClass()
# TODO: refactor into a pytest fixture
with create_app().app_context():
# get an admin and non-admin user
sess = GlobalDB.db().session
cls.session = sess
admin_user = sess.query(User).filter(
User.email == cls.test_users['admin_user']).one()
cls.admin_user_id = admin_user.user_id
other_user = sess.query(User).filter(
User.email == cls.test_users['agency_user']).one()
cls.other_user_id = other_user.user_id
# set up submissions for dabs
cls.non_admin_dabs_sub_id = insert_submission(
sess,
cls.other_user_id,
cgac_code="SYS",
start_date="10/2015",
end_date="12/2015",
is_quarter=True,
is_fabs=False,
publish_status_id=PUBLISH_STATUS_DICT['unpublished'],
updated_at='01/01/2010')
cls.admin_dabs_sub_id = insert_submission(
sess,
cls.admin_user_id,
cgac_code="000",
start_date="10/2015",
end_date="12/2015",
is_quarter=True,
is_fabs=False,
publish_status_id=PUBLISH_STATUS_DICT['unpublished'],
updated_at='01/01/2012')
cls.certified_dabs_sub_id = insert_submission(
sess,
cls.admin_user_id,
cgac_code="SYS",
start_date="10/2015",
end_date="12/2015",
is_quarter=True,
is_fabs=False,
publish_status_id=PUBLISH_STATUS_DICT['published'])
# Add a couple jobs for dabs files
insert_job(sess,
FILE_TYPE_DICT['appropriations'],
FILE_STATUS_DICT['complete'],
JOB_TYPE_DICT['file_upload'],
cls.non_admin_dabs_sub_id,
filename='/path/to/test/file_1.csv',
file_size=123,
num_rows=3)
insert_job(sess,
FILE_TYPE_DICT['award'],
FILE_STATUS_DICT['complete'],
JOB_TYPE_DICT['file_upload'],
cls.non_admin_dabs_sub_id,
filename='/path/to/test/file_2.csv',
file_size=123,
num_rows=3)
insert_job(sess,
FILE_TYPE_DICT['award'],
FILE_STATUS_DICT['complete'],
JOB_TYPE_DICT['file_upload'],
cls.certified_dabs_sub_id,
filename='/path/to/test/file_part_2.csv',
file_size=123,
num_rows=3)
# set up submissions for fabs
cls.non_admin_fabs_sub_id = insert_submission(
sess,
cls.admin_user_id,
cgac_code="SYS",
start_date="10/2015",
end_date="12/2015",
is_fabs=True,
publish_status_id=PUBLISH_STATUS_DICT['unpublished'])
cls.admin_fabs_sub_id = insert_submission(
sess,
cls.other_user_id,
cgac_code="000",
start_date="10/2015",
end_date="12/2015",
is_fabs=True,
publish_status_id=PUBLISH_STATUS_DICT['unpublished'])
cls.published_fabs_sub_id = insert_submission(
sess,
cls.other_user_id,
cgac_code="000",
start_date="10/2015",
end_date="12/2015",
is_fabs=True,
publish_status_id=PUBLISH_STATUS_DICT['published'])
# Add a job for a FABS submission
insert_job(sess,
FILE_TYPE_DICT['fabs'],
FILE_STATUS_DICT['complete'],
JOB_TYPE_DICT['file_upload'],
cls.admin_fabs_sub_id,
filename=str(cls.admin_fabs_sub_id) + '/test_file.csv',
file_size=123,
num_rows=3)
def setUpClass(cls):
"""Set up class-wide resources (test data)"""
super(ListSubmissionTests, cls).setUpClass()
# TODO: refactor into a pytest fixture
with create_app().app_context():
# get an admin and non-admin user
sess = GlobalDB.db().session
cls.session = sess
admin_user = sess.query(User).filter(
User.email == cls.test_users['admin_user']).one()
cls.admin_user_id = admin_user.user_id
other_user = sess.query(User).filter(
User.email == cls.test_users['agency_user']).one()
cls.other_user_id = other_user.user_id
# set up submissions for dabs
cls.non_admin_dabs_sub_id = insert_submission(
sess,
cls.other_user_id,
cgac_code='SYS',
start_date='10/2015',
end_date='12/2015',
is_quarter=True,
is_fabs=False,
publish_status_id=PUBLISH_STATUS_DICT['unpublished'],
updated_at='01/01/2010')
cls.admin_dabs_sub_id = insert_submission(
sess,
cls.admin_user_id,
cgac_code='000',
start_date='10/2015',
end_date='12/2015',
is_quarter=True,
is_fabs=False,
publish_status_id=PUBLISH_STATUS_DICT['unpublished'],
updated_at='01/01/2012')
cls.test_sub_id = insert_submission(
sess,
cls.admin_user_id,
cgac_code='SYS',
start_date='10/2015',
end_date='12/2015',
is_quarter=True,
is_fabs=False,
publish_status_id=PUBLISH_STATUS_DICT['unpublished'],
updated_at='01/02/2012',
test_submission=True)
# This is the min date, but the date everything should be using is the one in the job (MAX_UPDATED_AT)
cls.certified_dabs_sub_id = insert_submission(
sess,
cls.admin_user_id,
cgac_code='SYS',
start_date='10/2015',
end_date='12/2015',
is_quarter=True,
is_fabs=False,
publish_status_id=PUBLISH_STATUS_DICT['published'],
updated_at='01/01/2000')
# Add a couple jobs for dabs files, make sure the updated at is the same as or earlier than the one on
# the submission itself
insert_job(sess,
FILE_TYPE_DICT['appropriations'],
FILE_STATUS_DICT['complete'],
JOB_TYPE_DICT['file_upload'],
cls.non_admin_dabs_sub_id,
filename='/path/to/test/file_1.csv',
file_size=123,
num_rows=3,
updated_at='01/01/2009')
insert_job(sess,
FILE_TYPE_DICT['award'],
FILE_STATUS_DICT['complete'],
JOB_TYPE_DICT['file_upload'],
cls.non_admin_dabs_sub_id,
filename='/path/to/test/file_2.csv',
file_size=123,
num_rows=3,
updated_at='01/01/2009')
# Min updated at date
insert_job(sess,
FILE_TYPE_DICT['award'],
FILE_STATUS_DICT['complete'],
JOB_TYPE_DICT['file_upload'],
cls.certified_dabs_sub_id,
filename='/path/to/test/file_part_2.csv',
file_size=123,
num_rows=3,
updated_at=cls.MAX_UPDATED_AT)
# set up submissions for fabs
cls.non_admin_fabs_sub_id = insert_submission(
sess,
cls.admin_user_id,
cgac_code='SYS',
start_date='10/2015',
end_date='12/2015',
is_fabs=True,
publish_status_id=PUBLISH_STATUS_DICT['unpublished'],
updated_at='01/01/2016')
# This is the min date, but the date everything should be using is the one in the job (MAX_UPDATED_AT)
cls.admin_fabs_sub_id = insert_submission(
sess,
cls.other_user_id,
cgac_code='000',
start_date='10/2015',
end_date='12/2015',
is_fabs=True,
publish_status_id=PUBLISH_STATUS_DICT['unpublished'],
updated_at='01/01/2000')
cls.published_fabs_sub_id = insert_submission(
sess,
cls.other_user_id,
cgac_code='000',
start_date='10/2015',
end_date='12/2015',
is_fabs=True,
publish_status_id=PUBLISH_STATUS_DICT['published'],
updated_at='01/02/2000')
# Add a job for a FABS submission
insert_job(sess,
FILE_TYPE_DICT['fabs'],
FILE_STATUS_DICT['complete'],
JOB_TYPE_DICT['file_upload'],
cls.admin_fabs_sub_id,
filename=str(cls.admin_fabs_sub_id) + '/test_file.csv',
file_size=123,
num_rows=3,
updated_at=cls.MAX_UPDATED_AT)
def setUpClass(cls):
""" Set up class-wide resources (test data) """
super(ErrorWarningTests, cls).setUpClass()
logging.getLogger('dataactcore').setLevel(logging.ERROR)
logging.getLogger('dataactvalidator').setLevel(logging.ERROR)
with create_app().app_context():
# get the submission test users
sess = GlobalDB.db().session
cls.session = sess
# set up default e-mails for tests
admin_user = sess.query(User).filter(
User.email == cls.test_users['admin_user']).one()
cls.validator = ValidationManager(
directory=CONFIG_SERVICES['error_report_path'])
# Just have one valid submission and then keep on reloading files
cls.submission_id = insert_submission(sess,
admin_user.user_id,
cgac_code='SYS',
start_date='01/2001',
end_date='03/2001',
is_quarter=True)
cls.submission = sess.query(Submission).filter_by(
submission_id=cls.submission_id).one()
cls.val_job = insert_job(
cls.session,
FILE_TYPE_DICT['appropriations'],
JOB_STATUS_DICT['ready'],
JOB_TYPE_DICT['csv_record_validation'],
cls.submission_id,
filename=JOB_TYPE_DICT['csv_record_validation'])
cls.original_reports = set(
os.listdir(CONFIG_SERVICES['error_report_path']))
# adding TAS to ensure valid file is valid
tas1 = TASFactory(account_num=1,
allocation_transfer_agency='019',
agency_identifier='072',
beginning_period_of_availa=None,
ending_period_of_availabil=None,
availability_type_code='X',
main_account_code='0306',
sub_account_code='000',
internal_start_date='01-01-2000')
tas2 = TASFactory(account_num=2,
allocation_transfer_agency=None,
agency_identifier='019',
beginning_period_of_availa='2016',
ending_period_of_availabil='2016',
availability_type_code=None,
main_account_code='0113',
sub_account_code='000',
internal_start_date='01-01-2000')
tas3 = TASFactory(account_num=3,
allocation_transfer_agency=None,
agency_identifier='028',
beginning_period_of_availa=None,
ending_period_of_availabil=None,
availability_type_code='X',
main_account_code='0406',
sub_account_code='000',
internal_start_date='01-01-2000')
tas4 = TASFactory(account_num=4,
allocation_transfer_agency=None,
agency_identifier='028',
beginning_period_of_availa='2010',
ending_period_of_availabil='2011',
availability_type_code=None,
main_account_code='0406',
sub_account_code='000',
internal_start_date='01-01-2000')
tas5 = TASFactory(account_num=5,
allocation_transfer_agency='069',
agency_identifier='013',
beginning_period_of_availa=None,
ending_period_of_availabil=None,
availability_type_code='X',
main_account_code='2050',
sub_account_code='005',
internal_start_date='01-01-2000')
tas6 = TASFactory(account_num=6,
allocation_transfer_agency='028',
agency_identifier='028',
beginning_period_of_availa=None,
ending_period_of_availabil=None,
availability_type_code='X',
main_account_code='8007',
sub_account_code='000',
internal_start_date='01-01-2000')
tas7 = TASFactory(account_num=7,
allocation_transfer_agency=None,
agency_identifier='049',
beginning_period_of_availa=None,
ending_period_of_availabil=None,
availability_type_code='X',
main_account_code='0100',
sub_account_code='000',
internal_start_date='01-01-2000')
tas8 = TASFactory(account_num=8,
allocation_transfer_agency=None,
agency_identifier='049',
beginning_period_of_availa='2010',
ending_period_of_availabil='2011',
availability_type_code=None,
main_account_code='0100',
sub_account_code='000',
internal_start_date='01-01-2000')
tas9 = TASFactory(account_num=9,
allocation_transfer_agency=None,
agency_identifier='049',
beginning_period_of_availa='2014',
ending_period_of_availabil='2015',
availability_type_code=None,
main_account_code='0100',
sub_account_code='000',
internal_start_date='01-01-2000')
tas10 = TASFactory(account_num=10,
allocation_transfer_agency=None,
agency_identifier='049',
beginning_period_of_availa='2015',
ending_period_of_availabil='2016',
availability_type_code=None,
main_account_code='0100',
sub_account_code='000',
internal_start_date='01-01-2000')
sess.add_all(
[tas1, tas2, tas3, tas4, tas5, tas6, tas7, tas8, tas9, tas10])
# adding GTAS to ensure valid file is valid
gtas1 = SF133Factory(tas=concat_tas_dict(tas1.component_dict()),
allocation_transfer_agency='019',
agency_identifier='072',
beginning_period_of_availa=None,
line=1009,
ending_period_of_availabil=None,
availability_type_code='X',
main_account_code='0306',
sub_account_code='000',
period=6,
fiscal_year=2001)
gtas2 = SF133Factory(tas=concat_tas_dict(tas2.component_dict()),
allocation_transfer_agency=None,
agency_identifier='019',
beginning_period_of_availa='2016',
line=1009,
ending_period_of_availabil='2016',
availability_type_code=None,
main_account_code='0113',
sub_account_code='000',
period=6,
fiscal_year=2001)
gtas3 = SF133Factory(tas=concat_tas_dict(tas3.component_dict()),
allocation_transfer_agency=None,
agency_identifier='028',
beginning_period_of_availa=None,
line=1009,
ending_period_of_availabil=None,
availability_type_code='X',
main_account_code='0406',
sub_account_code='000',
period=6,
fiscal_year=2001)
gtas4 = SF133Factory(tas=concat_tas_dict(tas4.component_dict()),
allocation_transfer_agency=None,
agency_identifier='028',
beginning_period_of_availa='2010',
line=1009,
ending_period_of_availabil='2011',
availability_type_code=None,
main_account_code='0406',
sub_account_code='000',
period=6,
fiscal_year=2001)
gtas5 = SF133Factory(tas=concat_tas_dict(tas5.component_dict()),
allocation_transfer_agency='069',
agency_identifier='013',
beginning_period_of_availa=None,
line=1009,
ending_period_of_availabil=None,
availability_type_code='X',
main_account_code='2050',
sub_account_code='005',
period=6,
fiscal_year=2001)
gtas6 = SF133Factory(tas=concat_tas_dict(tas6.component_dict()),
allocation_transfer_agency='028',
agency_identifier='028',
beginning_period_of_availa=None,
line=1009,
ending_period_of_availabil=None,
availability_type_code='X',
main_account_code='8007',
sub_account_code='000',
period=6,
fiscal_year=2001)
gtas7 = SF133Factory(tas=concat_tas_dict(tas7.component_dict()),
allocation_transfer_agency=None,
agency_identifier='049',
beginning_period_of_availa=None,
line=1009,
ending_period_of_availabil=None,
availability_type_code='X',
main_account_code='0100',
sub_account_code='000',
period=6,
fiscal_year=2001)
gtas8 = SF133Factory(tas=concat_tas_dict(tas8.component_dict()),
allocation_transfer_agency=None,
agency_identifier='049',
beginning_period_of_availa='2010',
line=1009,
ending_period_of_availabil='2011',
availability_type_code=None,
main_account_code='0100',
sub_account_code='000',
period=6,
fiscal_year=2001)
gtas9 = SF133Factory(tas=concat_tas_dict(tas9.component_dict()),
allocation_transfer_agency=None,
agency_identifier='049',
beginning_period_of_availa='2014',
line=1009,
ending_period_of_availabil='2015',
availability_type_code=None,
main_account_code='0100',
sub_account_code='000',
period=6,
fiscal_year=2001)
gtas10 = SF133Factory(tas=concat_tas_dict(tas10.component_dict()),
allocation_transfer_agency=None,
agency_identifier='049',
beginning_period_of_availa='2015',
line=1009,
ending_period_of_availabil='2016',
availability_type_code=None,
main_account_code='0100',
sub_account_code='000',
period=6,
fiscal_year=2001)
sess.add_all([
gtas1, gtas2, gtas3, gtas4, gtas5, gtas6, gtas7, gtas8, gtas9,
gtas10
])
sess.commit()