def setUpClass(cls):
"""Set up class-wide resources like submissions and jobs."""
super(UserTests, cls).setUpClass()
# Add submissions to one of the users
jobDb = cls.jobTracker
# Delete existing submissions for approved user
jobDb.deleteSubmissionsForUserId(cls.approved_user_id)
for i in range(0,5):
sub = Submission(user_id = cls.approved_user_id)
jobDb.session.add(sub)
jobDb.session.commit()
if i == 0:
cls.submission_id = sub.submission_id
# Add submissions for agency user
jobDb.deleteSubmissionsForUserId(cls.agency_user_id)
for i in range(0,6):
sub = Submission(user_id = cls.agency_user_id)
sub.cgac_code = "SYS"
jobDb.session.add(sub)
jobDb.session.commit()
# Add job to first submission
job = Job(submission_id=cls.submission_id, job_status_id=3, job_type_id=1, file_type_id=1)
jobDb.session.add(job)
jobDb.session.commit()
cls.uploadId = job.job_id
def setUpClass(cls):
"""Set up class-wide resources like submissions and jobs."""
super(UserTests, cls).setUpClass()
with create_app().app_context():
sess = GlobalDB.db().session
# Add submissions for agency user
sess.query(Submission).filter(Submission.user_id == cls.agency_user_id).delete()
sess.commit()
for i in range(0, 6):
sub = Submission(user_id=cls.agency_user_id)
sub.reporting_start_date = datetime(2015, 10, 1)
sub.reporting_end_date = datetime(2015, 12, 31)
sub.cgac_code = cls.admin_cgac_code
sess.add(sub)
sess.commit()
if i == 0:
cls.submission_id = sub.submission_id
# Add job to first submission
job = Job(
submission_id=cls.submission_id,
job_status_id=JOB_STATUS_DICT['running'],
job_type_id=JOB_TYPE_DICT['file_upload'],
file_type_id=FILE_TYPE_DICT['appropriations']
)
sess.add(job)
sess.commit()
cls.uploadId = job.job_id
def setUpClass(cls):
"""Set up class-wide resources like submissions and jobs."""
super(UserTests, cls).setUpClass()
with create_app().app_context():
sess = GlobalDB.db().session
# Add submissions for agency user
sess.query(Submission).filter(Submission.user_id == cls.agency_user_id).delete()
sess.commit()
for i in range(0, 6):
sub = Submission(user_id=cls.agency_user_id)
sub.reporting_start_date = datetime(2015, 10, 1)
sub.reporting_end_date = datetime(2015, 12, 31)
sub.cgac_code = cls.admin_cgac_code
sess.add(sub)
sess.commit()
if i == 0:
cls.submission_id = sub.submission_id
# Add job to first submission
job = Job(
submission_id=cls.submission_id,
job_status_id=JOB_STATUS_DICT['running'],
job_type_id=JOB_TYPE_DICT['file_upload'],
file_type_id=FILE_TYPE_DICT['appropriations']
)
sess.add(job)
sess.commit()
cls.uploadId = job.job_id
def createSubmission(self, userId, requestDict):
""" Create a new submission
Arguments:
userId: User to associate with this submission
requestDict: Dictionary of keys provided in request, may contain "existing_submission_id", "agency_name", "reporting_period_start_date", "reporting_period_end_date"
Returns:
submission ID
"""
# submissionValues is a dictionary with keys determined by JobHandler.metaDataFieldMap, and existingId is the existing submission ID if it exists
submissionValues,existingId = self.loadSubmitParams(requestDict)
# Create submission entry
if existingId is None:
submission = Submission(datetime_utc = datetime.utcnow(), **submissionValues)
submission.user_id = userId
self.session.add(submission)
else:
submissionQuery = self.session.query(Submission).filter(Submission.submission_id == existingId)
submission = self.runUniqueQuery(submissionQuery,"No submission found with provided ID", "Multiple submissions found with provided ID")
#if "reporting_start_date" in submissionValues:
# submission.reporting_start_date = submissionValues["reporting_start_date"]
for key in submissionValues:
# Update existing submission with any values provided
#submission.__dict__[key] = submissionValues[key]
setattr(submission,key,submissionValues[key])
self.session.commit()
self.session.commit()
# Calling submission_id to force query to load this
return submission.submission_id
def insertSubmission(jobTracker,
submission_user_id,
submission=None,
cgac_code=None,
startDate=None,
endDate=None,
is_quarter=False):
"""Insert one submission into job tracker and get submission ID back."""
if submission:
sub = Submission(
submission_id=submission,
datetime_utc=datetime.utcnow(),
user_id=submission_user_id,
cgac_code=cgac_code,
reporting_start_date=JobHandler.createDate(startDate),
reporting_end_date=JobHandler.createDate(endDate),
is_quarter_format=is_quarter)
else:
sub = Submission(
datetime_utc=datetime.utcnow(),
user_id=submission_user_id,
cgac_code=cgac_code,
reporting_start_date=JobHandler.createDate(startDate),
reporting_end_date=JobHandler.createDate(endDate),
is_quarter_format=is_quarter)
jobTracker.session.add(sub)
jobTracker.session.commit()
return sub.submission_id
def createSubmission(self, userId, requestDict):
""" Create a new submission
Arguments:
userId: User to associate with this submission
requestDict: Dictionary of keys provided in request, may contain "existing_submission_id", "agency_name", "reporting_period_start_date", "reporting_period_end_date"
Returns:
submission ID
"""
# submissionValues is a dictionary with keys determined by JobHandler.metaDataFieldMap, and existingId is the existing submission ID if it exists
submissionValues, existingId = self.loadSubmitParams(requestDict)
# Create submission entry
if existingId is None:
submission = Submission(datetime_utc=datetime.utcnow(),
**submissionValues)
submission.user_id = userId
self.session.add(submission)
else:
submissionQuery = self.session.query(Submission).filter(
Submission.submission_id == existingId)
submission = self.runUniqueQuery(
submissionQuery, "No submission found with provided ID",
"Multiple submissions found with provided ID")
#if "reporting_start_date" in submissionValues:
# submission.reporting_start_date = submissionValues["reporting_start_date"]
for key in submissionValues:
# Update existing submission with any values provided
#submission.__dict__[key] = submissionValues[key]
setattr(submission, key, submissionValues[key])
self.session.commit()
self.session.commit()
# Calling submission_id to force query to load this
return submission.submission_id
def create_submission(user_id, submission_values, existing_submission):
""" Create a new submission
Arguments:
user_id: user to associate with this submission
submission_values: metadata about the submission
existing_submission: id of existing submission (blank for new submissions)
Returns:
submission object
"""
if existing_submission is None:
submission = Submission(created_at=datetime.utcnow(), **submission_values)
submission.user_id = user_id
submission.publish_status_id = PUBLISH_STATUS_DICT['unpublished']
else:
submission = existing_submission
if submission.publish_status_id == PUBLISH_STATUS_DICT['published']:
submission.publish_status_id = PUBLISH_STATUS_DICT['updated']
# submission is being updated, so turn off publishable flag
submission.publishable = False
for key in submission_values:
# update existing submission with any values provided
setattr(submission, key, submission_values[key])
return submission
def setUpClass(cls):
"""Set up class-wide resources like submissions and jobs."""
super(UserTests, cls).setUpClass()
# Add submissions to one of the users
jobDb = cls.jobTracker
# Delete existing submissions for approved user
jobDb.deleteSubmissionsForUserId(cls.approved_user_id)
for i in range(0, 5):
sub = Submission(user_id=cls.approved_user_id)
jobDb.session.add(sub)
jobDb.session.commit()
if i == 0:
cls.submission_id = sub.submission_id
# Add submissions for agency user
jobDb.deleteSubmissionsForUserId(cls.agency_user_id)
for i in range(0, 6):
sub = Submission(user_id=cls.agency_user_id)
sub.cgac_code = "SYS"
jobDb.session.add(sub)
jobDb.session.commit()
# Add job to first submission
job = Job(submission_id=cls.submission_id,
job_status_id=3,
job_type_id=1,
file_type_id=1)
jobDb.session.add(job)
jobDb.session.commit()
cls.uploadId = job.job_id
def insertSubmission(jobTracker, submission_user_id, submission=None, agency = None, startDate = None, endDate = None):
"""Insert one submission into job tracker and get submission ID back."""
if submission:
sub = Submission(submission_id=submission,
datetime_utc=datetime.utcnow(), user_id=submission_user_id, agency_name = agency, reporting_start_date = JobHandler.createDate(startDate), reporting_end_date = JobHandler.createDate(endDate))
else:
sub = Submission(datetime_utc=datetime.utcnow(), user_id=submission_user_id, agency_name = agency, reporting_start_date = JobHandler.createDate(startDate), reporting_end_date = JobHandler.createDate(endDate))
jobTracker.session.add(sub)
jobTracker.session.commit()
return sub.submission_id
def insert_submission(sess,
submission_user_id,
cgac_code=None,
start_date=None,
end_date=None,
is_quarter=False,
publish_status_id=1,
d2_submission=True):
""" Insert one submission into job tracker and get submission ID back.
Args:
sess: the current session
submission_user_id: the ID of the user that owns the submission
cgac_code: cgac code of the agency that the submission is to, default None
start_date: the reporting start date of the submission, default None
end_date: the reporting start date of the submission, default None
is_quarter: boolean indicating if the submission is a quarterly (false is monthly), default False
publish_status_id: the publish status of the submission, default 1 (unpublished)
d2_submission: boolean indicating if the submission is FABS or DABS (true for FABS), default True
Returns:
the submission ID of the created submission
"""
sub = Submission(
created_at=datetime.utcnow(),
user_id=submission_user_id,
cgac_code=cgac_code,
reporting_start_date=datetime.strptime(start_date, '%m/%Y'),
reporting_end_date=datetime.strptime(end_date, '%m/%Y'),
is_quarter_format=is_quarter,
publish_status_id=publish_status_id,
d2_submission=d2_submission)
sess.add(sub)
sess.commit()
return sub.submission_id
def error_rows(rule_file,
staging_db,
submission=None,
models=None,
assert_num=None):
"""Insert the models into the database, then run the rule SQL against
those models. Return the resulting (invalid) rows"""
if submission is None:
submission = Submission(user_id=None,
reporting_start_date=datetime(2015, 10, 1),
reporting_end_date=datetime(2015, 10, 31))
if models is None:
models = []
submission_id = insert_submission(staging_db, submission)
sql = SQLLoader.readSqlStr(rule_file).format(submission_id)
for model in models:
model.submission_id = submission_id
staging_db.session.add(model)
staging_db.session.commit()
result = staging_db.connection.execute(sql).fetchall()
if assert_num is not None:
assert (len(result) == assert_num)
# clean up
for model in models:
staging_db.session.delete(model)
staging_db.session.commit()
return result
def insert_submission(sess,
submission_user_id,
cgac_code=None,
start_date=None,
end_date=None,
is_quarter=False,
number_of_errors=0,
publish_status_id=1,
is_fabs=False,
updated_at=datetime.utcnow()):
"""Insert one submission into job tracker and get submission ID back."""
publishable = True if number_of_errors == 0 else False
end_date = datetime.strptime(end_date, '%m/%Y')
end_date = datetime.strptime(
str(end_date.year) + '/' + str(end_date.month) + '/' +
str(calendar.monthrange(end_date.year, end_date.month)[1]),
'%Y/%m/%d').date()
sub = Submission(created_at=datetime.utcnow(),
updated_at=updated_at,
user_id=submission_user_id,
cgac_code=cgac_code,
reporting_start_date=datetime.strptime(
start_date, '%m/%Y'),
reporting_end_date=end_date,
is_quarter_format=is_quarter,
number_of_errors=number_of_errors,
publish_status_id=publish_status_id,
publishable=publishable,
d2_submission=is_fabs)
sess.add(sub)
sess.commit()
return sub.submission_id
def insertSubmission(cls, sess, userId=None, reporting_end_date=None):
"""Insert submission and return id."""
reporting_start_date = cls.SUBMISSION_START_DEFAULT
if reporting_end_date is None:
reporting_end_date = cls.SUBMISSION_END_DEFAULT
sub = Submission(datetime_utc=datetime.utcnow(),
user_id=userId,
reporting_start_date=reporting_start_date,
reporting_end_date=reporting_end_date)
sess.add(sub)
sess.commit()
return sub.submission_id
def create_submission(user_id, submission_values, existing_submission):
""" Create a new submission
Arguments:
user_id: user to associate with this submission
submission_values: metadata about the submission
existing_submission_id: id of existing submission (blank for new submissions)
Returns:
submission object
"""
sess = GlobalDB.db().session
if existing_submission is None:
submission = Submission(datetime_utc = datetime.utcnow(), **submission_values)
submission.user_id = user_id
submission.publish_status_id = PUBLISH_STATUS_DICT['unpublished']
sess.add(submission)
else:
submission = existing_submission
if submission.publish_status_id == PUBLISH_STATUS_DICT['published']:
submission.publish_status_id = PUBLISH_STATUS_DICT['updated']
# submission is being updated, so turn off publishable flag
submission.publishable = False
for key in submission_values:
# update existing submission with any values provided
setattr(submission, key, submission_values[key])
sess.commit()
return submission
def create_submission(user_id, submission_values, existing_submission):
""" Create a new submission if one doesn't exist, otherwise update the existing one
Args:
user_id: user to associate with this submission
submission_values: metadata about the submission
existing_submission: id of existing submission (blank for new submissions)
Returns:
submission object
"""
if existing_submission is None:
submission = Submission(created_at=datetime.utcnow(),
**submission_values)
submission.user_id = user_id
submission.publish_status_id = PUBLISH_STATUS_DICT['unpublished']
else:
submission = existing_submission
if submission.publish_status_id == PUBLISH_STATUS_DICT['published']:
submission.publish_status_id = PUBLISH_STATUS_DICT['updated']
# submission is being updated, so turn off publishable flag
submission.publishable = False
for key in submission_values:
# update existing submission with any values provided
setattr(submission, key, submission_values[key])
return submission
def setUpClass(cls):
"""Set up class-wide resources like submissions and jobs."""
super(UserTests, cls).setUpClass()
with createApp().app_context():
sess = GlobalDB.db().session
# Add submissions to one of the users
# Delete existing submissions for approved user
sess.query(Submission).filter(
Submission.user_id == cls.approved_user_id).delete()
sess.commit()
for i in range(0, 5):
sub = Submission(user_id=cls.approved_user_id)
sub.reporting_start_date = datetime(2015, 10, 1)
sub.reporting_end_date = datetime(2015, 12, 31)
sess.add(sub)
sess.commit()
# Add submissions for agency user
sess.query(Submission).filter(
Submission.user_id == cls.agency_user_id).delete()
sess.commit()
for i in range(0, 6):
sub = Submission(user_id=cls.agency_user_id)
sub.reporting_start_date = datetime(2015, 10, 1)
sub.reporting_end_date = datetime(2015, 12, 31)
sub.cgac_code = "SYS"
sess.add(sub)
sess.commit()
if i == 0:
cls.submission_id = sub.submission_id
# Add job to first submission
job = Job(submission_id=cls.submission_id,
job_status_id=cls.jobStatusDict['running'],
job_type_id=cls.jobTypeDict['file_upload'],
file_type_id=cls.fileTypeDict['appropriations'])
sess.add(job)
sess.commit()
cls.uploadId = job.job_id
def insert_submission(cls, sess, user_id=None, reporting_end_date=None):
"""Insert submission and return id."""
if reporting_end_date is None:
reporting_start_date = cls.SUBMISSION_START_DEFAULT
reporting_end_date = cls.SUBMISSION_END_DEFAULT
else:
reporting_start_date = reporting_end_date - timedelta(days=30)
sub = Submission(created_at=datetime.utcnow(),
user_id=user_id,
reporting_start_date=reporting_start_date,
reporting_end_date=reporting_end_date)
sess.add(sub)
sess.commit()
return sub.submission_id
def test_revert_submission_fabs_submission(database):
""" Tests reverting an updated DABS certification failure for FABS submission """
sess = database.session
sub = Submission(d2_submission=True)
sess.add(sub)
sess.commit()
file_handler = fileHandler.FileHandler({}, is_local=True)
with pytest.raises(ResponseException) as resp_except:
revert_to_certified(sub, file_handler)
assert resp_except.value.status == 400
assert str(resp_except.value) == 'Submission must be a DABS submission.'
def test_revert_submission_not_updated_submission(database):
""" Tests reverting an updated DABS certification failure for non-updated submission """
sess = database.session
sub1 = Submission(publish_status_id=PUBLISH_STATUS_DICT['published'], d2_submission=False)
sub2 = Submission(publish_status_id=PUBLISH_STATUS_DICT['unpublished'], d2_submission=False)
sess.add_all([sub1, sub2])
sess.commit()
file_handler = fileHandler.FileHandler({}, is_local=True)
# Certified submission
with pytest.raises(ResponseException) as resp_except:
revert_to_certified(sub1, file_handler)
assert resp_except.value.status == 400
assert str(resp_except.value) == 'Submission has not been certified or has not been updated since certification.'
# Uncertified submission
with pytest.raises(ResponseException) as resp_except:
revert_to_certified(sub2, file_handler)
assert resp_except.value.status == 400
assert str(resp_except.value) == 'Submission has not been certified or has not been updated since certification.'
def insert_submission(sess,
submission_user_id,
cgac_code=None,
frec_code=None,
start_date=None,
end_date=None,
is_quarter=False,
number_of_errors=0,
publish_status_id=1,
is_fabs=False,
updated_at=datetime.utcnow(),
test_submission=False,
published_submission_ids=[],
certified=False,
reporting_fiscal_year=None,
reporting_fisacal_period=None):
"""Insert one submission into job tracker and get submission ID back."""
publishable = True if number_of_errors == 0 else False
if start_date is not None:
start_date = datetime.strptime(start_date, '%m/%Y')
if end_date is not None:
end_date = datetime.strptime(end_date, '%m/%Y')
end_date = datetime.strptime(
str(end_date.year) + '/' + str(end_date.month) + '/' +
str(calendar.monthrange(end_date.year, end_date.month)[1]),
'%Y/%m/%d').date()
sub = Submission(created_at=datetime.utcnow(),
updated_at=updated_at,
user_id=submission_user_id,
cgac_code=cgac_code,
frec_code=frec_code,
reporting_start_date=start_date,
reporting_end_date=end_date,
reporting_fiscal_year=reporting_fiscal_year,
reporting_fiscal_period=reporting_fisacal_period,
is_quarter_format=is_quarter,
number_of_errors=number_of_errors,
publish_status_id=publish_status_id,
publishable=publishable,
is_fabs=is_fabs,
test_submission=test_submission,
published_submission_ids=published_submission_ids,
certified=certified)
sess.add(sub)
sess.commit()
return sub.submission_id
def insertSubmission(sess,
submission_user_id,
cgac_code=None,
startDate=None,
endDate=None,
is_quarter=False,
number_of_errors=0):
"""Insert one submission into job tracker and get submission ID back."""
sub = Submission(
datetime_utc=datetime.utcnow(),
user_id=submission_user_id,
cgac_code=cgac_code,
reporting_start_date=datetime.strptime(startDate, '%m/%Y'),
reporting_end_date=datetime.strptime(endDate, '%m/%Y'),
is_quarter_format=is_quarter,
number_of_errors=number_of_errors)
sess.add(sub)
sess.commit()
return sub.submission_id
def insert_submission(sess,
submission_user_id,
cgac_code=None,
start_date=None,
end_date=None,
is_quarter=False,
publish_status_id=1,
d2_submission=True):
"""Insert one submission into job tracker and get submission ID back."""
sub = Submission(
created_at=datetime.utcnow(),
user_id=submission_user_id,
cgac_code=cgac_code,
reporting_start_date=datetime.strptime(start_date, '%m/%Y'),
reporting_end_date=datetime.strptime(end_date, '%m/%Y'),
is_quarter_format=is_quarter,
publish_status_id=publish_status_id,
d2_submission=d2_submission)
sess.add(sub)
sess.commit()
return sub.submission_id
def insertSubmission(jobTracker, userId):
"""Insert submission into job tracker and return submission ID"""
sub = Submission(datetime_utc=datetime.utcnow(), user_id=userId)
jobTracker.session.add(sub)
jobTracker.session.commit()
return sub.submission_id
def test_revert_submission(database, monkeypatch):
""" Tests reverting an updated DABS certification """
sess = database.session
sub = Submission(publish_status_id=PUBLISH_STATUS_DICT['updated'], is_quarter_format=True, d2_submission=False,
publishable=False, number_of_errors=20, number_of_warnings=15)
sess.add(sub)
sess.commit()
job = Job(submission_id=sub.submission_id, job_status_id=JOB_STATUS_DICT['finished'],
job_type_id=JOB_TYPE_DICT['csv_record_validation'], file_type_id=FILE_TYPE_DICT['appropriations'],
number_of_warnings=0, number_of_errors=10, filename='new/test/file.csv', number_of_rows=5,
number_of_rows_valid=0)
cert_history = CertifyHistory(submission_id=sub.submission_id)
sess.add_all([job, cert_history])
sess.commit()
cert_approp = CertifiedAppropriation(submission_id=sub.submission_id, job_id=job.job_id, row_number=1,
spending_authority_from_of_cpe=2, tas='test')
approp = Appropriation(submission_id=sub.submission_id, job_id=job.job_id, row_number=1,
spending_authority_from_of_cpe=15, tas='test')
cert_files = CertifiedFilesHistory(certify_history_id=cert_history.certify_history_id,
submission_id=sub.submission_id, filename='old/test/file2.csv',
file_type_id=FILE_TYPE_DICT['appropriations'], warning_filename='a/warning.csv')
cert_meta1 = CertifiedErrorMetadata(job_id=job.job_id, file_type_id=FILE_TYPE_DICT['appropriations'],
target_file_type_id=None, occurrences=15)
cert_meta2 = CertifiedErrorMetadata(job_id=job.job_id, file_type_id=FILE_TYPE_DICT['appropriations'],
target_file_type_id=None, occurrences=10)
file_entry = File(file_id=FILE_TYPE_DICT['appropriations'], job_id=job.job_id,
file_status_id=FILE_STATUS_DICT['incomplete'], headers_missing='something')
sess.add_all([cert_approp, approp, cert_files, cert_meta1, cert_meta2, file_entry])
sess.commit()
file_handler = fileHandler.FileHandler({}, is_local=True)
monkeypatch.setattr(file_handler, 'revert_certified_error_files', Mock())
revert_to_certified(sub, file_handler)
# Test that certified data is moved back
approp_query = sess.query(Appropriation).filter_by(submission_id=sub.submission_id).all()
assert len(approp_query) == 1
assert approp_query[0].spending_authority_from_of_cpe == 2
# Test that the job got updated
job_query = sess.query(Job).filter_by(submission_id=sub.submission_id).all()
assert len(job_query) == 1
assert job_query[0].filename == CONFIG_BROKER['broker_files'] + 'file2.csv'
assert job_query[0].number_of_warnings == 25
assert job_query[0].number_of_errors == 0
assert job_query[0].number_of_rows == 2
assert job_query[0].number_of_rows_valid == 1
# Test that File got updated
file_query = sess.query(File).filter_by(job_id=job.job_id).all()
assert len(file_query) == 1
assert file_query[0].headers_missing is None
assert file_query[0].file_status_id == FILE_STATUS_DICT['complete']
# Make sure submission got updated
sub_query = sess.query(Submission).filter_by(submission_id=sub.submission_id).all()
assert len(sub_query) == 1
assert sub_query[0].publishable is True
assert sub_query[0].number_of_errors == 0
assert sub_query[0].number_of_warnings == 25
