def test_retrain_check(self):
with session_scope(self.dal) as session:
insert_updated_nightly_file(session, self.predicted_nightly_data)
with session_scope(self.dal) as session:
result = retrain_check(session)
expected = False
self.assertEqual(result, expected)
def main():
'''
Run all of the scripts together.
'''
logger.info("Smartie is downloading the most recent nightly FBO file...")
nightly_data = fbo_nightly_scraper.get_nightly_data()
logger.info(
"Smartie is done downloading the most recent nightly FBO file!")
logger.info(
"Smartie is getting the attachments and their text from each FBO notice..."
)
fboa = get_fbo_attachments.FboAttachments(nightly_data)
updated_nightly_data = fboa.update_nightly_data()
logger.info(
"Smartie is done getting the attachments and their text from each FBO notice!"
)
logger.info("Smartie is making predictions for each notice attachment...")
predict = Predict(updated_nightly_data)
updated_nightly_data_with_predictions = predict.insert_predictions()
logger.info(
"Smartie is done making predictions for each notice attachment!")
logger.info("Smartie is inserting data into the database...")
with session_scope(dal) as session:
insert_updated_nightly_file(session,
updated_nightly_data_with_predictions)
logger.info("Smartie is done inserting data into database!")
logger.info("Smartie is performing the retrain check...")
with session_scope(dal) as session:
retrain(session)
logger.info("*" * 80)
def test_get_validated_untrained_count(self):
with session_scope(self.dal) as session:
insert_updated_nightly_file(session, self.predicted_nightly_data)
with session_scope(self.dal) as session:
result = get_validated_untrained_count(session)
expected = 0
self.assertEqual(result, expected)
def test_fetch_notices_by_solnbr_bogus_solnbr(self):
with session_scope(self.dal) as session:
insert_updated_nightly_file(session, self.predicted_nightly_data)
with session_scope(self.dal) as session:
notices = fetch_notices_by_solnbr('test123', session)
result = len(notices)
expected = 0
self.assertEqual(result, expected)
def test_fetch_validated_attachments(self):
with session_scope(self.dal) as session:
insert_updated_nightly_file(session, self.predicted_nightly_data)
with session_scope(self.dal) as session:
attachments = fetch_validated_attachments(session)
result = len(attachments)
expected = 993
self.assertEqual(result, expected)
def test_fetch_notices_by_solnbr(self):
with session_scope(self.dal) as session:
insert_updated_nightly_file(session, self.predicted_nightly_data)
with session_scope(self.dal) as session:
notices = fetch_notices_by_solnbr(
'rfp-e-bpm-djf-18-0800-pr-0000828', session)
result = len(notices)
expected = 1
self.assertEqual(result, expected)
def test_insert_updated_nightly_file_day_two(self):
'''
Simulate a second batch entry with a repeating solnbr that now has attachments
'''
with session_scope(self.dal) as session:
insert_updated_nightly_file(session, self.predicted_nightly_data)
with session_scope(self.dal) as session:
insert_updated_nightly_file(session,
self.predicted_nightly_data_day_two)
result = []
with session_scope(self.dal) as session:
notices = session.query(Notice).all()
for n in notices:
notice = object_as_dict(n)
#pop the date and createdAt attributes since they're constructed programmatically
notice.pop('date')
notice.pop('createdAt')
if notice['history']:
notice['history'][0]['date'] = "test date"
result.append(notice)
expected = [{
'id': 1,
'notice_type_id': 6,
'solicitation_number': 'rfp-e-bpm-djf-18-0800-pr-0000828',
'agency': 'department of justice',
'notice_data': {
'url': 'url',
'zip': '20535',
'date': '0506',
'desc': ' link to document',
'year': '18',
'naics': '511210',
'ntype': 'combine',
'offadd': '935 pennsylvania avenue, n.w. washington dc 20535',
'office': 'federal bureau of investigation',
'popzip': '20535',
'contact':
'clark kent, contracting officer, phone 5555555555, email [email protected]',
'subject':
'enterprise business process management software tool',
'classcod': '70',
'location': 'procurement section',
'setaside': 'n/a',
'popaddress': '935 pennsylvania ave. n.w. washington, dc ',
'popcountry': 'us'
},
'compliant': 0,
'feedback': None,
'history': None,
'action': None,
'updatedAt': None
}, {
'id': 2,
'notice_type_id': 5,
'solicitation_number': 'spe4a618t934n',
'agency': 'defense logistics agency',
'notice_data': {
'url': 'test_url',
'zip': '23297',
'date': '0506',
'desc': 'test123',
'year': '18',
'naics': '334511',
'offadd': '334511',
'office': 'dla acquisition locations',
'contact': '*****@*****.**',
'subject': 'subject',
'archdate': '06132018',
'classcod': '66',
'location': 'dla aviation - bsm',
'respdate': '051418',
'setaside': 'n/a '
},
'compliant': 0,
'feedback': None,
'history': None,
'action': None,
'updatedAt': None
}, {
'id':
3,
'notice_type_id':
6,
'solicitation_number':
'spe4a618t934n',
'agency':
'defense logistics agency',
'notice_data': {
'url': 'test_url',
'zip': '23297',
'date': '0506',
'desc': 'test123',
'year': '17',
'naics': '334511',
'offadd': '334511',
'office': 'dla acquisition locations',
'contact': '*****@*****.**',
'subject': 'subject',
'archdate': '06132018',
'classcod': '66',
'location': 'dla aviation - bsm',
'respdate': '051418',
'setaside': 'n/a '
},
'compliant':
0,
'feedback':
None,
'history': [{
"date": "test date",
"user": "",
"action": "Solicitation Updated on FBO.gov",
"status": ""
}],
'action':
None,
'updatedAt':
None
}]
self.assertEqual(result, expected)
