def test_get_validated_untrained_count(self):
with session_scope(self.dal) as session:
insert_updated_nightly_file(session, self.predicted_nightly_data)
with session_scope(self.dal) as session:
result = get_validated_untrained_count(session)
expected = 0
self.assertEqual(result, expected)
def test_retrain_check(self):
with session_scope(self.dal) as session:
insert_updated_nightly_file(session, self.predicted_nightly_data)
with session_scope(self.dal) as session:
result = retrain_check(session)
expected = False
self.assertEqual(result, expected)
def test_retrain_check(self):
with session_scope(self.dal) as session:
insert_data(session, self.data)
with session_scope(self.dal) as session:
result = retrain_check(session)
expected = False
self.assertEqual(result, expected)
def tearDown(self):
with session_scope(self.dal) as session:
clear_data(session)
with session_scope(self.dal) as session:
session.close_all()
self.dal.drop_test_postgres_db()
self.dal = None
def test_get_trained_count(self):
with session_scope(self.dal) as session:
insert_data(session, self.data)
with session_scope(self.dal) as session:
result = get_trained_count(session)
expected = 0
self.assertEqual(result, expected)
def main():
'''
Run all of the scripts together.
'''
logger.info("Smartie is downloading the most recent nightly FBO file...")
nightly_data = fbo_nightly_scraper.get_nightly_data()
logger.info(
"Smartie is done downloading the most recent nightly FBO file!")
logger.info(
"Smartie is getting the attachments and their text from each FBO notice..."
)
fboa = get_fbo_attachments.FboAttachments(nightly_data)
updated_nightly_data = fboa.update_nightly_data()
logger.info(
"Smartie is done getting the attachments and their text from each FBO notice!"
)
logger.info("Smartie is making predictions for each notice attachment...")
predict = Predict(updated_nightly_data)
updated_nightly_data_with_predictions = predict.insert_predictions()
logger.info(
"Smartie is done making predictions for each notice attachment!")
logger.info("Smartie is inserting data into the database...")
with session_scope(dal) as session:
insert_updated_nightly_file(session,
updated_nightly_data_with_predictions)
logger.info("Smartie is done inserting data into database!")
logger.info("Smartie is performing the retrain check...")
with session_scope(dal) as session:
retrain(session)
logger.info("*" * 80)
def test_insert_data(self):
with session_scope(self.dal) as session:
insert_data(session, self.data)
result = []
with session_scope(self.dal) as session:
notices = session.query(Notice).all()
for n in notices:
notice = object_as_dict(n)
#pop the date and createdAt attributes since they're constructed programmatically
notice.pop('date')
notice.pop('createdAt')
#pop this as it'll vary
notice.pop('notice_type_id')
result.append(notice)
expected = [{
'id': 1,
'solicitation_number': 'test',
'agency': 'agency',
'notice_data': {
'url': 'url',
'naics': 'test',
'office': 'office',
'subject': 'test',
'classcod': 'test',
'setaside': 'test',
'emails': ['*****@*****.**']
},
'compliant': 0,
'feedback': None,
'history': None,
'action': None,
'updatedAt': None,
'na_flag': False
}]
self.assertCountEqual(result, expected)
def test_fetch_validated_attachments(self):
with session_scope(self.dal) as session:
insert_updated_nightly_file(session, self.predicted_nightly_data)
with session_scope(self.dal) as session:
attachments = fetch_validated_attachments(session)
result = len(attachments)
expected = 993
self.assertEqual(result, expected)
def test_fetch_notices_by_solnbr_bogus_solnbr(self):
with session_scope(self.dal) as session:
insert_data(session, self.data)
with session_scope(self.dal) as session:
notices = fetch_notices_by_solnbr('notexist', session)
result = len(notices)
expected = 0
self.assertEqual(result, expected)
def test_fetch_notices_by_solnbr_bogus_solnbr(self):
with session_scope(self.dal) as session:
insert_updated_nightly_file(session, self.predicted_nightly_data)
with session_scope(self.dal) as session:
notices = fetch_notices_by_solnbr('test123', session)
result = len(notices)
expected = 0
self.assertEqual(result, expected)
def tearDown(self):
with session_scope(self.dal) as session:
clear_data(session)
with session_scope(self.dal) as session:
session.close_all()
self.dal.drop_test_postgres_db()
self.dal = None
self.predicted_nightly_data = None
self.predicted_nightly_data_day_two = None
def test_fetch_notices_by_solnbr(self):
with session_scope(self.dal) as session:
insert_updated_nightly_file(session, self.predicted_nightly_data)
with session_scope(self.dal) as session:
notices = fetch_notices_by_solnbr(
'rfp-e-bpm-djf-18-0800-pr-0000828', session)
result = len(notices)
expected = 1
self.assertEqual(result, expected)
def test_fetch_validated_attachments(self):
with session_scope(self.dal) as session:
insert_data(session, self.data)
with session_scope(self.dal) as session:
attachments = fetch_validated_attachments(session)
result = len(attachments)
# 993 since that's how many docs were initially labeled
expected = 993
self.assertEqual(result, expected)
def test_fetch_last_score(self):
results = {'c': 'd'}
params = {'a': 'b'}
score = .99
with session_scope(self.dal) as session:
insert_model(session, results=results, params=params, score=score)
with session_scope(self.dal) as session:
score = fetch_last_score(session)
result = score
expected = .99
self.assertEqual(result, expected)
def test_insert_data_with_new_notice_type(self):
opp = self.data[0].copy()
nnt = "new notice type"
opp['notice type'] = nnt
with session_scope(self.dal) as session:
insert_data(session, [opp])
result = []
with session_scope(self.dal) as session:
notices = session.query(Notice).all()
for n in notices:
notice = object_as_dict(n)
notice_type_id = int(notice['notice_type_id'])
notice_type = fetch_notice_type_by_id(notice_type_id, session)
self.assertCountEqual(notice_type.notice_type, nnt)
def test_insert_data_into_solicitations_table(self):
with session_scope(self.dal) as session:
try:
insert_data_into_solicitations_table(
session, [mock_schematized_opp_two])
except Exception as e:
print(e)
def main(limit=None, updateOld=True, opportunity_filter_function=None, target_sol_types="o,k", skip_attachments=False, from_date = 'yesterday', to_date='yesterday'):
try:
if limit:
logger.error("Artifical limit of {} placed on the number of opportunities processed. Should not happen in production.".format(limit))
if not updateOld:
logger.error("Set to NOT update old solicitations. Should not happen in production.".format(limit))
with session_scope(dal) as session:
# make sure that the notice types are configured and committed before going further
insert_notice_types(session)
logger.info("Connecting with database at {}".format(conn_string))
logger.info("Smartie is fetching opportunties from SAM...")
data = get_opps.main(limit, opportunity_filter_function=opportunity_filter_function, target_sol_types=target_sol_types, skip_attachments=skip_attachments, from_date=from_date, to_date=to_date)
if not data:
logger.info("Smartie didn't find any opportunities!")
else:
logger.info("Smartie is done fetching opportunties from SAM!")
logger.info("Smartie is making predictions for each notice attachment...")
predict = Predict(data)
data = predict.insert_predictions()
logger.info("Smartie is done making predictions for each notice attachment!")
with session_scope(dal) as session:
if data:
# insert_data(session, data)
logger.info("Smartie is inserting data into the database...")
insert_data_into_solicitations_table(session, data)
logger.info("Smartie is done inserting data into database!")
else:
logger.error("No data to insert. Something went wrong.")
if updateOld:
update_old_solicitations(session, max_tests=10)
logger.info("Run complete without major errors.")
except Exception as e:
logger.error("Unhandled error. Data for the day may be lost.")
logger.error(f"Exception: {e}", exc_info=True)
logger.error("Unexpected error: {}".format(str(sys.exc_info()[0])))
def test_update_old_solicitations(self):
conn_string = get_db_url()
dal = DataAccessLayer(conn_string)
dal.connect()
with session_scope(dal) as session:
sam_utils.update_old_solicitations(session,
age_cutoff=365,
max_tests=5)
def test_insert_notice_types(self):
with session_scope(self.dal) as session:
insert_notice_types(session)
types = [
'Presolicitation', 'Solicitation',
'Combined Synopsis/Solicitation', 'TRAIN'
]
notice_type_ids = []
for notice_type in types:
with session_scope(self.dal) as session:
notice_type_id = session.query(NoticeType.id).filter(
NoticeType.notice_type == notice_type).first().id
notice_type_ids.append(notice_type_id)
notice_type_ids = set(notice_type_ids)
result = len(notice_type_ids)
expected = len(types)
self.assertEqual(result, expected)
def setUp(self):
self.data = [mock_data_for_db.copy()]
self.dal = DataAccessLayer(conn_string=get_db_url())
self.dal.create_test_postgres_db()
self.dal.connect()
with session_scope(self.dal) as session:
insert_notice_types(session)
self.maxDiff = None
def test_insert_model(self):
results = {'c': 'd'}
params = {'a': 'b'}
score = .99
with session_scope(self.dal) as session:
insert_model(session, results=results, params=params, score=score)
result = []
with session_scope(self.dal) as session:
models = session.query(Model).all()
for m in models:
model = object_as_dict(m)
model.pop('create_date')
result.append(model)
expected = [{
'id': 1,
'results': results,
'params': params,
'score': score
}]
self.assertCountEqual(result, expected)
def test_insert_notice_types(self):
with session_scope(self.dal) as session:
insert_notice_types(session)
notice_types = ['MOD', 'PRESOL', 'COMBINE', 'AMDCSS', 'TRAIN']
notice_type_ids = []
for notice_type in notice_types:
notice_type_id = session.query(NoticeType.id).filter(
NoticeType.notice_type == notice_type).first().id
notice_type_ids.append(notice_type_id)
notice_type_ids = set(notice_type_ids)
result = len(notice_type_ids)
expected = len(notice_types)
self.assertEqual(result, expected)
def test_insert_bad_notice(self):
call_count = 0
with session_scope(self.dal) as session:
# intentionally bad notice type
data = mock_data_for_db.copy()
data['notice type'] = "not to be found"
self.assertNotEqual(mock_data_for_db['notice type'],
data['notice type'])
logger = logging.getLogger("utils.db.db_utils")
print(logger)
with mock.patch.object(logger, 'warning', wraps=logger.warning):
insert_data(session, [data])
call_count = logger.warning.call_count
self.assertEqual(
1, call_count,
"We should get one warning when adding a notice with a new notice type."
)
def setUp(self):
self.dal = DataAccessLayer(conn_string=get_db_url())
self.dal.connect()
with session_scope(self.dal) as session:
insert_notice_types(session)
def tearDown(self):
with session_scope(self.dal) as session:
session.close_all()
self.dal = None
self.data = None
def test_insert_updated_nightly_file_day_two(self):
'''
Simulate a second batch entry with a repeating solnbr that now has attachments
'''
with session_scope(self.dal) as session:
insert_updated_nightly_file(session, self.predicted_nightly_data)
with session_scope(self.dal) as session:
insert_updated_nightly_file(session,
self.predicted_nightly_data_day_two)
result = []
with session_scope(self.dal) as session:
notices = session.query(Notice).all()
for n in notices:
notice = object_as_dict(n)
#pop the date and createdAt attributes since they're constructed programmatically
notice.pop('date')
notice.pop('createdAt')
if notice['history']:
notice['history'][0]['date'] = "test date"
result.append(notice)
expected = [{
'id': 1,
'notice_type_id': 6,
'solicitation_number': 'rfp-e-bpm-djf-18-0800-pr-0000828',
'agency': 'department of justice',
'notice_data': {
'url': 'url',
'zip': '20535',
'date': '0506',
'desc': ' link to document',
'year': '18',
'naics': '511210',
'ntype': 'combine',
'offadd': '935 pennsylvania avenue, n.w. washington dc 20535',
'office': 'federal bureau of investigation',
'popzip': '20535',
'contact':
'clark kent, contracting officer, phone 5555555555, email [email protected]',
'subject':
'enterprise business process management software tool',
'classcod': '70',
'location': 'procurement section',
'setaside': 'n/a',
'popaddress': '935 pennsylvania ave. n.w. washington, dc ',
'popcountry': 'us'
},
'compliant': 0,
'feedback': None,
'history': None,
'action': None,
'updatedAt': None
}, {
'id': 2,
'notice_type_id': 5,
'solicitation_number': 'spe4a618t934n',
'agency': 'defense logistics agency',
'notice_data': {
'url': 'test_url',
'zip': '23297',
'date': '0506',
'desc': 'test123',
'year': '18',
'naics': '334511',
'offadd': '334511',
'office': 'dla acquisition locations',
'contact': '*****@*****.**',
'subject': 'subject',
'archdate': '06132018',
'classcod': '66',
'location': 'dla aviation - bsm',
'respdate': '051418',
'setaside': 'n/a '
},
'compliant': 0,
'feedback': None,
'history': None,
'action': None,
'updatedAt': None
}, {
'id':
3,
'notice_type_id':
6,
'solicitation_number':
'spe4a618t934n',
'agency':
'defense logistics agency',
'notice_data': {
'url': 'test_url',
'zip': '23297',
'date': '0506',
'desc': 'test123',
'year': '17',
'naics': '334511',
'offadd': '334511',
'office': 'dla acquisition locations',
'contact': '*****@*****.**',
'subject': 'subject',
'archdate': '06132018',
'classcod': '66',
'location': 'dla aviation - bsm',
'respdate': '051418',
'setaside': 'n/a '
},
'compliant':
0,
'feedback':
None,
'history': [{
"date": "test date",
"user": "",
"action": "Solicitation Updated on FBO.gov",
"status": ""
}],
'action':
None,
'updatedAt':
None
}]
self.assertEqual(result, expected)
