def test_get_validated_untrained_count(self): with session_scope(self.dal) as session: insert_updated_nightly_file(session, self.predicted_nightly_data) with session_scope(self.dal) as session: result = get_validated_untrained_count(session) expected = 0 self.assertEqual(result, expected)
def test_retrain_check(self): with session_scope(self.dal) as session: insert_updated_nightly_file(session, self.predicted_nightly_data) with session_scope(self.dal) as session: result = retrain_check(session) expected = False self.assertEqual(result, expected)
def test_retrain_check(self): with session_scope(self.dal) as session: insert_data(session, self.data) with session_scope(self.dal) as session: result = retrain_check(session) expected = False self.assertEqual(result, expected)
def tearDown(self): with session_scope(self.dal) as session: clear_data(session) with session_scope(self.dal) as session: session.close_all() self.dal.drop_test_postgres_db() self.dal = None
def test_get_trained_count(self): with session_scope(self.dal) as session: insert_data(session, self.data) with session_scope(self.dal) as session: result = get_trained_count(session) expected = 0 self.assertEqual(result, expected)
def main(): ''' Run all of the scripts together. ''' logger.info("Smartie is downloading the most recent nightly FBO file...") nightly_data = fbo_nightly_scraper.get_nightly_data() logger.info( "Smartie is done downloading the most recent nightly FBO file!") logger.info( "Smartie is getting the attachments and their text from each FBO notice..." ) fboa = get_fbo_attachments.FboAttachments(nightly_data) updated_nightly_data = fboa.update_nightly_data() logger.info( "Smartie is done getting the attachments and their text from each FBO notice!" ) logger.info("Smartie is making predictions for each notice attachment...") predict = Predict(updated_nightly_data) updated_nightly_data_with_predictions = predict.insert_predictions() logger.info( "Smartie is done making predictions for each notice attachment!") logger.info("Smartie is inserting data into the database...") with session_scope(dal) as session: insert_updated_nightly_file(session, updated_nightly_data_with_predictions) logger.info("Smartie is done inserting data into database!") logger.info("Smartie is performing the retrain check...") with session_scope(dal) as session: retrain(session) logger.info("*" * 80)
def test_insert_data(self): with session_scope(self.dal) as session: insert_data(session, self.data) result = [] with session_scope(self.dal) as session: notices = session.query(Notice).all() for n in notices: notice = object_as_dict(n) #pop the date and createdAt attributes since they're constructed programmatically notice.pop('date') notice.pop('createdAt') #pop this as it'll vary notice.pop('notice_type_id') result.append(notice) expected = [{ 'id': 1, 'solicitation_number': 'test', 'agency': 'agency', 'notice_data': { 'url': 'url', 'naics': 'test', 'office': 'office', 'subject': 'test', 'classcod': 'test', 'setaside': 'test', 'emails': ['*****@*****.**'] }, 'compliant': 0, 'feedback': None, 'history': None, 'action': None, 'updatedAt': None, 'na_flag': False }] self.assertCountEqual(result, expected)
def test_fetch_validated_attachments(self): with session_scope(self.dal) as session: insert_updated_nightly_file(session, self.predicted_nightly_data) with session_scope(self.dal) as session: attachments = fetch_validated_attachments(session) result = len(attachments) expected = 993 self.assertEqual(result, expected)
def test_fetch_notices_by_solnbr_bogus_solnbr(self): with session_scope(self.dal) as session: insert_data(session, self.data) with session_scope(self.dal) as session: notices = fetch_notices_by_solnbr('notexist', session) result = len(notices) expected = 0 self.assertEqual(result, expected)
def test_fetch_notices_by_solnbr_bogus_solnbr(self): with session_scope(self.dal) as session: insert_updated_nightly_file(session, self.predicted_nightly_data) with session_scope(self.dal) as session: notices = fetch_notices_by_solnbr('test123', session) result = len(notices) expected = 0 self.assertEqual(result, expected)
def tearDown(self): with session_scope(self.dal) as session: clear_data(session) with session_scope(self.dal) as session: session.close_all() self.dal.drop_test_postgres_db() self.dal = None self.predicted_nightly_data = None self.predicted_nightly_data_day_two = None
def test_fetch_notices_by_solnbr(self): with session_scope(self.dal) as session: insert_updated_nightly_file(session, self.predicted_nightly_data) with session_scope(self.dal) as session: notices = fetch_notices_by_solnbr( 'rfp-e-bpm-djf-18-0800-pr-0000828', session) result = len(notices) expected = 1 self.assertEqual(result, expected)
def test_fetch_validated_attachments(self): with session_scope(self.dal) as session: insert_data(session, self.data) with session_scope(self.dal) as session: attachments = fetch_validated_attachments(session) result = len(attachments) # 993 since that's how many docs were initially labeled expected = 993 self.assertEqual(result, expected)
def test_fetch_last_score(self): results = {'c': 'd'} params = {'a': 'b'} score = .99 with session_scope(self.dal) as session: insert_model(session, results=results, params=params, score=score) with session_scope(self.dal) as session: score = fetch_last_score(session) result = score expected = .99 self.assertEqual(result, expected)
def test_insert_data_with_new_notice_type(self): opp = self.data[0].copy() nnt = "new notice type" opp['notice type'] = nnt with session_scope(self.dal) as session: insert_data(session, [opp]) result = [] with session_scope(self.dal) as session: notices = session.query(Notice).all() for n in notices: notice = object_as_dict(n) notice_type_id = int(notice['notice_type_id']) notice_type = fetch_notice_type_by_id(notice_type_id, session) self.assertCountEqual(notice_type.notice_type, nnt)
def test_insert_data_into_solicitations_table(self): with session_scope(self.dal) as session: try: insert_data_into_solicitations_table( session, [mock_schematized_opp_two]) except Exception as e: print(e)
def main(limit=None, updateOld=True, opportunity_filter_function=None, target_sol_types="o,k", skip_attachments=False, from_date = 'yesterday', to_date='yesterday'): try: if limit: logger.error("Artifical limit of {} placed on the number of opportunities processed. Should not happen in production.".format(limit)) if not updateOld: logger.error("Set to NOT update old solicitations. Should not happen in production.".format(limit)) with session_scope(dal) as session: # make sure that the notice types are configured and committed before going further insert_notice_types(session) logger.info("Connecting with database at {}".format(conn_string)) logger.info("Smartie is fetching opportunties from SAM...") data = get_opps.main(limit, opportunity_filter_function=opportunity_filter_function, target_sol_types=target_sol_types, skip_attachments=skip_attachments, from_date=from_date, to_date=to_date) if not data: logger.info("Smartie didn't find any opportunities!") else: logger.info("Smartie is done fetching opportunties from SAM!") logger.info("Smartie is making predictions for each notice attachment...") predict = Predict(data) data = predict.insert_predictions() logger.info("Smartie is done making predictions for each notice attachment!") with session_scope(dal) as session: if data: # insert_data(session, data) logger.info("Smartie is inserting data into the database...") insert_data_into_solicitations_table(session, data) logger.info("Smartie is done inserting data into database!") else: logger.error("No data to insert. Something went wrong.") if updateOld: update_old_solicitations(session, max_tests=10) logger.info("Run complete without major errors.") except Exception as e: logger.error("Unhandled error. Data for the day may be lost.") logger.error(f"Exception: {e}", exc_info=True) logger.error("Unexpected error: {}".format(str(sys.exc_info()[0])))
def test_update_old_solicitations(self): conn_string = get_db_url() dal = DataAccessLayer(conn_string) dal.connect() with session_scope(dal) as session: sam_utils.update_old_solicitations(session, age_cutoff=365, max_tests=5)
def test_insert_notice_types(self): with session_scope(self.dal) as session: insert_notice_types(session) types = [ 'Presolicitation', 'Solicitation', 'Combined Synopsis/Solicitation', 'TRAIN' ] notice_type_ids = [] for notice_type in types: with session_scope(self.dal) as session: notice_type_id = session.query(NoticeType.id).filter( NoticeType.notice_type == notice_type).first().id notice_type_ids.append(notice_type_id) notice_type_ids = set(notice_type_ids) result = len(notice_type_ids) expected = len(types) self.assertEqual(result, expected)
def setUp(self): self.data = [mock_data_for_db.copy()] self.dal = DataAccessLayer(conn_string=get_db_url()) self.dal.create_test_postgres_db() self.dal.connect() with session_scope(self.dal) as session: insert_notice_types(session) self.maxDiff = None
def test_insert_model(self): results = {'c': 'd'} params = {'a': 'b'} score = .99 with session_scope(self.dal) as session: insert_model(session, results=results, params=params, score=score) result = [] with session_scope(self.dal) as session: models = session.query(Model).all() for m in models: model = object_as_dict(m) model.pop('create_date') result.append(model) expected = [{ 'id': 1, 'results': results, 'params': params, 'score': score }] self.assertCountEqual(result, expected)
def test_insert_notice_types(self): with session_scope(self.dal) as session: insert_notice_types(session) notice_types = ['MOD', 'PRESOL', 'COMBINE', 'AMDCSS', 'TRAIN'] notice_type_ids = [] for notice_type in notice_types: notice_type_id = session.query(NoticeType.id).filter( NoticeType.notice_type == notice_type).first().id notice_type_ids.append(notice_type_id) notice_type_ids = set(notice_type_ids) result = len(notice_type_ids) expected = len(notice_types) self.assertEqual(result, expected)
def test_insert_bad_notice(self): call_count = 0 with session_scope(self.dal) as session: # intentionally bad notice type data = mock_data_for_db.copy() data['notice type'] = "not to be found" self.assertNotEqual(mock_data_for_db['notice type'], data['notice type']) logger = logging.getLogger("utils.db.db_utils") print(logger) with mock.patch.object(logger, 'warning', wraps=logger.warning): insert_data(session, [data]) call_count = logger.warning.call_count self.assertEqual( 1, call_count, "We should get one warning when adding a notice with a new notice type." )
def setUp(self): self.dal = DataAccessLayer(conn_string=get_db_url()) self.dal.connect() with session_scope(self.dal) as session: insert_notice_types(session)
def tearDown(self): with session_scope(self.dal) as session: session.close_all() self.dal = None self.data = None
def test_insert_updated_nightly_file_day_two(self): ''' Simulate a second batch entry with a repeating solnbr that now has attachments ''' with session_scope(self.dal) as session: insert_updated_nightly_file(session, self.predicted_nightly_data) with session_scope(self.dal) as session: insert_updated_nightly_file(session, self.predicted_nightly_data_day_two) result = [] with session_scope(self.dal) as session: notices = session.query(Notice).all() for n in notices: notice = object_as_dict(n) #pop the date and createdAt attributes since they're constructed programmatically notice.pop('date') notice.pop('createdAt') if notice['history']: notice['history'][0]['date'] = "test date" result.append(notice) expected = [{ 'id': 1, 'notice_type_id': 6, 'solicitation_number': 'rfp-e-bpm-djf-18-0800-pr-0000828', 'agency': 'department of justice', 'notice_data': { 'url': 'url', 'zip': '20535', 'date': '0506', 'desc': ' link to document', 'year': '18', 'naics': '511210', 'ntype': 'combine', 'offadd': '935 pennsylvania avenue, n.w. washington dc 20535', 'office': 'federal bureau of investigation', 'popzip': '20535', 'contact': 'clark kent, contracting officer, phone 5555555555, email [email protected]', 'subject': 'enterprise business process management software tool', 'classcod': '70', 'location': 'procurement section', 'setaside': 'n/a', 'popaddress': '935 pennsylvania ave. n.w. washington, dc ', 'popcountry': 'us' }, 'compliant': 0, 'feedback': None, 'history': None, 'action': None, 'updatedAt': None }, { 'id': 2, 'notice_type_id': 5, 'solicitation_number': 'spe4a618t934n', 'agency': 'defense logistics agency', 'notice_data': { 'url': 'test_url', 'zip': '23297', 'date': '0506', 'desc': 'test123', 'year': '18', 'naics': '334511', 'offadd': '334511', 'office': 'dla acquisition locations', 'contact': '*****@*****.**', 'subject': 'subject', 'archdate': '06132018', 'classcod': '66', 'location': 'dla aviation - bsm', 'respdate': '051418', 'setaside': 'n/a ' }, 'compliant': 0, 'feedback': None, 'history': None, 'action': None, 'updatedAt': None }, { 'id': 3, 'notice_type_id': 6, 'solicitation_number': 'spe4a618t934n', 'agency': 'defense logistics agency', 'notice_data': { 'url': 'test_url', 'zip': '23297', 'date': '0506', 'desc': 'test123', 'year': '17', 'naics': '334511', 'offadd': '334511', 'office': 'dla acquisition locations', 'contact': '*****@*****.**', 'subject': 'subject', 'archdate': '06132018', 'classcod': '66', 'location': 'dla aviation - bsm', 'respdate': '051418', 'setaside': 'n/a ' }, 'compliant': 0, 'feedback': None, 'history': [{ "date": "test date", "user": "", "action": "Solicitation Updated on FBO.gov", "status": "" }], 'action': None, 'updatedAt': None }] self.assertEqual(result, expected)