def test_update_old_solicitations(self): conn_string = get_db_url() dal = DataAccessLayer(conn_string) dal.connect() with session_scope(dal) as session: sam_utils.update_old_solicitations(session, age_cutoff=365, max_tests=5)
def setUp(self): self.data = [mock_data_for_db.copy()] self.dal = DataAccessLayer(conn_string=get_db_url()) self.dal.create_test_postgres_db() self.dal.connect() with session_scope(self.dal) as session: insert_notice_types(session) self.maxDiff = None
class DBTestCase(unittest.TestCase): def setUp(self): self.dal = DataAccessLayer(conn_string=get_db_url()) self.dal.connect() with session_scope(self.dal) as session: insert_notice_types(session) def tearDown(self): with session_scope(self.dal) as session: session.close_all() self.dal = None self.data = None def test_insert_data_into_solicitations_table(self): with session_scope(self.dal) as session: try: insert_data_into_solicitations_table( session, [mock_schematized_opp_two]) except Exception as e: print(e)
class EndToEndTest(unittest.TestCase): def setUp(self): conn_string = get_db_url() self.dal = DataAccessLayer(conn_string) self.dal.create_test_postgres_db() self.dal.connect() self.main = main def tearDown(self): with session_scope(self.dal) as session: clear_data(session) with session_scope(self.dal) as session: session.close_all() self.dal.drop_test_postgres_db() self.dal = None self.main = None @patch('utils.fbo_nightly_scraper') def test_main(self, fbo_mock): nfbo = fbo_mock.NightlyFBONotices.return_value # use 10/28 since the 28th's file is only 325 kB nfbo.ftp_url = 'ftp://ftp.fbo.gov/FBOFeed20181028' with self.subTest(): self.main() self.assertTrue(True) with self.subTest(): cwd = os.getcwd() attachments_dir = os.path.join(cwd, 'attachments') dir_exists = os.path.isdir(attachments_dir) self.assertFalse(dir_exists)
class EndToEndTest(unittest.TestCase): def setUp(self): conn_string = get_db_url() self.dal = DataAccessLayer(conn_string) self.dal.create_test_postgres_db() self.dal.connect() def tearDown(self): with session_scope(self.dal) as session: clear_data(session) with session_scope(self.dal) as session: session.close_all() self.dal.drop_test_postgres_db() self.dal = None def test_main(self): with self.subTest(): main(limit=20) self.assertTrue(True)
import logging from utils import get_opps from utils.predict import Predict from utils.db.db_utils import get_db_url, session_scope, DataAccessLayer, insert_data_into_solicitations_table, insert_notice_types from utils.json_log_formatter import CustomJsonFormatter, configureLogger from utils.sam_utils import update_old_solicitations, opportunity_filter_function, set_psc_code_download_list import sys import datetime import os logger = logging.getLogger() configureLogger(logger, stdout_level=logging.INFO) conn_string = get_db_url() dal = DataAccessLayer(conn_string) dal.connect() def main(limit=None, updateOld=True, opportunity_filter_function=None, target_sol_types="o,k", skip_attachments=False, from_date = 'yesterday', to_date='yesterday'): try: if limit: logger.error("Artifical limit of {} placed on the number of opportunities processed. Should not happen in production.".format(limit)) if not updateOld: logger.error("Set to NOT update old solicitations. Should not happen in production.".format(limit)) with session_scope(dal) as session: # make sure that the notice types are configured and committed before going further insert_notice_types(session)
def setUp(self): conn_string = get_db_url() self.dal = DataAccessLayer(conn_string) self.dal.create_test_postgres_db() self.dal.connect() self.main = main
def setUp(self): conn_string = get_db_url() self.predicted_nightly_data = { 'AMDCSS': [{ 'date': '0506', 'year': '18', 'agency': 'department of justice', 'office': 'federal bureau of investigation', 'location': 'procurement section', 'zip': '20535', 'classcod': '70', 'naics': '511210', 'offadd': '935 pennsylvania avenue, n.w. washington dc 20535', 'subject': 'enterprise business process management software tool', 'solnbr': 'rfp-e-bpm-djf-18-0800-pr-0000828', 'ntype': 'combine', 'contact': 'clark kent, contracting officer, phone 5555555555, email [email protected]', 'desc': ' link to document', 'url': 'url', 'setaside': 'n/a', 'popcountry': 'us', 'popzip': '20535', 'popaddress': '935 pennsylvania ave. n.w. washington, dc ', 'attachments': [{ 'filename': 'test.txt', 'machine_readable': True, 'text': 'test_text_0', 'url': 'test_url_0', 'prediction': 1, 'decision_boundary': 0, 'validation': None, 'trained': False }, { 'filename': 'test.txt', 'machine_readable': True, 'text': 'test_text_1', 'url': 'test_url_1', 'prediction': 1, 'decision_boundary': 0, 'validation': None, 'trained': False }, { 'filename': 'test.txt', 'machine_readable': True, 'text': 'test_text_2', 'url': 'test_url_2', 'prediction': 1, 'decision_boundary': 0, 'validation': None, 'trained': False }, { 'filename': 'test.txt', 'machine_readable': True, 'text': 'test_text_3', 'url': 'test_url_3', 'prediction': 1, 'decision_boundary': 0, 'validation': None, 'trained': False }, { 'filename': 'test.txt', 'machine_readable': True, 'text': 'test_text_4', 'url': 'test_url_4', 'prediction': 1, 'decision_boundary': 0, 'validation': None, 'trained': False }, { 'filename': 'test.txt', 'machine_readable': True, 'text': 'test_text_5', 'url': 'test_url_5', 'prediction': 1, 'decision_boundary': 0, 'validation': None, 'trained': False }, { 'filename': 'test.txt', 'machine_readable': True, 'text': 'test_text_6', 'url': 'test_url_6', 'prediction': 1, 'decision_boundary': 0, 'validation': None, 'trained': False }], 'compliant': 0 }], 'MOD': [], 'COMBINE': [{ 'date': '0506', 'year': '18', 'agency': 'defense logistics agency', 'office': 'dla acquisition locations', 'location': 'dla aviation - bsm', 'zip': '23297', 'classcod': '66', 'naics': '334511', 'offadd': '334511', 'subject': 'subject', 'solnbr': 'spe4a618t934n', 'respdate': '051418', 'archdate': '06132018', 'contact': '*****@*****.**', 'desc': 'test123', 'url': 'test_url', 'setaside': 'n/a ', 'attachments': [], 'compliant': 0 }], 'PRESOL': [] } self.predicted_nightly_data_day_two = { 'AMDCSS': [{ 'date': '0506', 'year': '17', 'agency': 'defense logistics agency', 'office': 'dla acquisition locations', 'location': 'dla aviation - bsm', 'zip': '23297', 'classcod': '66', 'naics': '334511', 'offadd': '334511', 'subject': 'subject', 'solnbr': 'spe4a618t934n', 'respdate': '051418', 'archdate': '06132018', 'contact': '*****@*****.**', 'desc': 'test123', 'url': 'test_url', 'setaside': 'n/a ', 'attachments': [{ 'filename': 'test.txt', 'machine_readable': True, 'text': 'test_text_0', 'url': 'test_url_0', 'prediction': 1, 'decision_boundary': 0, 'validation': None, 'trained': False }], 'compliant': 0 }] } self.dal = DataAccessLayer(conn_string=conn_string) self.dal.create_test_postgres_db() self.dal.connect() self.maxDiff = None
class DBTestCase(unittest.TestCase): def setUp(self): conn_string = get_db_url() self.predicted_nightly_data = { 'AMDCSS': [{ 'date': '0506', 'year': '18', 'agency': 'department of justice', 'office': 'federal bureau of investigation', 'location': 'procurement section', 'zip': '20535', 'classcod': '70', 'naics': '511210', 'offadd': '935 pennsylvania avenue, n.w. washington dc 20535', 'subject': 'enterprise business process management software tool', 'solnbr': 'rfp-e-bpm-djf-18-0800-pr-0000828', 'ntype': 'combine', 'contact': 'clark kent, contracting officer, phone 5555555555, email [email protected]', 'desc': ' link to document', 'url': 'url', 'setaside': 'n/a', 'popcountry': 'us', 'popzip': '20535', 'popaddress': '935 pennsylvania ave. n.w. washington, dc ', 'attachments': [{ 'filename': 'test.txt', 'machine_readable': True, 'text': 'test_text_0', 'url': 'test_url_0', 'prediction': 1, 'decision_boundary': 0, 'validation': None, 'trained': False }, { 'filename': 'test.txt', 'machine_readable': True, 'text': 'test_text_1', 'url': 'test_url_1', 'prediction': 1, 'decision_boundary': 0, 'validation': None, 'trained': False }, { 'filename': 'test.txt', 'machine_readable': True, 'text': 'test_text_2', 'url': 'test_url_2', 'prediction': 1, 'decision_boundary': 0, 'validation': None, 'trained': False }, { 'filename': 'test.txt', 'machine_readable': True, 'text': 'test_text_3', 'url': 'test_url_3', 'prediction': 1, 'decision_boundary': 0, 'validation': None, 'trained': False }, { 'filename': 'test.txt', 'machine_readable': True, 'text': 'test_text_4', 'url': 'test_url_4', 'prediction': 1, 'decision_boundary': 0, 'validation': None, 'trained': False }, { 'filename': 'test.txt', 'machine_readable': True, 'text': 'test_text_5', 'url': 'test_url_5', 'prediction': 1, 'decision_boundary': 0, 'validation': None, 'trained': False }, { 'filename': 'test.txt', 'machine_readable': True, 'text': 'test_text_6', 'url': 'test_url_6', 'prediction': 1, 'decision_boundary': 0, 'validation': None, 'trained': False }], 'compliant': 0 }], 'MOD': [], 'COMBINE': [{ 'date': '0506', 'year': '18', 'agency': 'defense logistics agency', 'office': 'dla acquisition locations', 'location': 'dla aviation - bsm', 'zip': '23297', 'classcod': '66', 'naics': '334511', 'offadd': '334511', 'subject': 'subject', 'solnbr': 'spe4a618t934n', 'respdate': '051418', 'archdate': '06132018', 'contact': '*****@*****.**', 'desc': 'test123', 'url': 'test_url', 'setaside': 'n/a ', 'attachments': [], 'compliant': 0 }], 'PRESOL': [] } self.predicted_nightly_data_day_two = { 'AMDCSS': [{ 'date': '0506', 'year': '17', 'agency': 'defense logistics agency', 'office': 'dla acquisition locations', 'location': 'dla aviation - bsm', 'zip': '23297', 'classcod': '66', 'naics': '334511', 'offadd': '334511', 'subject': 'subject', 'solnbr': 'spe4a618t934n', 'respdate': '051418', 'archdate': '06132018', 'contact': '*****@*****.**', 'desc': 'test123', 'url': 'test_url', 'setaside': 'n/a ', 'attachments': [{ 'filename': 'test.txt', 'machine_readable': True, 'text': 'test_text_0', 'url': 'test_url_0', 'prediction': 1, 'decision_boundary': 0, 'validation': None, 'trained': False }], 'compliant': 0 }] } self.dal = DataAccessLayer(conn_string=conn_string) self.dal.create_test_postgres_db() self.dal.connect() self.maxDiff = None def tearDown(self): with session_scope(self.dal) as session: clear_data(session) with session_scope(self.dal) as session: session.close_all() self.dal.drop_test_postgres_db() self.dal = None self.predicted_nightly_data = None self.predicted_nightly_data_day_two = None def test_insert_notice_types(self): with session_scope(self.dal) as session: insert_notice_types(session) notice_types = ['MOD', 'PRESOL', 'COMBINE', 'AMDCSS', 'TRAIN'] notice_type_ids = [] for notice_type in notice_types: notice_type_id = session.query(NoticeType.id).filter( NoticeType.notice_type == notice_type).first().id notice_type_ids.append(notice_type_id) notice_type_ids = set(notice_type_ids) result = len(notice_type_ids) expected = len(notice_types) self.assertEqual(result, expected) def test_insert_updated_nightly_file(self): with session_scope(self.dal) as session: insert_updated_nightly_file(session, self.predicted_nightly_data) result = [] with session_scope(self.dal) as session: notices = session.query(Notice).all() for n in notices: notice = object_as_dict(n) #pop the date and createdAt attributes since they're constructed programmatically notice.pop('date') notice.pop('createdAt') result.append(notice) expected = [{ 'id': 1, 'notice_type_id': 6, 'solicitation_number': 'rfp-e-bpm-djf-18-0800-pr-0000828', 'agency': 'department of justice', 'notice_data': { 'url': 'url', 'zip': '20535', 'date': '0506', 'desc': ' link to document', 'year': '18', 'naics': '511210', 'ntype': 'combine', 'offadd': '935 pennsylvania avenue, n.w. washington dc 20535', 'office': 'federal bureau of investigation', 'popzip': '20535', 'contact': 'clark kent, contracting officer, phone 5555555555, email [email protected]', 'subject': 'enterprise business process management software tool', 'classcod': '70', 'location': 'procurement section', 'setaside': 'n/a', 'popaddress': '935 pennsylvania ave. n.w. washington, dc ', 'popcountry': 'us' }, 'compliant': 0, 'feedback': None, 'history': None, 'action': None, 'updatedAt': None }, { 'id': 2, 'notice_type_id': 5, 'solicitation_number': 'spe4a618t934n', 'agency': 'defense logistics agency', 'notice_data': { 'url': 'test_url', 'zip': '23297', 'date': '0506', 'desc': 'test123', 'year': '18', 'naics': '334511', 'offadd': '334511', 'office': 'dla acquisition locations', 'contact': '*****@*****.**', 'subject': 'subject', 'archdate': '06132018', 'classcod': '66', 'location': 'dla aviation - bsm', 'respdate': '051418', 'setaside': 'n/a ' }, 'compliant': 0, 'feedback': None, 'history': None, 'action': None, 'updatedAt': None }] self.assertCountEqual(result, expected) def test_insert_model(self): results = {'c': 'd'} params = {'a': 'b'} score = .99 with session_scope(self.dal) as session: insert_model(session, results=results, params=params, score=score) result = [] with session_scope(self.dal) as session: models = session.query(Model).all() for m in models: model = object_as_dict(m) model.pop('create_date') result.append(model) expected = [{ 'id': 1, 'results': results, 'params': params, 'score': score }] self.assertCountEqual(result, expected) def test_fetch_last_score(self): results = {'c': 'd'} params = {'a': 'b'} score = .99 with session_scope(self.dal) as session: insert_model(session, results=results, params=params, score=score) with session_scope(self.dal) as session: score = fetch_last_score(session) result = score expected = .99 self.assertEqual(result, expected) def test_insert_updated_nightly_file_day_two(self): ''' Simulate a second batch entry with a repeating solnbr that now has attachments ''' with session_scope(self.dal) as session: insert_updated_nightly_file(session, self.predicted_nightly_data) with session_scope(self.dal) as session: insert_updated_nightly_file(session, self.predicted_nightly_data_day_two) result = [] with session_scope(self.dal) as session: notices = session.query(Notice).all() for n in notices: notice = object_as_dict(n) #pop the date and createdAt attributes since they're constructed programmatically notice.pop('date') notice.pop('createdAt') if notice['history']: notice['history'][0]['date'] = "test date" result.append(notice) expected = [{ 'id': 1, 'notice_type_id': 6, 'solicitation_number': 'rfp-e-bpm-djf-18-0800-pr-0000828', 'agency': 'department of justice', 'notice_data': { 'url': 'url', 'zip': '20535', 'date': '0506', 'desc': ' link to document', 'year': '18', 'naics': '511210', 'ntype': 'combine', 'offadd': '935 pennsylvania avenue, n.w. washington dc 20535', 'office': 'federal bureau of investigation', 'popzip': '20535', 'contact': 'clark kent, contracting officer, phone 5555555555, email [email protected]', 'subject': 'enterprise business process management software tool', 'classcod': '70', 'location': 'procurement section', 'setaside': 'n/a', 'popaddress': '935 pennsylvania ave. n.w. washington, dc ', 'popcountry': 'us' }, 'compliant': 0, 'feedback': None, 'history': None, 'action': None, 'updatedAt': None }, { 'id': 2, 'notice_type_id': 5, 'solicitation_number': 'spe4a618t934n', 'agency': 'defense logistics agency', 'notice_data': { 'url': 'test_url', 'zip': '23297', 'date': '0506', 'desc': 'test123', 'year': '18', 'naics': '334511', 'offadd': '334511', 'office': 'dla acquisition locations', 'contact': '*****@*****.**', 'subject': 'subject', 'archdate': '06132018', 'classcod': '66', 'location': 'dla aviation - bsm', 'respdate': '051418', 'setaside': 'n/a ' }, 'compliant': 0, 'feedback': None, 'history': None, 'action': None, 'updatedAt': None }, { 'id': 3, 'notice_type_id': 6, 'solicitation_number': 'spe4a618t934n', 'agency': 'defense logistics agency', 'notice_data': { 'url': 'test_url', 'zip': '23297', 'date': '0506', 'desc': 'test123', 'year': '17', 'naics': '334511', 'offadd': '334511', 'office': 'dla acquisition locations', 'contact': '*****@*****.**', 'subject': 'subject', 'archdate': '06132018', 'classcod': '66', 'location': 'dla aviation - bsm', 'respdate': '051418', 'setaside': 'n/a ' }, 'compliant': 0, 'feedback': None, 'history': [{ "date": "test date", "user": "", "action": "Solicitation Updated on FBO.gov", "status": "" }], 'action': None, 'updatedAt': None }] self.assertEqual(result, expected) def test_get_validation_count(self): with session_scope(self.dal) as session: insert_updated_nightly_file(session, self.predicted_nightly_data) with session_scope(self.dal) as session: result = get_validation_count(session) expected = 0 self.assertEqual(result, expected) def test_get_trained_count(self): with session_scope(self.dal) as session: insert_updated_nightly_file(session, self.predicted_nightly_data) with session_scope(self.dal) as session: result = get_trained_count(session) expected = 0 self.assertEqual(result, expected) def test_get_validated_untrained_count(self): with session_scope(self.dal) as session: insert_updated_nightly_file(session, self.predicted_nightly_data) with session_scope(self.dal) as session: result = get_validated_untrained_count(session) expected = 0 self.assertEqual(result, expected) def test_retrain_check(self): with session_scope(self.dal) as session: insert_updated_nightly_file(session, self.predicted_nightly_data) with session_scope(self.dal) as session: result = retrain_check(session) expected = False self.assertEqual(result, expected) def test_fetch_validated_attachments(self): with session_scope(self.dal) as session: insert_updated_nightly_file(session, self.predicted_nightly_data) with session_scope(self.dal) as session: attachments = fetch_validated_attachments(session) result = len(attachments) expected = 993 self.assertEqual(result, expected) def test_fetch_notices_by_solnbr(self): with session_scope(self.dal) as session: insert_updated_nightly_file(session, self.predicted_nightly_data) with session_scope(self.dal) as session: notices = fetch_notices_by_solnbr( 'rfp-e-bpm-djf-18-0800-pr-0000828', session) result = len(notices) expected = 1 self.assertEqual(result, expected) def test_fetch_notices_by_solnbr_bogus_solnbr(self): with session_scope(self.dal) as session: insert_updated_nightly_file(session, self.predicted_nightly_data) with session_scope(self.dal) as session: notices = fetch_notices_by_solnbr('test123', session) result = len(notices) expected = 0 self.assertEqual(result, expected)
class DBTestCase(unittest.TestCase): def setUp(self): self.data = [mock_data_for_db.copy()] self.dal = DataAccessLayer(conn_string=get_db_url()) self.dal.create_test_postgres_db() self.dal.connect() with session_scope(self.dal) as session: insert_notice_types(session) self.maxDiff = None def tearDown(self): with session_scope(self.dal) as session: clear_data(session) with session_scope(self.dal) as session: session.close_all() self.dal.drop_test_postgres_db() self.dal = None self.data = None def test_insert_bad_notice(self): call_count = 0 with session_scope(self.dal) as session: # intentionally bad notice type data = mock_data_for_db.copy() data['notice type'] = "not to be found" self.assertNotEqual(mock_data_for_db['notice type'], data['notice type']) logger = logging.getLogger("utils.db.db_utils") print(logger) with mock.patch.object(logger, 'warning', wraps=logger.warning): insert_data(session, [data]) call_count = logger.warning.call_count self.assertEqual( 1, call_count, "We should get one warning when adding a notice with a new notice type." ) def test_insert_notice_types(self): with session_scope(self.dal) as session: insert_notice_types(session) types = [ 'Presolicitation', 'Solicitation', 'Combined Synopsis/Solicitation', 'TRAIN' ] notice_type_ids = [] for notice_type in types: with session_scope(self.dal) as session: notice_type_id = session.query(NoticeType.id).filter( NoticeType.notice_type == notice_type).first().id notice_type_ids.append(notice_type_id) notice_type_ids = set(notice_type_ids) result = len(notice_type_ids) expected = len(types) self.assertEqual(result, expected) def test_insert_data(self): with session_scope(self.dal) as session: insert_data(session, self.data) result = [] with session_scope(self.dal) as session: notices = session.query(Notice).all() for n in notices: notice = object_as_dict(n) #pop the date and createdAt attributes since they're constructed programmatically notice.pop('date') notice.pop('createdAt') #pop this as it'll vary notice.pop('notice_type_id') result.append(notice) expected = [{ 'id': 1, 'solicitation_number': 'test', 'agency': 'agency', 'notice_data': { 'url': 'url', 'naics': 'test', 'office': 'office', 'subject': 'test', 'classcod': 'test', 'setaside': 'test', 'emails': ['*****@*****.**'] }, 'compliant': 0, 'feedback': None, 'history': None, 'action': None, 'updatedAt': None, 'na_flag': False }] self.assertCountEqual(result, expected) def test_insert_data_with_new_notice_type(self): opp = self.data[0].copy() nnt = "new notice type" opp['notice type'] = nnt with session_scope(self.dal) as session: insert_data(session, [opp]) result = [] with session_scope(self.dal) as session: notices = session.query(Notice).all() for n in notices: notice = object_as_dict(n) notice_type_id = int(notice['notice_type_id']) notice_type = fetch_notice_type_by_id(notice_type_id, session) self.assertCountEqual(notice_type.notice_type, nnt) def test_insert_model(self): results = {'c': 'd'} params = {'a': 'b'} score = .99 with session_scope(self.dal) as session: insert_model(session, results=results, params=params, score=score) result = [] with session_scope(self.dal) as session: models = session.query(Model).all() for m in models: model = object_as_dict(m) model.pop('create_date') result.append(model) expected = [{ 'id': 1, 'results': results, 'params': params, 'score': score }] self.assertCountEqual(result, expected) def test_fetch_last_score(self): results = {'c': 'd'} params = {'a': 'b'} score = .99 with session_scope(self.dal) as session: insert_model(session, results=results, params=params, score=score) with session_scope(self.dal) as session: score = fetch_last_score(session) result = score expected = .99 self.assertEqual(result, expected) def test_get_validation_count(self): with session_scope(self.dal) as session: insert_data(session, self.data) with session_scope(self.dal) as session: result = get_validation_count(session) expected = 0 self.assertEqual(result, expected) def test_get_trained_count(self): with session_scope(self.dal) as session: insert_data(session, self.data) with session_scope(self.dal) as session: result = get_trained_count(session) expected = 0 self.assertEqual(result, expected) def test_get_validated_untrained_count(self): with session_scope(self.dal) as session: insert_data(session, self.data) with session_scope(self.dal) as session: result = get_validated_untrained_count(session) expected = 0 self.assertEqual(result, expected) def test_retrain_check(self): with session_scope(self.dal) as session: insert_data(session, self.data) with session_scope(self.dal) as session: result = retrain_check(session) expected = False self.assertEqual(result, expected) def test_fetch_validated_attachments(self): with session_scope(self.dal) as session: insert_data(session, self.data) with session_scope(self.dal) as session: attachments = fetch_validated_attachments(session) result = len(attachments) # 993 since that's how many docs were initially labeled expected = 993 self.assertEqual(result, expected) def test_fetch_notices_by_solnbr(self): with session_scope(self.dal) as session: insert_data(session, self.data) with session_scope(self.dal) as session: notices = fetch_notices_by_solnbr('test', session) result = len(notices) expected = 1 self.assertEqual(result, expected) def test_fetch_notices_by_solnbr_bogus_solnbr(self): with session_scope(self.dal) as session: insert_data(session, self.data) with session_scope(self.dal) as session: notices = fetch_notices_by_solnbr('notexist', session) result = len(notices) expected = 0 self.assertEqual(result, expected)
def setUp(self): self.dal = DataAccessLayer(conn_string=get_db_url()) self.dal.connect() with session_scope(self.dal) as session: insert_notice_types(session)