def test_retrain_check(self): with session_scope(self.dal) as session: insert_updated_nightly_file(session, self.predicted_nightly_data) with session_scope(self.dal) as session: result = retrain_check(session) expected = False self.assertEqual(result, expected)
def main(): ''' Run all of the scripts together. ''' logger.info("Smartie is downloading the most recent nightly FBO file...") nightly_data = fbo_nightly_scraper.get_nightly_data() logger.info( "Smartie is done downloading the most recent nightly FBO file!") logger.info( "Smartie is getting the attachments and their text from each FBO notice..." ) fboa = get_fbo_attachments.FboAttachments(nightly_data) updated_nightly_data = fboa.update_nightly_data() logger.info( "Smartie is done getting the attachments and their text from each FBO notice!" ) logger.info("Smartie is making predictions for each notice attachment...") predict = Predict(updated_nightly_data) updated_nightly_data_with_predictions = predict.insert_predictions() logger.info( "Smartie is done making predictions for each notice attachment!") logger.info("Smartie is inserting data into the database...") with session_scope(dal) as session: insert_updated_nightly_file(session, updated_nightly_data_with_predictions) logger.info("Smartie is done inserting data into database!") logger.info("Smartie is performing the retrain check...") with session_scope(dal) as session: retrain(session) logger.info("*" * 80)
def test_get_validated_untrained_count(self): with session_scope(self.dal) as session: insert_updated_nightly_file(session, self.predicted_nightly_data) with session_scope(self.dal) as session: result = get_validated_untrained_count(session) expected = 0 self.assertEqual(result, expected)
def test_fetch_notices_by_solnbr_bogus_solnbr(self): with session_scope(self.dal) as session: insert_updated_nightly_file(session, self.predicted_nightly_data) with session_scope(self.dal) as session: notices = fetch_notices_by_solnbr('test123', session) result = len(notices) expected = 0 self.assertEqual(result, expected)
def test_fetch_validated_attachments(self): with session_scope(self.dal) as session: insert_updated_nightly_file(session, self.predicted_nightly_data) with session_scope(self.dal) as session: attachments = fetch_validated_attachments(session) result = len(attachments) expected = 993 self.assertEqual(result, expected)
def test_fetch_notices_by_solnbr(self): with session_scope(self.dal) as session: insert_updated_nightly_file(session, self.predicted_nightly_data) with session_scope(self.dal) as session: notices = fetch_notices_by_solnbr( 'rfp-e-bpm-djf-18-0800-pr-0000828', session) result = len(notices) expected = 1 self.assertEqual(result, expected)
def test_insert_updated_nightly_file_day_two(self): ''' Simulate a second batch entry with a repeating solnbr that now has attachments ''' with session_scope(self.dal) as session: insert_updated_nightly_file(session, self.predicted_nightly_data) with session_scope(self.dal) as session: insert_updated_nightly_file(session, self.predicted_nightly_data_day_two) result = [] with session_scope(self.dal) as session: notices = session.query(Notice).all() for n in notices: notice = object_as_dict(n) #pop the date and createdAt attributes since they're constructed programmatically notice.pop('date') notice.pop('createdAt') if notice['history']: notice['history'][0]['date'] = "test date" result.append(notice) expected = [{ 'id': 1, 'notice_type_id': 6, 'solicitation_number': 'rfp-e-bpm-djf-18-0800-pr-0000828', 'agency': 'department of justice', 'notice_data': { 'url': 'url', 'zip': '20535', 'date': '0506', 'desc': ' link to document', 'year': '18', 'naics': '511210', 'ntype': 'combine', 'offadd': '935 pennsylvania avenue, n.w. washington dc 20535', 'office': 'federal bureau of investigation', 'popzip': '20535', 'contact': 'clark kent, contracting officer, phone 5555555555, email [email protected]', 'subject': 'enterprise business process management software tool', 'classcod': '70', 'location': 'procurement section', 'setaside': 'n/a', 'popaddress': '935 pennsylvania ave. n.w. washington, dc ', 'popcountry': 'us' }, 'compliant': 0, 'feedback': None, 'history': None, 'action': None, 'updatedAt': None }, { 'id': 2, 'notice_type_id': 5, 'solicitation_number': 'spe4a618t934n', 'agency': 'defense logistics agency', 'notice_data': { 'url': 'test_url', 'zip': '23297', 'date': '0506', 'desc': 'test123', 'year': '18', 'naics': '334511', 'offadd': '334511', 'office': 'dla acquisition locations', 'contact': '*****@*****.**', 'subject': 'subject', 'archdate': '06132018', 'classcod': '66', 'location': 'dla aviation - bsm', 'respdate': '051418', 'setaside': 'n/a ' }, 'compliant': 0, 'feedback': None, 'history': None, 'action': None, 'updatedAt': None }, { 'id': 3, 'notice_type_id': 6, 'solicitation_number': 'spe4a618t934n', 'agency': 'defense logistics agency', 'notice_data': { 'url': 'test_url', 'zip': '23297', 'date': '0506', 'desc': 'test123', 'year': '17', 'naics': '334511', 'offadd': '334511', 'office': 'dla acquisition locations', 'contact': '*****@*****.**', 'subject': 'subject', 'archdate': '06132018', 'classcod': '66', 'location': 'dla aviation - bsm', 'respdate': '051418', 'setaside': 'n/a ' }, 'compliant': 0, 'feedback': None, 'history': [{ "date": "test date", "user": "", "action": "Solicitation Updated on FBO.gov", "status": "" }], 'action': None, 'updatedAt': None }] self.assertEqual(result, expected)