def test_print_iterator_on_2019_21(self):
fetcher_2019_21 = blf.BillsOverview()
mock_7_days_ago = datetime.datetime.now() - datetime.timedelta(days=7)
fetcher_2019_21.mock_datetime_last_scraped(mock_7_days_ago)
fetcher_2019_21.get_changed_bills_in_session(session_name="2019-21")
print_all_info_using_iterator(fetcher_2019_21)
def test_correct_originating_house_obtained(self):
fetcher_2019_21 = blf.BillsOverview()
mock_datetime = datetime.datetime(2021, 9, 16, 18)
fetcher_2019_21.mock_datetime_last_scraped(mock_datetime)
fetcher_2019_21.get_changed_bills_in_session(session_name="2019-21")
last_bill = get_last_bill_details(fetcher_2019_21)
self.assertTrue(last_bill.originating_house ==
blf.BillsOverview.OriginatingHouse.HOUSE_OF_COMMONS)
def test_correct_bill_stripped_title_obtained(self):
fetcher_2019_21 = blf.BillsOverview()
mock_datetime = datetime.datetime(2021, 9, 16, 18)
fetcher_2019_21.mock_datetime_last_scraped(mock_datetime)
fetcher_2019_21.get_changed_bills_in_session(session_name="2019-21")
last_bill = get_last_bill_details(fetcher_2019_21)
self.assertTrue(
last_bill.title_stripped == "Telecommunications (Security)")
def test_correct_bill_stripped_title_obtained_chronological(self):
fetcher_2019_21 = blf.BillsOverview()
mock_datetime = datetime.datetime(2021, 9, 16, 18)
fetcher_2019_21.mock_datetime_last_scraped(mock_datetime)
fetcher_2019_21.get_changed_bills_in_session(session_name="2019-21")
# run generator once as we want the first item
for l in bdi.get_bill_details(fetcher_2019_21, chronological=True):
first_bill = l
break
self.assertTrue(
first_bill.title_stripped == "Telecommunications (Security)")
def insert_bills_and_divisions_data(conn,
cursor,
fresh=False,
session="2019-21"):
if fresh == True:
os.remove("datetime_last_scraped.p")
clear_table(conn, cursor, "MPVotes")
clear_table(conn, cursor, "Bills")
bills_overview = blf.BillsOverview()
bills_overview.get_changed_bills_in_session(session_name=session)
print("new bills_overview.bills_overview_data")
print(bills_overview.bills_overview_data)
print("bills overview object obtained, scraping complete")
print(f"item counts: {bills_overview.bills_overview_data.count()}")
pd.set_option('display.max_columns', 20)
print(bills_overview.bills_overview_data)
# insert everything back in
put_bill_and_division_data_in_db(conn, cursor, bills_overview)
def upsert_bills_and_divisions_data(fresh=False,
session="All",
to_run_on_app_engine=True):
if fresh == True:
os.remove("datetime_last_scraped.p")
clear_table("MPVotes")
clear_table("Bills")
# TODO IMPORTANT UNCOMMENT THIS AND DELETE NEXT
bills_overview = blf.BillsOverview(run_on_app_engine=to_run_on_app_engine,
project_name="bills-app-305000",
debug=True)
#bills_overview = blf.BillsOverview(run_on_app_engine=False, project_name="bills-app-305000", debug=True)
mock_datetime = datetime.datetime.now() - datetime.timedelta(days=7)
bills_overview.mock_datetime_last_scraped(mock_datetime)
bills_overview.get_changed_bills_in_session(session_name=session)
filename = datetime.datetime.now().isoformat() + "_bills_overview"
#write_to_log_file(bills_overview.bills_overview_data.to_string(), filename)
# insert everything back in
put_bill_and_division_data_in_db(bills_overview)
def test_print_iterator_on_2004_05_session(self):
fetcher_2004_05 = blf.BillsOverview()
fetcher_2004_05.update_all_bills_in_session(session_name="2004-05")
print_all_info_using_iterator(fetcher_2004_05)
def test_iterator_on_2019_21(self):
fetcher_2019_21 = blf.BillsOverview()
fetcher_2019_21.update_all_bills_in_session(session_name="2019-21")
print_all_info_using_iterator(fetcher_2019_21)
def test_iterator_on_2015_16(self):
fetcher_2015_16 = blf.BillsOverview()
fetcher_2015_16.update_all_bills_in_session(session_name="2015-16")
print_all_info_using_iterator(fetcher_2015_16)