def copa_scrape():
scraped_data = scrape_data()
transformer = CopaScrapeTransformer()
transformer.transform(scraped_data)
transformed_data = transformer.get_transformed_data()
loader = Loader()
loader.load_into_db(transformed_data)
saver = Saver()
saver.save_to_csv(
strip_zeroes_from_beat_id(loader.get_allegation_matches()),
"match_data.csv")
saver.save_to_csv(
strip_zeroes_from_beat_id(loader.get_new_allegation_data()),
"new_allegation_data.csv")
saver.save_to_csv(
cast_col_to_int(loader.new_officer_unknown_data,
"data_officerallegation_id"),
"new_officer_unknown.csv")
saver.save_to_csv(loader.new_officer_allegation_data,
"new_officer_allegation.csv")
# do further processing on officer unknown
return Response(status=200, response='Success')
def copa_scrape():
scraped_data = scrape_data()
transformer = CopaScrapeTransformer()
transformer.transform(scraped_data)
transformed_data = transformer.get_transformed_data()
loader = Loader()
loader.load_into_db(transformed_data)
saver = Saver()
saver.save_to_csv(loader.get_new_data(), "new_data.csv")
saver.save_to_csv(loader.get_matches(), "match_data.csv")
return Response(status=200, response='Success')
def test_copa_scrape_with_count_errors(self):
with pytest.raises(ConnectionError):
scrape_data()
def test_copa_scrape(self):
should_be_bubbles = scrape_data()
assert should_be_bubbles == "bubbles"