def copa_scrape():
scraped_data = scrape_data()
transformer = CopaScrapeTransformer()
transformer.transform(scraped_data)
transformed_data = transformer.get_transformed_data()
loader = Loader()
loader.load_into_db(transformed_data)
saver = Saver()
saver.save_to_csv(
strip_zeroes_from_beat_id(loader.get_allegation_matches()),
"match_data.csv")
saver.save_to_csv(
strip_zeroes_from_beat_id(loader.get_new_allegation_data()),
"new_allegation_data.csv")
saver.save_to_csv(
cast_col_to_int(loader.new_officer_unknown_data,
"data_officerallegation_id"),
"new_officer_unknown.csv")
saver.save_to_csv(loader.new_officer_allegation_data,
"new_officer_allegation.csv")
# do further processing on officer unknown
return Response(status=200, response='Success')
def test_get_new_data(self):
expected_new_data = expected_load_data
testLoader = Loader()
testLoader.load_into_db(expected_transformed_data_with_beat_id)
new_data = testLoader.get_new_allegation_data()
assert_frame_equal(new_data, expected_new_data, check_dtype=False, check_like=True)