def copa_scrape(): scraped_data = scrape_data() transformer = CopaScrapeTransformer() transformer.transform(scraped_data) transformed_data = transformer.get_transformed_data() loader = Loader() loader.load_into_db(transformed_data) saver = Saver() saver.save_to_csv( strip_zeroes_from_beat_id(loader.get_allegation_matches()), "match_data.csv") saver.save_to_csv( strip_zeroes_from_beat_id(loader.get_new_allegation_data()), "new_allegation_data.csv") saver.save_to_csv( cast_col_to_int(loader.new_officer_unknown_data, "data_officerallegation_id"), "new_officer_unknown.csv") saver.save_to_csv(loader.new_officer_allegation_data, "new_officer_allegation.csv") # do further processing on officer unknown return Response(status=200, response='Success')
def test_get_matches(self): expected_matches = expected_load_data testLoader = Loader() testLoader.load_into_db(expected_transformed_data_with_beat_id) testLoader.load_into_db(expected_transformed_data_with_beat_id) matches = testLoader.get_allegation_matches() assert_frame_equal(matches, expected_matches, check_dtype=False, check_like=True)