def test_load_data_with_matches_into_database(self):
insert_allegation_into_database(DataAllegation(cr_id="1087378"))
insert_allegation_into_database(DataAllegation(cr_id="1087387"))
expected_matches = [
transformed_data_with_rows.iloc[1],
transformed_data_with_rows.iloc[2]
]
expected_new_data = [
pd.Series(transformed_data.iloc[0][0]),
pd.Series(transformed_data.iloc[3][0]),
pd.Series(transformed_data.iloc[4][0])
]
testLoader = Loader()
testLoader.load_into_db(transformed_data_with_rows)
matches = testLoader.get_matches()
assert (pd.Series(expected_matches[0][0]).equals(matches[0]))
assert (pd.Series(expected_matches[1][0]).equals(matches[1]))
new_data = testLoader.get_new_data()
assert (expected_new_data[0].equals(new_data[0]))
assert (expected_new_data[1].equals(new_data[1]))
assert (expected_new_data[2].equals(new_data[2]))
queried_officer_data = DataOfficerAllegation.query.all()
testLoader.load_into_db(transformed_data_with_rows)
assert (len(queried_officer_data) == len(
DataOfficerAllegation.query.all()))
def copa_scrape():
scraped_data = scrape_data()
transformer = CopaScrapeTransformer()
transformer.transform(scraped_data)
transformed_data = transformer.get_transformed_data()
loader = Loader()
loader.load_into_db(transformed_data)
saver = Saver()
saver.save_to_csv(loader.get_new_data(), "new_data.csv")
saver.save_to_csv(loader.get_matches(), "match_data.csv")
return Response(status=200, response='Success')