def lambda_handler(event, context):
try:
covid_df = us_covid_etl.prepare_data(NYT_FILE_PATH, JH_FILE_PATH)
except Exception as e:
print('Error while preparing data:' + str(e))
send_notification(
'ETL job failed during processing of csv file. Error: ' + str(e),
'ETL job result')
exit(1)
print('Retrieved data successfully.')
try:
save_data(covid_df)
except ClientError as e:
print(f'Error while saving data: {e.response["Error"]["Message"]}')
except Exception as e:
print('Error while saving data:' + str(e))
send_notification(
'ETL job failed during saving the data. Error: ' + str(e),
'ETL job result')
exit(1)
def test_missing_us_country(self):
df = us_covid_etl.prepare_data(self.TESTS_PATH+'test_nyt_join.csv', self.TESTS_PATH+'test_jh_missing_country.csv')
self.assertEquals(df.shape, (0,4))
def test_not_integer(self):
with self.assertRaises(ValueError):
df = us_covid_etl.prepare_data(self.TESTS_PATH+'test_not_integer.csv', self.TESTS_PATH+'test_jh.csv')
def test_negative_value(self):
with self.assertRaises(ValueError):
df = us_covid_etl.prepare_data(self.TESTS_PATH+'test_negative_value.csv', self.TESTS_PATH+'test_jh.csv')
def test_date_parsing(self):
with self.assertRaises(Exception):
df = us_covid_etl.prepare_data(self.TESTS_PATH+'test_wrong_date.csv', self.TESTS_PATH+'test_jh.csv')
def test_join_with_extra_date(self):
df = us_covid_etl.prepare_data(self.TESTS_PATH+'test_nyt_join.csv', self.TESTS_PATH+'test_jh_join_extra_date.csv')
self.assertEquals(df.shape, (9,4))
def test_correct_table_join(self):
df = us_covid_etl.prepare_data(self.TESTS_PATH+'test_nyt_join.csv', self.TESTS_PATH+'test_jh_join.csv')
self.assertEquals(df.shape, (9,4))