def run_etl(filename):
logger.info("application ran")
start = time.time()
app = Extract()
raw_data_list = app.get_data_from_bucket(filename) # extract output
end_extract = time.time()
extract_time = round(end_extract - start, 4)
print(f"Extract time: {extract_time}")
logger.info(f"Extract time: {extract_time}")
apple = Transform()
transformed_data, transformed_drink_menu_data = apple.transform_new_data(raw_data_list) # raw data into transform returns transformed data and drinks dic
end_transform = time.time()
transform_time = round(end_transform - end_extract,4)
logger.info(f"Transform time: {transform_time}")
print(f"Transform time: {transform_time}")
appley = Load()
appley.save_transaction(transformed_data) # populate RDS instance with cleaned data.
appley.save_drink_menu(transformed_drink_menu_data) # generate drinks menu
end_load = time.time()
load_time = round(end_load - end_transform, 4)
logger.info(f"Loading time: {load_time}")
total_time = extract_time + transform_time + load_time
logger.info(f"total time: {total_time}")
print(f"Load time: {load_time}\nTotal time: {total_time}")
# raw_data_list = app.load_all_data() # extract output from all time
end_extract = time.time()
extract_time = round(end_extract - start, 4)
print(f"Extract time: {extract_time}")
logger.info(f"Extract time: {extract_time}")
apple = Transform()
transformed_data, new_drinks, new_locations, basket = apple.transform(
raw_data_list
) # raw data into transform returns transformed data and drinks dic
# transformed_data, basket = apple.transform(raw_data_list) # raw data into transform returns transformed data and drinks dic
end_transform = time.time()
transform_time = round(end_transform - end_extract, 4)
logger.info(f"Transform time: {transform_time}")
print(f"Transform time: {transform_time}")
appley = Load()
appley.save_transaction(
transformed_data) # populate RDS instance with cleaned data.
appley.save_drink_menu(new_drinks) # generate drinks menu
appley.save_location_menu(new_locations) # generate locations menu
appley.save_basket(basket) # generate drinks menu
end_load = time.time()
load_time = round(end_load - end_transform, 4)
logger.info(f"Loading time: {load_time}")
total_time = extract_time + transform_time + load_time
logger.info(f"total time: {total_time}")
print(f"Load time: {load_time}\nTotal time: {total_time}")
if __name__ == "__main__":
logger.info("application ran")
start = time.time()
app = Extract()
raw_data_list = app.load_data() # extract output
end_extract = time.time()
extract_time = round(end_extract - start, 4)
print(f"Extract time: {extract_time}")
logger.info(f"Extract time: {extract_time}")
apple = Transform()
transformed_data, transformed_drink_menu_data = apple.transform(
raw_data_list
) # raw data into transform returns transformed data and drinks dic
end_transform = time.time()
transform_time = round(end_transform - end_extract, 4)
logger.info(f"Transform time: {transform_time}")
print(f"Transform time: {transform_time}")
appley = Load()
appley.save_transaction(
transformed_data) # populate RDS instance with cleaned data.
appley.save_drink_menu(transformed_drink_menu_data) # generate drinks menu
end_load = time.time()
load_time = round(end_load - end_transform, 4)
logger.info(f"Loading time: {load_time}")
total_time = extract_time + transform_time + load_time
logger.info(f"total time: {total_time}")
print(f"Load time: {load_time}\nTotal time: {total_time}")
