def rebuild_ft_data(process_day, service):
deleted_rows = delete_billing_data_for_service_for_day(
process_day, service)
current_app.logger.info(
'deleted {} existing billing rows for {} on {}'.format(
deleted_rows, service, process_day))
transit_data = fetch_billing_data_for_day(process_day=process_day,
service_id=service)
# transit_data = every row that should exist
for data in transit_data:
# upsert existing rows
update_fact_billing(data, process_day)
current_app.logger.info(
'added/updated {} billing rows for {} on {}'.format(
len(transit_data), service, process_day))
def create_nightly_billing_for_day(process_day):
process_day = datetime.strptime(process_day, "%Y-%m-%d").date()
start = datetime.utcnow()
transit_data = fetch_billing_data_for_day(process_day=process_day)
end = datetime.utcnow()
current_app.logger.info("create-nightly-billing-for-day {} fetched in {} seconds".format(process_day, (end - start).seconds))
for data in transit_data:
update_fact_billing(data, process_day)
current_app.logger.info(
"create-nightly-billing-for-day task complete. {} rows updated for day: {}".format(len(transit_data), process_day)
)
def create_nightly_billing_for_day(process_day):
process_day = datetime.strptime(process_day, "%Y-%m-%d").date()
current_app.logger.info(
f'create-nightly-billing-for-day task for {process_day}: started')
start = datetime.utcnow()
transit_data = fetch_billing_data_for_day(process_day=process_day)
end = datetime.utcnow()
current_app.logger.info(
f'create-nightly-billing-for-day task for {process_day}: data fetched in {(end - start).seconds} seconds'
)
for data in transit_data:
update_fact_billing(data, process_day)
current_app.logger.info(
f"create-nightly-billing-for-day task for {process_day}: "
f"task complete. {len(transit_data)} rows updated")
def create_nightly_billing(day_start=None):
# day_start is a datetime.date() object. e.g.
# 3 days of data counting back from day_start is consolidated
if day_start is None:
day_start = datetime.today() - timedelta(days=1)
else:
# When calling the task its a string in the format of "YYYY-MM-DD"
day_start = datetime.strptime(day_start, "%Y-%m-%d")
for i in range(0, 10):
process_day = day_start - timedelta(days=i)
transit_data = fetch_billing_data_for_day(process_day=process_day)
for data in transit_data:
update_fact_billing(data, process_day)
current_app.logger.info(
"create-nightly-billing task complete. {} rows updated for day: {}"
.format(len(transit_data), process_day))
