def replace_single_day(
bq_client,
datasource,
dt,
project_id=DEFAULT_BQ_PROJECT,
dataset_id=DEFAULT_BQ_DATASET,
table_id=DEFAULT_BQ_TABLE,
):
model_date = date.fromisoformat(dt)
data = {}
kpi_data = get_kpi_data(bq_client, types=[datasource])
data.update(kpi_data)
if datasource.lower() == 'mobile':
nondesktop_data = get_nondesktop_data(bq_client)
data.update(nondesktop_data)
if datasource.lower() == 'fxa':
fxasub_data = get_fxasub_data(bq_client)
data.update(fxasub_data)
partition_decorator = "$" + model_date.isoformat().replace('-', '')
table = '.'.join([project_id, dataset_id, table_id]) + partition_decorator
records = []
for product in data.keys():
logging.info("Processing {} forecast for {}".format(
product, model_date))
records += prepare_records(model_date, FORECAST_HORIZON, data[product],
product)
logging.info("Replacing results for {} in {}".format(model_date, table))
write_records(
bq_client,
records,
table,
write_disposition=bigquery.job.WriteDisposition.WRITE_TRUNCATE)
def update_table(bq_client,
project_id=DEFAULT_BQ_PROJECT,
dataset_id=DEFAULT_BQ_DATASET,
table_id=DEFAULT_BQ_TABLE):
kpi_data = get_kpi_data(bq_client)
nondesktop_data = get_nondesktop_data(bq_client)
data = kpi_data
data.update(nondesktop_data)
dataset = bq_client.dataset(dataset_id)
tableref = dataset.table(table_id)
table = bq_client.get_table(tableref)
for product in data.keys():
logging.info("Processing forecasts for {}".format(product))
latest_date = get_latest_date(bq_client, project_id, dataset_id,
table_id, product, "asofdate")
if latest_date is not None:
start_date = latest_date + timedelta(days=1)
else:
start_date = FIRST_MODEL_DATES[product]
model_dates = pd.date_range(start_date, data[product].ds.max())
for model_date in model_dates:
logging.info("Processing {} forecast for {}".format(
product, model_date))
write_forecasts(bq_client, table, model_date.date(),
FORECAST_HORIZON, data[product], product)
def replace_table(bq_client,
project_id=DEFAULT_BQ_PROJECT,
dataset_id=DEFAULT_BQ_DATASET,
table_id=DEFAULT_BQ_TABLE):
kpi_data = get_kpi_data(bq_client)
nondesktop_data = get_nondesktop_data(bq_client)
data = kpi_data
data.update(nondesktop_data)
table = reset_output_table(bq_client, project_id, dataset_id, table_id)
for product in data.keys():
logging.info("Processing forecasts for {}".format(product))
model_dates = pd.date_range(FIRST_MODEL_DATES[product],
data[product].ds.max())
for model_date in model_dates:
logging.info("Processing {} forecast for {}".format(
product, model_date))
write_forecasts(bq_client, table, model_date.date(),
FORECAST_HORIZON, data[product], product)