def generate_per_app_id(self, project_id, baseline_table, output_dir=None):
"""Generate the baseline table query per app_id."""
if not self.per_app_id_enabled:
return
tables = table_names_from_baseline(baseline_table,
include_project_id=False)
init_filename = f"{self.target_table_id}.init.sql"
query_filename = f"{self.target_table_id}.query.sql"
view_filename = f"{self.target_table_id[:-3]}.view.sql"
view_metadata_filename = f"{self.target_table_id[:-3]}.metadata.yaml"
table = tables[f"{self.prefix}_table"]
view = tables[f"{self.prefix}_view"]
render_kwargs = dict(
header="-- Generated via bigquery_etl.glean_usage\n",
project_id=project_id,
)
render_kwargs.update(self.custom_render_kwargs)
render_kwargs.update(tables)
query_sql = render(query_filename, **render_kwargs)
view_sql = render(view_filename, **render_kwargs)
view_metadata = render(view_metadata_filename,
format=False,
**render_kwargs)
if not self.no_init:
try:
init_sql = render(init_filename, **render_kwargs)
except TemplateNotFound:
init_sql = render(query_filename, init=True, **render_kwargs)
if not (referenced_table_exists(view_sql)):
logging.info("Skipping view for table which doesn't exist:"
f" {table}")
return
if output_dir:
write_sql(output_dir, view, "metadata.yaml", view_metadata)
write_sql(output_dir, view, "view.sql", view_sql)
write_sql(output_dir, table, "query.sql", query_sql)
if not self.no_init:
write_sql(output_dir, table, "init.sql", init_sql)
write_dataset_metadata(output_dir, view)
def _write_sql(project, dataset, slug, kwargs, basename, init):
write_sql(
OUTPUT_DIR,
f"{project}.{dataset}.{slug}",
basename,
render(
basename,
template_folder="operational_monitoring",
**kwargs,
init=init,
),
)
def _write_sql(project,
dataset,
slug,
kwargs,
template_filename,
output_filename=None,
init=False):
output_filename = output_filename or template_filename
write_sql(
OUTPUT_DIR,
f"{project}.{dataset}.{slug}",
output_filename,
render(
template_filename,
template_folder="operational_monitoring",
**kwargs,
init=init,
),
)
def generate_per_app(self, project_id, app_info, output_dir=None):
"""Generate the baseline table query per app_name."""
if not self.per_app_enabled:
return
target_view_name = "_".join(self.target_table_id.split("_")[:-1])
target_dataset = app_info[0]["app_name"]
datasets = [(a["bq_dataset_family"], a.get("app_channel", "release"))
for a in app_info]
if len(datasets) == 1 and target_dataset == datasets[0][0]:
# This app only has a single channel, and the app_name
# exactly matches the generated bq_dataset_family, so
# the existing per-app_id dataset also serves as the
# per-app dataset, thus we don't have to provision
# union views.
if self.per_app_id_enabled:
return
render_kwargs = dict(
header="-- Generated via bigquery_etl.glean_usage\n",
project_id=project_id,
target_view=f"{target_dataset}.{target_view_name}",
datasets=datasets,
table=target_view_name,
app_name=app_info[0]["app_name"],
)
sql = render(self.cross_channel_template, **render_kwargs)
view = f"{project_id}.{target_dataset}.{target_view_name}"
if output_dir:
write_dataset_metadata(output_dir, view)
if not (referenced_table_exists(sql)):
logging.info("Skipping view for table which doesn't exist:"
f" {view}")
return
if output_dir:
write_sql(output_dir, view, "view.sql", sql)