def get_referencing_views(
view_collector: DirectIngestPreProcessedIngestViewCollector,
) -> Dict[str, List[str]]:
"""Generates a dictionary mapping raw files to ingest views that reference them"""
views_by_raw_file = defaultdict(list)
for builder in view_collector.collect_view_builders():
ingest_view = builder.build()
dependency_configs = ingest_view.raw_table_dependency_configs
for config in dependency_configs:
views_by_raw_file[config.file_tag].append(ingest_view.file_tag)
return views_by_raw_file
)
for metadata in metadata_list
]
if __name__ == "__main__":
# Update these variables and run to print an export query you can run in the BigQuery UI
region_code_: str = "us_mo"
ingest_view_name_: str = "tak001_offender_identification"
upper_bound_datetime_prev_: datetime.datetime = datetime.datetime(2020, 10, 15)
upper_bound_datetime_to_export_: datetime.datetime = datetime.datetime(2020, 12, 18)
with local_project_id_override(GCP_PROJECT_STAGING):
region_ = regions.get_region(region_code_, is_direct_ingest=True)
view_collector_ = DirectIngestPreProcessedIngestViewCollector(region_, [])
views_by_tag_ = {
builder.file_tag: builder.build()
for builder in view_collector_.collect_view_builders()
}
debug_query = DirectIngestIngestViewExportManager.debug_query_for_args(
views_by_tag_,
GcsfsIngestViewExportArgs(
ingest_view_name=ingest_view_name_,
upper_bound_datetime_prev=upper_bound_datetime_prev_,
upper_bound_datetime_to_export=upper_bound_datetime_to_export_,
),
)
print(debug_query)