def __init__(self,
*,
project_id: str = None,
region_code: str,
view_id: str,
view_query_template: str,
raw_file_config: DirectIngestRawFileConfig):
if not raw_file_config.primary_key_cols:
raise ValueError(
f'Empty primary key list in raw file config with tag [{raw_file_config.file_tag}] during '
f'construction of DirectIngestRawDataTableBigQueryView')
view_dataset_id = f'{region_code.lower()}_raw_data_up_to_date_views'
raw_table_dataset_id = DirectIngestRawFileImportManager.raw_tables_dataset_for_region(
region_code)
except_clause = self._except_clause_for_config(raw_file_config)
datetime_cols_clause = self._datetime_cols_clause_for_config(
raw_file_config)
supplemental_order_by_clause = self._supplemental_order_by_clause_for_config(
raw_file_config)
super().__init__(
project_id=project_id,
dataset_id=view_dataset_id,
view_id=view_id,
view_query_template=view_query_template,
raw_table_dataset_id=raw_table_dataset_id,
raw_table_name=raw_file_config.file_tag,
raw_table_primary_key_str=raw_file_config.primary_key_str,
except_clause=except_clause,
datetime_cols_clause=datetime_cols_clause,
supplemental_order_by_clause=supplemental_order_by_clause)
def __init__(self,
*,
project_id: str = None,
region_code: str,
view_id: str,
view_query_template: str,
raw_file_config: DirectIngestRawFileConfig):
view_dataset_id = f'{region_code.lower()}_raw_data_up_to_date_views'
raw_table_dataset_id = DirectIngestRawFileImportManager.raw_tables_dataset_for_region(
region_code)
except_clause = self._except_clause_for_config(raw_file_config)
datetime_cols_clause = self._datetime_cols_clause_for_config(
raw_file_config)
supplemental_order_by_clause = self._supplemental_order_by_clause_for_config(
raw_file_config)
super().__init__(
project_id=project_id,
dataset_id=view_dataset_id,
view_id=view_id,
view_query_template=view_query_template,
raw_table_dataset_id=raw_table_dataset_id,
raw_table_name=raw_file_config.file_tag,
raw_table_primary_key_str=raw_file_config.primary_key_str,
except_clause=except_clause,
datetime_cols_clause=datetime_cols_clause,
supplemental_order_by_clause=supplemental_order_by_clause)
def compare_raw_data_between_projects(
region_code: str,
source_project_id: str = environment.GCP_PROJECT_STAGING,
comparison_project_id: str = environment.GCP_PROJECT_PRODUCTION,
) -> List[str]:
"""Compares the raw data between staging and production for a given region."""
logging.info(
"**** Ensuring all raw data for [%s] in [%s] also exists in [%s] ****",
region_code.upper(),
source_project_id,
comparison_project_id,
)
raw_file_config = DirectIngestRegionRawFileConfig(region_code)
bq_client = BigQueryClientImpl(project_id=source_project_id)
dataset_id = DirectIngestRawFileImportManager.raw_tables_dataset_for_region(
region_code)
source_dataset = bq_client.dataset_ref_for_id(dataset_id)
query_jobs: Dict[str, bigquery.QueryJob] = {}
for file_tag, file_config in raw_file_config.raw_file_configs.items():
if (not bq_client.table_exists(source_dataset, file_tag)
or file_config.is_undocumented
or not file_config.primary_key_cols):
continue
columns = ", ".join(
[column.name for column in file_config.available_columns])
query_job = bq_client.run_query_async(
query_str=COMPARISON_TEMPLATE.format(
source_project_id=source_project_id,
comparison_project_id=comparison_project_id,
raw_data_dataset_id=dataset_id,
raw_data_table_id=file_tag,
columns=columns,
))
query_jobs[file_tag] = query_job
table_column_width = min(
max(len(tag) for tag in raw_file_config.raw_file_configs), 30)
failed_tables: List[str] = []
for file_tag in sorted(raw_file_config.raw_file_tags):
justified_name = file_tag.ljust(table_column_width)
if file_tag not in query_jobs:
# This file did not exist in the project that is the source of truth.
continue
query_job = query_jobs[file_tag]
try:
rows = query_job.result()
except exceptions.NotFound:
logging.warning(
"%s | Missing table %s.%s.%s",
justified_name,
comparison_project_id,
dataset_id,
file_tag,
)
failed_tables.append(file_tag)
continue
counts: List[Tuple[datetime.datetime,
int]] = [row.values() for row in rows]
if counts:
logging.warning(
"%s | Missing data in the %s table",
justified_name,
comparison_project_id,
)
for update_datetime, num_missing in counts:
logging.warning("\t%ss: %d", update_datetime.isoformat(),
num_missing)
failed_tables.append(file_tag)
else:
logging.info(
"%s | %s contains all of the data from %s",
justified_name,
comparison_project_id,
source_project_id,
)
return failed_tables