def _add_row_to_metadata(
*,
row: _MetadataRowType,
metadata_type: MetadataType,
client: bigquery.Client,
project_id: str,
dry_run: bool,
file_tag: str,
) -> None:
"""Adds the given row to the raw_file_metadata table."""
table_name = get_table_name_for_type(metadata_type=metadata_type)
table = _get_table(
project_id=project_id,
client=client,
dataset="direct_ingest_processing_metadata",
table_name=table_name,
)
row_dict = row.__dict__
if dry_run:
logging.info(
"[DRY RUN] would have appended rows to table %s:\n%s\n",
file_tag,
[row_dict],
)
return
errors = client.insert_rows(table, [row_dict])
if errors:
raise ValueError(f"Encountered errors: {errors}")
logging.info("Appended rows to table %s:\n%s\n", file_tag, [row])
def scrape(
self,
bq_client: bigquery.Client,
table_path: str,
timestamp: datetime.datetime,
dry_run: bool = False,
):
table = bq_client.get_table(table_path)
rows = [{
"provider": self.name,
"timestamp": timestamp,
**asdict(row),
} for row in self.fetch_spaces()]
if not dry_run:
errors = bq_client.insert_rows(table, rows)
if len(errors) > 0:
raise ValueError(errors)
def _add_row_to_metadata(*, row: _MetadataRowType, metadata_type: MetadataType,
client: bigquery.Client, project_id: str,
dry_run: bool, file_tag: str):
table_name = get_table_name_for_type(metadata_type=metadata_type)
table = _get_table(project_id=project_id,
client=client,
dataset='direct_ingest_processing_metadata',
table_name=table_name)
row_dict = row.__dict__
if dry_run:
logging.info('[DRY RUN] would have appended rows to table %s:\n%s\n',
file_tag, [row_dict])
return
errors = client.insert_rows(table, [row_dict])
if errors:
raise ValueError(f'Encountered errors: {errors}')
logging.info('Appended rows to table %s:\n%s\n', file_tag, [row])
