def _add_row_to_metadata( *, row: _MetadataRowType, metadata_type: MetadataType, client: bigquery.Client, project_id: str, dry_run: bool, file_tag: str, ) -> None: """Adds the given row to the raw_file_metadata table.""" table_name = get_table_name_for_type(metadata_type=metadata_type) table = _get_table( project_id=project_id, client=client, dataset="direct_ingest_processing_metadata", table_name=table_name, ) row_dict = row.__dict__ if dry_run: logging.info( "[DRY RUN] would have appended rows to table %s:\n%s\n", file_tag, [row_dict], ) return errors = client.insert_rows(table, [row_dict]) if errors: raise ValueError(f"Encountered errors: {errors}") logging.info("Appended rows to table %s:\n%s\n", file_tag, [row])
def scrape( self, bq_client: bigquery.Client, table_path: str, timestamp: datetime.datetime, dry_run: bool = False, ): table = bq_client.get_table(table_path) rows = [{ "provider": self.name, "timestamp": timestamp, **asdict(row), } for row in self.fetch_spaces()] if not dry_run: errors = bq_client.insert_rows(table, rows) if len(errors) > 0: raise ValueError(errors)
def _add_row_to_metadata(*, row: _MetadataRowType, metadata_type: MetadataType, client: bigquery.Client, project_id: str, dry_run: bool, file_tag: str): table_name = get_table_name_for_type(metadata_type=metadata_type) table = _get_table(project_id=project_id, client=client, dataset='direct_ingest_processing_metadata', table_name=table_name) row_dict = row.__dict__ if dry_run: logging.info('[DRY RUN] would have appended rows to table %s:\n%s\n', file_tag, [row_dict]) return errors = client.insert_rows(table, [row_dict]) if errors: raise ValueError(f'Encountered errors: {errors}') logging.info('Appended rows to table %s:\n%s\n', file_tag, [row])