def initialize(self) -> bigquery.table.RowIterator:
"""Creates, if not found, a table.
Returns:
google.cloud.bigquery.table.RowIterator -- Result of the query.
Since this is a DDL query, this will always be empty if
it succeeded.
Raises:
google.cloud.exceptions.GoogleCloudError –- If the job failed.
concurrent.futures.TimeoutError –- If the job did not complete
in the default BigQuery job timeout.
"""
if not self.schema:
raise ValueError(
"No schema provided for table {}; writing is not supported.".
format(self.short_name))
bq_client = get_bq_client()
LOG.info("Creating table %s if not found.",
self.get_fully_qualified_name())
querytext = """
CREATE TABLE IF NOT EXISTS `{}` (
{}
)""".format(self.get_fully_qualified_name(), self.schema)
LOG.debug("Running query: \n%s", querytext)
query_job = bq_client.query(querytext)
return query_job.result()
def run_query_job(querytext: str,
temp_table: str = None,
query_job_config: QueryJobConfig = QueryJobConfig()
) -> QueryJob:
"""
Set up and run a query job.
Arguments:
querytext {str} -- The querytext for the job.
Keyword Arguments:
temp_table {str} -- A temporary table in which to materialize results.
The results will be streamed from this table when done. This is
required for all large queries, and strongly recommended.
(default: {None})
query_job_config {QueryJobConfig} -- A QueryJobConfig to start from.
(default: {QueryJobConfig()})
Returns:
QueryJob -- The resulting job.
"""
LOG.debug("Running query: %s", querytext)
client = get_bq_client()
if temp_table:
query_job_config.destination = temp_table
query_job_config.write_disposition = WriteDisposition.WRITE_TRUNCATE
return client.query(query=querytext, job_config=query_job_config)
def __init__(self, table: Table):
self.config = get_config()
self.lock = Lock()
self.client = get_bq_client()
self.rows = list()
self.tablename = table.get_fully_qualified_name()
self.batch_size = int(
self.config.get('BIGQUERY', 'BATCH_WRITE_SIZE', fallback=100))
self.insert_count = 0
table.initialize()
def drop(self) -> bigquery.table.RowIterator:
"""DROPs (deletes) the table. This cannot be undone.
Returns:
google.cloud.bigquery.table.RowIterator -- Result of the query.
Since this is a DDL query, this will always be empty if
it succeeded.
Raises:
google.cloud.exceptions.GoogleCloudError –- If the job failed.
concurrent.futures.TimeoutError –- If the job did not complete
in the default BigQuery job timeout.
"""
bq_client = get_bq_client()
LOG.info("Deleting table %s", self.get_fully_qualified_name())
querytext = "DROP TABLE `{}`".format(self.get_fully_qualified_name())
LOG.debug("Running query: \n%s", querytext)
query_job = bq_client.query(querytext)
return query_job.result()