def copy_table(dataset_name, table_name, new_table_name, project=None):
"""Copies a table.
If no project is specified, then the currently active project is used.
"""
bigquery_client = bigquery.Client(project=project)
dataset = bigquery_client.dataset(dataset_name)
table = dataset.table(table_name)
# This sample shows the destination table in the same dataset and project,
# however, it's possible to copy across datasets and projects. You can
# also copy muliple source tables into a single destination table by
# providing addtional arguments to `copy_table`.
destination_table = dataset.table(new_table_name)
# Create a job to copy the table to the destination table.
job_id = str(uuid.uuid4())
job = bigquery_client.copy_table(job_id, destination_table, table)
# Create the table if it doesn't exist.
job.create_disposition = (
google.cloud.bigquery.job.CreateDisposition.CREATE_IF_NEEDED)
# Start the job.
job.begin()
# Wait for the the job to finish.
print('Waiting for job to finish...')
wait_for_job(job)
print('Table {} copied to {}.'.format(table_name, new_table_name))
def copy_table(dataset_name, table_name, new_table_name, project=None):
"""Copies a table.
If no project is specified, then the currently active project is used.
"""
bigquery_client = bigquery.Client(project=project)
dataset = bigquery_client.dataset(dataset_name)
table = dataset.table(table_name)
# This sample shows the destination table in the same dataset and project,
# however, it's possible to copy across datasets and projects. You can
# also copy muliple source tables into a single destination table by
# providing addtional arguments to `copy_table`.
destination_table = dataset.table(new_table_name)
# Create a job to copy the table to the destination table.
job_id = str(uuid.uuid4())
job = bigquery_client.copy_table(
job_id, destination_table, table)
# Create the table if it doesn't exist.
job.create_disposition = (
google.cloud.bigquery.job.CreateDisposition.CREATE_IF_NEEDED)
job.begin() # Start the job.
print('Waiting for job to finish...')
job.result()
print('Table {} copied to {}.'.format(table_name, new_table_name))
def run_query(self, query, use_legacy_sql=False, query_id=str(uuid.uuid4()), use_query_cache=True ,destination_dataset=None, destination_table=None, truncate=False):
"""
Run a query
Args:
query (str): The query to be run
use_legacy_sql (bool): Is the input query written using legacy sql
query_id (str): The unique ID for the query
use_query_cache (bool): Should the query use cached data when available
destination_dataset (str): The dataset the destination_table should be saved to.
destination_table (str): If the data should be saved to a flat table, name it here.
truncate (bool): Should the destination_table be truncated before writing new rows.
Returns:
google.cloud.bigquery.query.QueryResults: The unprocessed results of the query
"""
job = self.client.run_async_query(query_id, query)
job.use_query_cache = use_query_cache
job.use_legacy_sql = use_legacy_sql
if destination_table is not None:
#NOTE: This is always set to true now, why would we ever not want large results, can be parameterized
job.allow_large_results = True
self.dataset = self.client.dataset(destination_dataset) #NOTE: This saved to the class before, is that still needed?
job.destination = self.dataset.table(destination_table)
job.create_disposition = (google.cloud.bigquery.job.CreateDisposition.CREATE_IF_NEEDED) #NOTE: This might need to be parameterized
if truncate:
job.write_disposition = (google.cloud.bigquery.job.WriteDisposition.WRITE_TRUNCATE)
else:
job.write_disposition = (google.cloud.bigquery.job.WriteDisposition.WRITE_APPEND)
if self.verbose: print("Starting Query")
job.begin()
self.wait_for_job(job)
results = job.query_results()
return results
def copyTable(projectFrom, projectTo, datasetFrom, datasetTo, tableName):
table_source = datasetFrom.table(tableName)
table_destination = datasetTo.table(tableName)
job_id = str(uuid.uuid4())
job = projectFrom.copy_table(job_id, table_destination, table_source)
job.create_disposition = (
google.cloud.bigquery.job.CreateDisposition.CREATE_IF_NEEDED)
job.begin() # Start the job.
print('Waiting for job to finish...')
job.result()
print('Table {} copied to {}.'.format(tableName, tableName))