def get_query_rows(self, job_id, offset=None, limit=None, timeout=0):
"""Retrieve a list of rows from a query table by job id.
This method will append results from multiple pages together. If you want
to manually page through results, you can use `get_query_results`
method directly.
Args:
job_id: The job id that references a BigQuery query.
offset: The offset of the rows to pull from BigQuery.
limit: The number of rows to retrieve from a query table.
timeout: Timeout in seconds.
Returns:
A list of dictionaries that represent table rows.
"""
# Get query results
query_reply = self.get_query_results(job_id, offset=offset, limit=limit, timeout=timeout)
if not query_reply['jobComplete']:
logging.warning('BigQuery job %s not complete' % job_id)
raise UnfinishedQueryException()
schema = query_reply["schema"]["fields"]
rows = query_reply.get('rows', [])
page_token = query_reply.get("pageToken")
records = [self._transform_row(row, schema) for row in rows]
# Append to records if there are multiple pages for query results
while page_token:
query_reply = self.get_query_results(job_id, offset=offset, limit=limit,
page_token=page_token, timeout=timeout)
page_token = query_reply.get("pageToken")
rows = query_reply.get('rows', [])
records += [self._transform_row(row, schema) for row in rows]
return records
def get_query_rows(self, job_id, offset=None, limit=None):
"""Retrieve a list of rows from a query table by job id.
Args:
job_id: The job id that references a BigQuery query.
offset: The offset of the rows to pull from BigQuery.
limit: The number of rows to retrieve from a query table.
Returns:
A list of dictionaries that represent table rows.
"""
job_collection = self.bigquery.jobs()
query_reply = self._get_query_results(
job_collection, self.project_id, job_id, offset=offset,
limit=limit)
if not query_reply['jobComplete']:
logging.warning('BigQuery job %s not complete' % job_id)
raise UnfinishedQueryException()
schema = query_reply['schema']['fields']
rows = query_reply.get('rows', [])
return [self._transform_row(row, schema) for row in rows]
def get_query_schema(self, job_id):
"""Retrieve the schema of a query by job id.
Args:
job_id: The job_id that references a BigQuery query.
Returns:
A list of dictionaries that represent the schema.
"""
query_reply = self.get_query_results(job_id, offset=0, limit=0)
if not query_reply['jobComplete']:
logging.warning('BigQuery job %s not complete' % job_id)
raise UnfinishedQueryException()
return query_reply['schema']['fields']