def test_get_bq_service():
import os
old_env = os.environ.copy()
if 'GOOGLE_APPLICATION_CREDENTIALS' in old_env:
del(old_env['GOOGLE_APPLICATION_CREDENTIALS'])
env = patch.dict('os.environ', old_env, clear=True)
with env:
service = bigquery.get_bq_service()
assert service.projects().list().body is None
def wait_for_job(job_id, project_id):
bq = bigquery.get_bq_service()
start = datetime.datetime.now()
counter = 0
while True:
time.sleep(1)
response = bq.jobs().get(
projectId=project_id,
jobId=job_id).execute()
counter += 1
if response['status']['state'] == 'DONE':
if 'errors' in response['status']:
error = json.dumps(response['status']['errors'], indent=2)
if 'query' in response['configuration']:
query = str(response['configuration']['query']['query'])
for i, l in enumerate(query.split("\n")):
# print SQL query with line numbers for debugging
logging.error(
error + ":\n" + "{:>3}: {}".format(i + 1, l))
raise StandardError(error)
else:
break
elapsed = (datetime.datetime.now() - start).total_seconds()
if 'query' in response['statistics']:
bytes_billed = float(
response['statistics']['query']['totalBytesBilled'])
gb_processed = round(bytes_billed / 1024 / 1024 / 1024, 2)
est_cost = round(bytes_billed / 1e+12 * 5.0, 2)
# Add our own metadata
response['openp'] = {'est_cost': est_cost,
'time': elapsed,
'gb_processed': gb_processed}
else:
est_cost = 'n/a'
logging.warn("Time %ss, cost $%s" % (elapsed, est_cost))
return response
def wait_for_job(job_id, project_id):
bq = bigquery.get_bq_service()
start = datetime.datetime.now()
counter = 0
while True:
time.sleep(1)
response = bq.jobs().get(
projectId=project_id,
jobId=job_id).execute()
counter += 1
if response['status']['state'] == 'DONE':
if 'errors' in response['status']:
error = json.dumps(response['status']['errors'], indent=2)
if 'query' in response['configuration']:
query = str(response['configuration']['query']['query'])
for i, l in enumerate(query.split("\n")):
# print SQL query with line numbers for debugging
logging.error(
error + ":\n" + "{:>3}: {}".format(i + 1, l))
raise StandardError(error)
else:
break
elapsed = (datetime.datetime.now() - start).total_seconds()
if 'query' in response['statistics']:
bytes_billed = float(
response['statistics']['query']['totalBytesBilled'])
gb_processed = round(bytes_billed / 1024 / 1024 / 1024, 2)
est_cost = round(bytes_billed / 1e+12 * 5.0, 2)
# Add our own metadata
response['openp'] = {'est_cost': est_cost,
'time': elapsed,
'gb_processed': gb_processed}
else:
est_cost = 'n/a'
logging.warn("Time %ss, cost $%s" % (elapsed, est_cost))
return response
def insert_job(project_id, payload):
bq = bigquery.get_bq_service()
response = bq.jobs().insert(
projectId=project_id,
body=payload).execute()
return response['jobReference']['jobId']
def insert_job(project_id, payload):
bq = bigquery.get_bq_service()
response = bq.jobs().insert(
projectId=project_id,
body=payload).execute()
return response['jobReference']['jobId']