def test_get_bq_service(): import os old_env = os.environ.copy() if 'GOOGLE_APPLICATION_CREDENTIALS' in old_env: del(old_env['GOOGLE_APPLICATION_CREDENTIALS']) env = patch.dict('os.environ', old_env, clear=True) with env: service = bigquery.get_bq_service() assert service.projects().list().body is None
def wait_for_job(job_id, project_id): bq = bigquery.get_bq_service() start = datetime.datetime.now() counter = 0 while True: time.sleep(1) response = bq.jobs().get( projectId=project_id, jobId=job_id).execute() counter += 1 if response['status']['state'] == 'DONE': if 'errors' in response['status']: error = json.dumps(response['status']['errors'], indent=2) if 'query' in response['configuration']: query = str(response['configuration']['query']['query']) for i, l in enumerate(query.split("\n")): # print SQL query with line numbers for debugging logging.error( error + ":\n" + "{:>3}: {}".format(i + 1, l)) raise StandardError(error) else: break elapsed = (datetime.datetime.now() - start).total_seconds() if 'query' in response['statistics']: bytes_billed = float( response['statistics']['query']['totalBytesBilled']) gb_processed = round(bytes_billed / 1024 / 1024 / 1024, 2) est_cost = round(bytes_billed / 1e+12 * 5.0, 2) # Add our own metadata response['openp'] = {'est_cost': est_cost, 'time': elapsed, 'gb_processed': gb_processed} else: est_cost = 'n/a' logging.warn("Time %ss, cost $%s" % (elapsed, est_cost)) return response
def insert_job(project_id, payload): bq = bigquery.get_bq_service() response = bq.jobs().insert( projectId=project_id, body=payload).execute() return response['jobReference']['jobId']