def execute(self, context):
if self.hook is None:
self.hook = GoogleDisplayVideo360Hook(gcp_conn_id=self.gcp_conn_id)
if self.bq_hook is None:
self.bq_hook = BigQueryHook(bigquery_conn_id=self.gcp_conn_id)
if self.gcs_hook is None:
self.gcs_hook = GoogleCloudStorageHook(
google_cloud_storage_conn_id=self.gcp_conn_id)
request_body = {
'fileTypes': self.file_types,
'filterType': self.filter_type,
'filterIds': self.filter_ids,
'version': self.api_version
}
logger.info('Request body: %s ' % request_body)
request = self.hook.get_service().sdf().download(body=request_body)
response = request.execute()
for file_type in self.file_types:
temp_file = None
try:
logger.info('Uploading SDF to GCS')
temp_file = tempfile.NamedTemporaryFile(delete=False)
response_key = self.sdf_api_response_keys.get(file_type)
temp_file.write(response[response_key].encode('utf-8'))
temp_file.close()
filename = '%d_%s_%s_%s.json' % (time.time() * 1e+9,
randint(1, 1000000),
response_key, 'sdf')
self.gcs_hook.upload(self.gcs_bucket, filename, temp_file.name)
logger.info('SDF upload to GCS complete')
finally:
if temp_file:
temp_file.close()
os.unlink(temp_file.name)
sdf_file = 'gs://%s/%s' % (self.gcs_bucket, filename)
bq_table = self.table_names.get(file_type)
bq_table = '%s.%s' % (self.bq_dataset, bq_table)
schema = SDF_VERSIONED_SCHEMA_TYPES.get(
self.api_version).get(file_type)
try:
bq_base_cursor = self.bq_hook.get_conn().cursor()
logger.info('Uploading SDF to BigQuery')
bq_base_cursor.run_load(
destination_project_dataset_table=bq_table,
schema_fields=schema,
source_uris=[sdf_file],
source_format='CSV',
skip_leading_rows=1,
write_disposition=self.write_disposition)
finally:
logger.info('Deleting SDF from GCS')
self.gcs_hook.delete(self.gcs_bucket, filename)
def execute(self, context):
if self.service is None:
hook = GoogleDisplayVideo360Hook(api_version=self.api_version,
gcp_conn_id=self.conn_id,
delegate_to=self.delegate_to)
self.service = hook.get_service()
request = self.service.queries().runquery(queryId=self.query_id,
body={})
request.execute()
def execute(self, context):
if self.hook is None:
self.hook = GoogleDisplayVideo360Hook(
gcp_conn_id=self.gcp_conn_id,
delegate_to=self.delegate_to)
report_body = json.loads(self.report)
request = self.hook.get_service().queries().createquery(body=report_body)
response = request.execute()
context['task_instance'].xcom_push('query_id', response['queryId'])
def execute(self, context):
if self.hook is None:
self.hook = GoogleDisplayVideo360Hook(gcp_conn_id=self.gcp_conn_id,
api_version=self.api_version,
delegate_to=self.delegate_to)
if self.query_id is not None:
self.hook.deletequery(self.query_id,
ignore_if_missing=self.ignore_if_missing)
if self.query_title is not None:
self.hook.deletequeries(self.query_title,
ignore_if_missing=self.ignore_if_missing)
def poke(self, context):
if self.hook is None:
self.hook = GoogleDisplayVideo360Hook(gcp_conn_id=self.gcp_conn_id,
delegate_to=self.delegate_to)
request = self.hook.get_service().queries().getquery(
queryId=self.query_id)
response = request.execute()
if response:
metadata = response.get('metadata')
if metadata and not metadata.get('running'):
report_url = metadata.get(
u'googleCloudStoragePathForLatestReport')
if report_url:
context['task_instance'].xcom_push('report_url',
report_url)
return True
return False