def execute(self, context: dict) -> str:
hook = GoogleDisplayVideo360Hook(
gcp_conn_id=self.gcp_conn_id,
delegate_to=self.delegate_to,
api_version=self.api_version,
impersonation_chain=self.impersonation_chain,
)
gcs_hook = GCSHook(
gcp_conn_id=self.gcp_conn_id,
delegate_to=self.delegate_to,
impersonation_chain=self.impersonation_chain,
)
self.log.info("Retrieving operation...")
operation = hook.get_sdf_download_operation(
operation_name=self.operation_name)
self.log.info("Creating file for upload...")
media = hook.download_media(resource_name=operation)
self.log.info("Sending file to the Google Cloud Storage...")
with tempfile.NamedTemporaryFile() as temp_file:
hook.download_content_from_request(temp_file,
media,
chunk_size=1024 * 1024)
temp_file.flush()
gcs_hook.upload(
bucket_name=self.bucket_name,
object_name=self.object_name,
filename=temp_file.name,
gzip=self.gzip,
)
return f"{self.bucket_name}/{self.object_name}"
def execute(self, context: dict) -> str:
gcs_hook = GCSHook(
gcp_conn_id=self.gcp_conn_id,
delegate_to=self.delegate_to,
impersonation_chain=self.impersonation_chain,
)
hook = GoogleDisplayVideo360Hook(
gcp_conn_id=self.gcp_conn_id,
api_version=self.api_version,
delegate_to=self.delegate_to,
impersonation_chain=self.impersonation_chain,
)
self.log.info("Retrieving report...")
content: List[str] = hook.download_line_items(
request_body=self.request_body)
with tempfile.NamedTemporaryFile("w+") as temp_file:
writer = csv.writer(temp_file)
writer.writerows(content)
temp_file.flush()
gcs_hook.upload(
bucket_name=self.bucket_name,
object_name=self.object_name,
filename=temp_file.name,
mime_type="text/csv",
gzip=self.gzip,
)
return f"{self.bucket_name}/{self.object_name}"
def poke(self, context: Dict) -> bool:
hook = GoogleDisplayVideo360Hook(
gcp_conn_id=self.gcp_conn_id,
delegate_to=self.delegate_to,
api_version=self.api_version,
)
response = hook.get_query(query_id=self.report_id)
if response and not response.get("metadata", {}).get("running"):
return True
return False
def execute(self, context: Dict):
hook = GoogleDisplayVideo360Hook(
gcp_conn_id=self.gcp_conn_id,
delegate_to=self.delegate_to,
api_version=self.api_version,
)
self.log.info("Creating Display & Video 360 report.")
response = hook.create_query(query=self.body)
report_id = response["queryId"]
self.xcom_push(context, key="report_id", value=report_id)
self.log.info("Created report with ID: %s", report_id)
return response
def execute(self, context: Dict):
hook = GoogleDisplayVideo360Hook(
gcp_conn_id=self.gcp_conn_id,
delegate_to=self.delegate_to,
api_version=self.api_version,
)
self.log.info(
"Running report %s with the following params:\n %s",
self.report_id,
self.params,
)
hook.run_query(query_id=self.report_id, params=self.params)
def execute(self, context: dict) -> Dict[str, Any]:
hook = GoogleDisplayVideo360Hook(
gcp_conn_id=self.gcp_conn_id,
delegate_to=self.delegate_to,
api_version=self.api_version,
impersonation_chain=self.impersonation_chain,
)
self.log.info("Creating operation for SDF download task...")
operation = hook.create_sdf_download_operation(body_request=self.body_request)
return operation
def execute(self, context: 'Context') -> None:
hook = GoogleDisplayVideo360Hook(
gcp_conn_id=self.gcp_conn_id,
delegate_to=self.delegate_to,
api_version=self.api_version,
impersonation_chain=self.impersonation_chain,
)
self.log.info(
"Running report %s with the following params:\n %s",
self.report_id,
self.parameters,
)
hook.run_query(query_id=self.report_id, params=self.parameters)
def poke(self, context: Dict) -> bool:
hook = GoogleDisplayVideo360Hook(
gcp_conn_id=self.gcp_conn_id,
delegate_to=self.delegate_to,
api_version=self.api_version,
)
operation = hook.get_sdf_download_operation(operation_name=self.operation_name)
if "error" in operation:
raise AirflowException(f'The operation finished in error with {operation["error"]}')
if operation and operation.get("done"):
return True
return False
def execute(self, context: 'Context'):
hook = GoogleDisplayVideo360Hook(
gcp_conn_id=self.gcp_conn_id,
delegate_to=self.delegate_to,
api_version=self.api_version,
impersonation_chain=self.impersonation_chain,
)
gcs_hook = GCSHook(
gcp_conn_id=self.gcp_conn_id,
delegate_to=self.delegate_to,
impersonation_chain=self.impersonation_chain,
)
resource = hook.get_query(query_id=self.report_id)
# Check if report is ready
if resource["metadata"]["running"]:
raise AirflowException(f"Report {self.report_id} is still running")
# If no custom report_name provided, use DV360 name
file_url = resource["metadata"][
"googleCloudStoragePathForLatestReport"]
report_name = self.report_name or urlparse(file_url).path.split(
"/")[-1]
report_name = self._resolve_file_name(report_name)
# Download the report
self.log.info("Starting downloading report %s", self.report_id)
with tempfile.NamedTemporaryFile(delete=False) as temp_file:
with urllib.request.urlopen(file_url) as response:
shutil.copyfileobj(response, temp_file, length=self.chunk_size)
temp_file.flush()
# Upload the local file to bucket
bucket_name = self._set_bucket_name(self.bucket_name)
gcs_hook.upload(
bucket_name=bucket_name,
object_name=report_name,
gzip=self.gzip,
filename=temp_file.name,
mime_type="text/csv",
)
self.log.info(
"Report %s was saved in bucket %s as %s.",
self.report_id,
self.bucket_name,
report_name,
)
self.xcom_push(context, key="report_name", value=report_name)
def execute(self, context: 'Context') -> Dict[str, Any]:
hook = GoogleDisplayVideo360Hook(
gcp_conn_id=self.gcp_conn_id,
delegate_to=self.delegate_to,
api_version=self.api_version,
impersonation_chain=self.impersonation_chain,
)
self.log.info("Creating operation for SDF download task...")
operation = hook.create_sdf_download_operation(
body_request=self.body_request)
name = operation["name"]
self.xcom_push(context, key="name", value=name)
self.log.info("Created SDF operation with name: %s", name)
return operation
def execute(self, context: dict) -> None:
hook = GoogleDisplayVideo360Hook(
gcp_conn_id=self.gcp_conn_id,
delegate_to=self.delegate_to,
api_version=self.api_version,
impersonation_chain=self.impersonation_chain,
)
if self.report_id:
reports_ids_to_delete = [self.report_id]
else:
reports = hook.list_queries()
reports_ids_to_delete = [
report["queryId"] for report in reports if report["metadata"]["title"] == self.report_name
]
for report_id in reports_ids_to_delete:
self.log.info("Deleting report with id: %s", report_id)
hook.delete_query(query_id=report_id)
self.log.info("Report deleted.")
def execute(self, context: Dict):
gcs_hook = GCSHook(gcp_conn_id=self.gcp_conn_id,
delegate_to=self.delegate_to)
hook = GoogleDisplayVideo360Hook(
gcp_conn_id=self.gcp_conn_id,
delegate_to=self.delegate_to,
api_version=self.api_version,
)
self.log.info("Uploading file %s...")
# Saving file in the temporary directory,
# downloaded file from the GCS could be a 1GB size or even more
with tempfile.NamedTemporaryFile("w+") as f:
line_items = gcs_hook.download(
bucket_name=self.bucket_name,
object_name=self.object_name,
filename=f.name,
)
f.flush()
hook.upload_line_items(line_items=line_items)
def setUp(self):
with mock.patch(
"airflow.providers.google.common.hooks.base_google.GoogleBaseHook.__init__",
new=mock_base_gcp_hook_default_project_id,
):
self.hook = GoogleDisplayVideo360Hook(gcp_conn_id=GCP_CONN_ID)
GoogleDisplayVideo360ReportSensor,
)
from airflow.utils import dates
# [START howto_display_video_env_variables]
BUCKET = os.environ.get("GMP_DISPLAY_VIDEO_BUCKET", "gs://INVALID BUCKET NAME")
ADVERTISER_ID = os.environ.get("GMP_ADVERTISER_ID", 1234567)
OBJECT_NAME = os.environ.get("GMP_OBJECT_NAME", "files/report.csv")
PATH_TO_UPLOAD_FILE = os.environ.get("GCP_GCS_PATH_TO_UPLOAD_FILE", "test-gcs-example.txt")
PATH_TO_SAVED_FILE = os.environ.get("GCP_GCS_PATH_TO_SAVED_FILE", "test-gcs-example-download.txt")
BUCKET_FILE_LOCATION = PATH_TO_UPLOAD_FILE.rpartition("/")[-1]
SDF_VERSION = os.environ.get("GMP_SDF_VERSION", "SDF_VERSION_5_1")
BQ_DATA_SET = os.environ.get("GMP_BQ_DATA_SET", "airflow_test")
GMP_PARTNER_ID = os.environ.get("GMP_PARTNER_ID", 123)
ENTITY_TYPE = os.environ.get("GMP_ENTITY_TYPE", "LineItem")
ERF_SOURCE_OBJECT = GoogleDisplayVideo360Hook.erf_uri(GMP_PARTNER_ID, ENTITY_TYPE)
REPORT = {
"kind": "doubleclickbidmanager#query",
"metadata": {
"title": "Polidea Test Report",
"dataRange": "LAST_7_DAYS",
"format": "CSV",
"sendNotification": False,
},
"params": {
"type": "TYPE_GENERAL",
"groupBys": ["FILTER_DATE", "FILTER_PARTNER"],
"filters": [{"type": "FILTER_PARTNER", "value": 1486931}],
"metrics": ["METRIC_IMPRESSIONS", "METRIC_CLICKS"],
"includeInviteData": True,
def setUp(self):
with mock.patch(
"airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook.__init__",
new=mock_base_gcp_hook_default_project_id,
):
self.hook = GoogleDisplayVideo360Hook(gcp_conn_id=GCP_CONN_ID)
