def execute(self, context: dict) -> str: hook = GoogleDisplayVideo360Hook( gcp_conn_id=self.gcp_conn_id, delegate_to=self.delegate_to, api_version=self.api_version, impersonation_chain=self.impersonation_chain, ) gcs_hook = GCSHook( gcp_conn_id=self.gcp_conn_id, delegate_to=self.delegate_to, impersonation_chain=self.impersonation_chain, ) self.log.info("Retrieving operation...") operation = hook.get_sdf_download_operation( operation_name=self.operation_name) self.log.info("Creating file for upload...") media = hook.download_media(resource_name=operation) self.log.info("Sending file to the Google Cloud Storage...") with tempfile.NamedTemporaryFile() as temp_file: hook.download_content_from_request(temp_file, media, chunk_size=1024 * 1024) temp_file.flush() gcs_hook.upload( bucket_name=self.bucket_name, object_name=self.object_name, filename=temp_file.name, gzip=self.gzip, ) return f"{self.bucket_name}/{self.object_name}"
def execute(self, context: dict) -> str: gcs_hook = GCSHook( gcp_conn_id=self.gcp_conn_id, delegate_to=self.delegate_to, impersonation_chain=self.impersonation_chain, ) hook = GoogleDisplayVideo360Hook( gcp_conn_id=self.gcp_conn_id, api_version=self.api_version, delegate_to=self.delegate_to, impersonation_chain=self.impersonation_chain, ) self.log.info("Retrieving report...") content: List[str] = hook.download_line_items( request_body=self.request_body) with tempfile.NamedTemporaryFile("w+") as temp_file: writer = csv.writer(temp_file) writer.writerows(content) temp_file.flush() gcs_hook.upload( bucket_name=self.bucket_name, object_name=self.object_name, filename=temp_file.name, mime_type="text/csv", gzip=self.gzip, ) return f"{self.bucket_name}/{self.object_name}"
def poke(self, context: Dict) -> bool: hook = GoogleDisplayVideo360Hook( gcp_conn_id=self.gcp_conn_id, delegate_to=self.delegate_to, api_version=self.api_version, ) response = hook.get_query(query_id=self.report_id) if response and not response.get("metadata", {}).get("running"): return True return False
def execute(self, context: Dict): hook = GoogleDisplayVideo360Hook( gcp_conn_id=self.gcp_conn_id, delegate_to=self.delegate_to, api_version=self.api_version, ) self.log.info("Creating Display & Video 360 report.") response = hook.create_query(query=self.body) report_id = response["queryId"] self.xcom_push(context, key="report_id", value=report_id) self.log.info("Created report with ID: %s", report_id) return response
def execute(self, context: Dict): hook = GoogleDisplayVideo360Hook( gcp_conn_id=self.gcp_conn_id, delegate_to=self.delegate_to, api_version=self.api_version, ) self.log.info( "Running report %s with the following params:\n %s", self.report_id, self.params, ) hook.run_query(query_id=self.report_id, params=self.params)
def execute(self, context: dict) -> Dict[str, Any]: hook = GoogleDisplayVideo360Hook( gcp_conn_id=self.gcp_conn_id, delegate_to=self.delegate_to, api_version=self.api_version, impersonation_chain=self.impersonation_chain, ) self.log.info("Creating operation for SDF download task...") operation = hook.create_sdf_download_operation(body_request=self.body_request) return operation
def execute(self, context: 'Context') -> None: hook = GoogleDisplayVideo360Hook( gcp_conn_id=self.gcp_conn_id, delegate_to=self.delegate_to, api_version=self.api_version, impersonation_chain=self.impersonation_chain, ) self.log.info( "Running report %s with the following params:\n %s", self.report_id, self.parameters, ) hook.run_query(query_id=self.report_id, params=self.parameters)
def poke(self, context: Dict) -> bool: hook = GoogleDisplayVideo360Hook( gcp_conn_id=self.gcp_conn_id, delegate_to=self.delegate_to, api_version=self.api_version, ) operation = hook.get_sdf_download_operation(operation_name=self.operation_name) if "error" in operation: raise AirflowException(f'The operation finished in error with {operation["error"]}') if operation and operation.get("done"): return True return False
def execute(self, context: 'Context'): hook = GoogleDisplayVideo360Hook( gcp_conn_id=self.gcp_conn_id, delegate_to=self.delegate_to, api_version=self.api_version, impersonation_chain=self.impersonation_chain, ) gcs_hook = GCSHook( gcp_conn_id=self.gcp_conn_id, delegate_to=self.delegate_to, impersonation_chain=self.impersonation_chain, ) resource = hook.get_query(query_id=self.report_id) # Check if report is ready if resource["metadata"]["running"]: raise AirflowException(f"Report {self.report_id} is still running") # If no custom report_name provided, use DV360 name file_url = resource["metadata"][ "googleCloudStoragePathForLatestReport"] report_name = self.report_name or urlparse(file_url).path.split( "/")[-1] report_name = self._resolve_file_name(report_name) # Download the report self.log.info("Starting downloading report %s", self.report_id) with tempfile.NamedTemporaryFile(delete=False) as temp_file: with urllib.request.urlopen(file_url) as response: shutil.copyfileobj(response, temp_file, length=self.chunk_size) temp_file.flush() # Upload the local file to bucket bucket_name = self._set_bucket_name(self.bucket_name) gcs_hook.upload( bucket_name=bucket_name, object_name=report_name, gzip=self.gzip, filename=temp_file.name, mime_type="text/csv", ) self.log.info( "Report %s was saved in bucket %s as %s.", self.report_id, self.bucket_name, report_name, ) self.xcom_push(context, key="report_name", value=report_name)
def execute(self, context: 'Context') -> Dict[str, Any]: hook = GoogleDisplayVideo360Hook( gcp_conn_id=self.gcp_conn_id, delegate_to=self.delegate_to, api_version=self.api_version, impersonation_chain=self.impersonation_chain, ) self.log.info("Creating operation for SDF download task...") operation = hook.create_sdf_download_operation( body_request=self.body_request) name = operation["name"] self.xcom_push(context, key="name", value=name) self.log.info("Created SDF operation with name: %s", name) return operation
def execute(self, context: dict) -> None: hook = GoogleDisplayVideo360Hook( gcp_conn_id=self.gcp_conn_id, delegate_to=self.delegate_to, api_version=self.api_version, impersonation_chain=self.impersonation_chain, ) if self.report_id: reports_ids_to_delete = [self.report_id] else: reports = hook.list_queries() reports_ids_to_delete = [ report["queryId"] for report in reports if report["metadata"]["title"] == self.report_name ] for report_id in reports_ids_to_delete: self.log.info("Deleting report with id: %s", report_id) hook.delete_query(query_id=report_id) self.log.info("Report deleted.")
def execute(self, context: Dict): gcs_hook = GCSHook(gcp_conn_id=self.gcp_conn_id, delegate_to=self.delegate_to) hook = GoogleDisplayVideo360Hook( gcp_conn_id=self.gcp_conn_id, delegate_to=self.delegate_to, api_version=self.api_version, ) self.log.info("Uploading file %s...") # Saving file in the temporary directory, # downloaded file from the GCS could be a 1GB size or even more with tempfile.NamedTemporaryFile("w+") as f: line_items = gcs_hook.download( bucket_name=self.bucket_name, object_name=self.object_name, filename=f.name, ) f.flush() hook.upload_line_items(line_items=line_items)
def setUp(self): with mock.patch( "airflow.providers.google.common.hooks.base_google.GoogleBaseHook.__init__", new=mock_base_gcp_hook_default_project_id, ): self.hook = GoogleDisplayVideo360Hook(gcp_conn_id=GCP_CONN_ID)
GoogleDisplayVideo360ReportSensor, ) from airflow.utils import dates # [START howto_display_video_env_variables] BUCKET = os.environ.get("GMP_DISPLAY_VIDEO_BUCKET", "gs://INVALID BUCKET NAME") ADVERTISER_ID = os.environ.get("GMP_ADVERTISER_ID", 1234567) OBJECT_NAME = os.environ.get("GMP_OBJECT_NAME", "files/report.csv") PATH_TO_UPLOAD_FILE = os.environ.get("GCP_GCS_PATH_TO_UPLOAD_FILE", "test-gcs-example.txt") PATH_TO_SAVED_FILE = os.environ.get("GCP_GCS_PATH_TO_SAVED_FILE", "test-gcs-example-download.txt") BUCKET_FILE_LOCATION = PATH_TO_UPLOAD_FILE.rpartition("/")[-1] SDF_VERSION = os.environ.get("GMP_SDF_VERSION", "SDF_VERSION_5_1") BQ_DATA_SET = os.environ.get("GMP_BQ_DATA_SET", "airflow_test") GMP_PARTNER_ID = os.environ.get("GMP_PARTNER_ID", 123) ENTITY_TYPE = os.environ.get("GMP_ENTITY_TYPE", "LineItem") ERF_SOURCE_OBJECT = GoogleDisplayVideo360Hook.erf_uri(GMP_PARTNER_ID, ENTITY_TYPE) REPORT = { "kind": "doubleclickbidmanager#query", "metadata": { "title": "Polidea Test Report", "dataRange": "LAST_7_DAYS", "format": "CSV", "sendNotification": False, }, "params": { "type": "TYPE_GENERAL", "groupBys": ["FILTER_DATE", "FILTER_PARTNER"], "filters": [{"type": "FILTER_PARTNER", "value": 1486931}], "metrics": ["METRIC_IMPRESSIONS", "METRIC_CLICKS"], "includeInviteData": True,
def setUp(self): with mock.patch( "airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook.__init__", new=mock_base_gcp_hook_default_project_id, ): self.hook = GoogleDisplayVideo360Hook(gcp_conn_id=GCP_CONN_ID)