def execute(self, context: 'Context') -> str: hook = DataFusionHook( gcp_conn_id=self.gcp_conn_id, delegate_to=self.delegate_to, api_version=self.api_version, impersonation_chain=self.impersonation_chain, ) self.log.info("Starting Data Fusion pipeline: %s", self.pipeline_name) instance = hook.get_instance( instance_name=self.instance_name, location=self.location, project_id=self.project_id, ) api_url = instance["apiEndpoint"] pipeline_id = hook.start_pipeline( pipeline_name=self.pipeline_name, instance_url=api_url, namespace=self.namespace, runtime_args=self.runtime_args, ) self.log.info("Pipeline %s submitted successfully.", pipeline_id) if not self.asynchronous: self.log.info("Waiting when pipeline %s will be in one of the success states", pipeline_id) hook.wait_for_pipeline_state( success_states=self.success_states, pipeline_id=pipeline_id, pipeline_name=self.pipeline_name, namespace=self.namespace, instance_url=api_url, timeout=self.pipeline_timeout, ) self.log.info("Job %s discover success state.", pipeline_id) return pipeline_id
def execute(self, context: Dict): hook = DataFusionHook( gcp_conn_id=self.gcp_conn_id, delegate_to=self.delegate_to, api_version=self.api_version, ) self.log.info("Starting Data Fusion pipeline: %s", self.pipeline_name) instance = hook.get_instance( instance_name=self.instance_name, location=self.location, project_id=self.project_id, ) api_url = instance["apiEndpoint"] pipeline_id = hook.start_pipeline( pipeline_name=self.pipeline_name, instance_url=api_url, namespace=self.namespace, runtime_args=self.runtime_args, ) self.log.info("Pipeline started") if self.success_states: hook.wait_for_pipeline_state(success_states=self.success_states, pipeline_id=pipeline_id, pipeline_name=self.pipeline_name, namespace=self.namespace, instance_url=api_url, timeout=self.pipeline_timeout)
def execute(self, context: Dict): hook = DataFusionHook( gcp_conn_id=self.gcp_conn_id, delegate_to=self.delegate_to, api_version=self.api_version, ) self.log.info("Starting Data Fusion pipeline: %s", self.pipeline_name) instance = hook.get_instance( instance_name=self.instance_name, location=self.location, project_id=self.project_id, ) api_url = instance["apiEndpoint"] hook.start_pipeline( pipeline_name=self.pipeline_name, instance_url=api_url, namespace=self.namespace, ) self.log.info("Pipeline started")