def execute(self, context: "Context") -> dict:
hook = DataplexHook(
gcp_conn_id=self.gcp_conn_id,
delegate_to=self.delegate_to,
api_version=self.api_version,
impersonation_chain=self.impersonation_chain,
)
self.log.info("Creating Dataplex task %s", self.dataplex_task_id)
DataplexTaskLink.persist(context=context, task_instance=self)
try:
operation = hook.create_task(
project_id=self.project_id,
region=self.region,
lake_id=self.lake_id,
body=self.body,
dataplex_task_id=self.dataplex_task_id,
validate_only=self.validate_only,
retry=self.retry,
timeout=self.timeout,
metadata=self.metadata,
)
if not self.asynchronous:
self.log.info("Waiting for Dataplex task %s to be created",
self.dataplex_task_id)
task = hook.wait_for_operation(timeout=self.timeout,
operation=operation)
self.log.info("Task %s created successfully",
self.dataplex_task_id)
else:
is_done = operation.done()
self.log.info("Is operation done already? %s", is_done)
return is_done
except HttpError as err:
if err.resp.status not in (409, '409'):
raise
self.log.info("Task %s already exists", self.dataplex_task_id)
# Wait for task to be ready
for time_to_wait in exponential_sleep_generator(initial=10,
maximum=120):
task = hook.get_task(
project_id=self.project_id,
region=self.region,
lake_id=self.lake_id,
dataplex_task_id=self.dataplex_task_id,
retry=self.retry,
timeout=self.timeout,
metadata=self.metadata,
)
if task['state'] != 'CREATING':
break
sleep(time_to_wait)
return Task.to_dict(task)
def execute(self, context: "Context") -> dict:
hook = DataplexHook(
gcp_conn_id=self.gcp_conn_id,
delegate_to=self.delegate_to,
api_version=self.api_version,
impersonation_chain=self.impersonation_chain,
)
self.log.info("Retrieving Dataplex task %s", self.dataplex_task_id)
DataplexTaskLink.persist(context=context, task_instance=self)
task = hook.get_task(
project_id=self.project_id,
region=self.region,
lake_id=self.lake_id,
dataplex_task_id=self.dataplex_task_id,
retry=self.retry,
timeout=self.timeout,
metadata=self.metadata,
)
return Task.to_dict(task)