def get_link(self, operator, dttm):
task_instance = TaskInstance(task=operator, execution_date=dttm)
gcp_metadata_dict = task_instance.xcom_pull(task_ids=operator.task_id, key="gcp_metadata")
if not gcp_metadata_dict:
return ''
job_id = gcp_metadata_dict['job_id']
project_id = gcp_metadata_dict['project_id']
console_link = f"https://console.cloud.google.com/ai-platform/jobs/{job_id}?project={project_id}"
return console_link
def get_link(self, operator, dttm):
ti = TaskInstance(task=operator, execution_date=dttm)
job_ids = ti.xcom_pull(task_ids=operator.task_id, key='job_id')
if not job_ids:
return None
if len(job_ids) < self.index:
return None
job_id = job_ids[self.index]
return BIGQUERY_JOB_DETAILS_LINK_FMT.format(job_id=job_id)
def get_link(self, operator, dttm):
"""
Get link to qubole command result page.
:param operator: operator
:param dttm: datetime
:return: url link
"""
ti = TaskInstance(task=operator, execution_date=dttm)
conn = BaseHook.get_connection(operator.kwargs['qubole_conn_id'])
if conn and conn.host:
host = re.sub(r'api$', 'v2/analyze?command_id=', conn.host)
else:
host = 'https://api.qubole.com/v2/analyze?command_id='
qds_command_id = ti.xcom_pull(task_ids=operator.task_id, key='qbol_cmd_id')
url = host + str(qds_command_id) if qds_command_id else ''
return url
def _process_user(ti: TaskInstance) -> None:
"""Process user from JSON and save as csv.
Args:
ti: the task instance
"""
users = ti.xcom_pull(task_ids=["extract_user"])
if not len(users) or "results" not in users[0]:
raise ValueError("User is empty")
user = users[0]["results"][0]
processed_user = json_normalize({
"name": f"{user['name']['first']} {user['name']['last']}",
"country": user["location"]["country"],
})
processed_user.to_csv(_PATH_DATA_TEMP, index=False, header=False)
return
def get_link(self, operator, dttm):
ti = TaskInstance(task=operator, execution_date=dttm)
job_id = ti.xcom_pull(task_ids=operator.task_id, key='job_id')
return 'https://console.cloud.google.com/bigquery?j={job_id}'.format(
job_id=job_id) if job_id else ''
def get_link(self, operator, dttm):
ti = TaskInstance(task=operator, execution_date=dttm)
job_id = ti.xcom_pull(task_ids=operator.task_id, key='job_id')
return 'https://console.cloud.google.com/bigquery?j={job_id}'.format(
job_id=job_id) if job_id else ''
def get_link(self, operator, dttm):
ti = TaskInstance(task=operator, execution_date=dttm)
job_id = ti.xcom_pull(task_ids=operator.task_id, key='job_id')
return BIGQUERY_JOB_DETAILS_LINK_FMT.format(job_id=job_id) if job_id else ''
def merge_task(task_instance: TaskInstance, **kwargs):
return task_instance.xcom_pull(task_ids=['hello_dataset', 'pandas_dataset'])