dagrun_timeout=timedelta(hours=6),
schedule_interval='@weekly')
create_gke_cluster = GKEClusterCreateOperator(task_id='create_gke_cluster',
project_id=connection.project_id,
location='us-west1-b',
gcp_conn_id=gcp_conn_id,
body=cluster_def,
dag=gke_dag)
run_prio = GKEPodOperator(task_id='run_prio_a',
gcp_conn_id=gcp_conn_id,
project_id=connection.project_id,
location='us-west1-b',
cluster_name=cluster_name,
name='run-prio-project-a',
namespace='default',
image='mozilla/python-libprio:latest',
arguments=['scripts/test-cli-integration'],
dag=gke_dag)
delete_gke_cluster = GKEClusterDeleteOperator(task_id='delete_gke_cluster',
project_id=connection.project_id,
location='us-west1-b',
name=cluster_name,
gcp_conn_id=gcp_conn_id,
dag=gke_dag)
create_gke_cluster.set_downstream(run_prio)
run_prio.set_downstream(delete_gke_cluster)
'--destination_table',
'moz-fx-data-derived-datasets:blpadi.adi_dimensional_by_date${{ ds_nodash }}',
'--use_legacy_sql=false',
"select tot_requests_on_date, _year_quarter, bl_date, product, v_prod_major, prod_os, v_prod_os, channel, locale, continent_code, cntry_code, distro_name, distro_version from blpadi.adi_dim_backfill where bl_date = '{{ ds }}'",
]
load_bq_to_tmp_tbl = GKEPodOperator(task_id='bq_load_tmp_tbl',
gcp_conn_id=gcp_conn_id,
project_id=connection.project_id,
location='us-central1-a',
cluster_name='bq-load-gke-1',
name='bq-load-tmp-tbl',
namespace='default',
image='google/cloud-sdk:242.0.0-alpine',
arguments=load_args,
dag=blp_dag)
select_insert_into_final_table = GKEPodOperator(
task_id='bigquery_insert_final_table',
gcp_conn_id=gcp_conn_id,
project_id='moz-fx-data-derived-datasets',
location='us-central1-a',
cluster_name='bq-load-gke-1',
name='bq-query-insert-final-tbl',
namespace='default',
image='google/cloud-sdk:242.0.0-alpine',
arguments=insert_args,
dag=blp_dag)
load_bq_to_tmp_tbl.set_downstream(select_insert_into_final_table)