"*****@*****.**",
"*****@*****.**",
"*****@*****.**",
],
date_partition_parameter="submission_date",
depends_on_past=False,
dag=dag,
)
search_derived__search_aggregates__v8.set_upstream(
search_derived__search_clients_daily__v8)
wait_for_telemetry_derived__clients_daily_joined__v1 = ExternalTaskCompletedSensor(
task_id="wait_for_telemetry_derived__clients_daily_joined__v1",
external_dag_id="bqetl_main_summary",
external_task_id="telemetry_derived__clients_daily_joined__v1",
execution_delta=datetime.timedelta(seconds=3600),
check_existence=True,
mode="reschedule",
pool="DATA_ENG_EXTERNALTASKSENSOR",
)
search_derived__search_clients_daily__v8.set_upstream(
wait_for_telemetry_derived__clients_daily_joined__v1)
search_derived__search_clients_last_seen__v1.set_upstream(
search_derived__search_clients_daily__v8)
search_derived__search_metric_contribution__v1.set_upstream(
search_derived__search_clients_daily__v8)
task_id="telemetry_derived__smoot_usage_nondesktop_compressed__v2",
destination_table="smoot_usage_nondesktop_compressed_v2",
dataset_id="telemetry_derived",
project_id="moz-fx-data-shared-prod",
owner="*****@*****.**",
email=["*****@*****.**", "*****@*****.**"],
date_partition_parameter="submission_date",
depends_on_past=False,
dag=dag,
)
wait_for_telemetry_derived__clients_last_seen__v1 = ExternalTaskCompletedSensor(
task_id="wait_for_telemetry_derived__clients_last_seen__v1",
external_dag_id="bqetl_main_summary",
external_task_id="telemetry_derived__clients_last_seen__v1",
execution_delta=datetime.timedelta(seconds=3600),
check_existence=True,
mode="reschedule",
pool="DATA_ENG_EXTERNALTASKSENSOR",
)
telemetry_derived__smoot_usage_desktop__v2.set_upstream(
wait_for_telemetry_derived__clients_last_seen__v1
)
telemetry_derived__smoot_usage_desktop_compressed__v2.set_upstream(
telemetry_derived__smoot_usage_desktop__v2
)
wait_for_firefox_accounts_derived__fxa_users_last_seen__v1 = (
ExternalTaskCompletedSensor(
task_id="monitoring_derived__telemetry_missing_columns__v3",
destination_table="telemetry_missing_columns_v3",
dataset_id="monitoring_derived",
project_id="moz-fx-data-shared-prod",
owner="*****@*****.**",
email=["*****@*****.**", "*****@*****.**"],
date_partition_parameter="submission_date",
depends_on_past=False,
dag=dag,
)
wait_for_copy_deduplicate_all = ExternalTaskCompletedSensor(
task_id="wait_for_copy_deduplicate_all",
external_dag_id="copy_deduplicate",
external_task_id="copy_deduplicate_all",
execution_delta=datetime.timedelta(seconds=3600),
check_existence=True,
mode="reschedule",
pool="DATA_ENG_EXTERNALTASKSENSOR",
)
monitoring_derived__average_ping_sizes__v1.set_upstream(
wait_for_copy_deduplicate_all
)
monitoring_derived__average_ping_sizes__v1.set_upstream(
monitoring_derived__stable_table_sizes__v1
)
wait_for_copy_deduplicate_main_ping = ExternalTaskCompletedSensor(
task_id="wait_for_copy_deduplicate_main_ping",
"*****@*****.**",
],
date_partition_parameter="submission_date",
depends_on_past=False,
dag=dag,
)
experiment_enrollment_daily_active_population.set_upstream(
telemetry_derived__experiments_daily_active_clients__v1
)
wait_for_bq_main_events = ExternalTaskCompletedSensor(
task_id="wait_for_bq_main_events",
external_dag_id="copy_deduplicate",
external_task_id="bq_main_events",
execution_delta=datetime.timedelta(seconds=7200),
check_existence=True,
mode="reschedule",
pool="DATA_ENG_EXTERNALTASKSENSOR",
)
telemetry_derived__experiment_enrollment_aggregates__v1.set_upstream(
wait_for_bq_main_events
)
wait_for_copy_deduplicate_all = ExternalTaskCompletedSensor(
task_id="wait_for_copy_deduplicate_all",
external_dag_id="copy_deduplicate",
external_task_id="copy_deduplicate_all",
execution_delta=datetime.timedelta(seconds=7200),
check_existence=True,
mode="reschedule",
mozilla_vpn_derived__site_metrics_summary__v1 = bigquery_etl_query(
task_id="mozilla_vpn_derived__site_metrics_summary__v1",
destination_table="site_metrics_summary_v1",
dataset_id="mozilla_vpn_derived",
project_id="moz-fx-data-shared-prod",
owner="*****@*****.**",
email=["*****@*****.**", "*****@*****.**"],
date_partition_parameter="date",
depends_on_past=False,
dag=dag,
)
wait_for_mozilla_vpn_derived__all_subscriptions__v1 = ExternalTaskCompletedSensor(
task_id="wait_for_mozilla_vpn_derived__all_subscriptions__v1",
external_dag_id="bqetl_subplat",
external_task_id="mozilla_vpn_derived__all_subscriptions__v1",
execution_delta=datetime.timedelta(seconds=47700),
check_existence=True,
mode="reschedule",
pool="DATA_ENG_EXTERNALTASKSENSOR",
)
mozilla_vpn_derived__funnel_ga_to_subscriptions__v1.set_upstream(
wait_for_mozilla_vpn_derived__all_subscriptions__v1)
mozilla_vpn_derived__funnel_ga_to_subscriptions__v1.set_upstream(
mozilla_vpn_derived__site_metrics_summary__v1)
mozilla_vpn_derived__site_metrics_summary__v1.set_upstream(
mozilla_vpn_derived__site_metrics_empty_check__v1)
destination_table="event_aggregates_v1",
dataset_id="contextual_services_derived",
project_id="moz-fx-data-shared-prod",
owner="*****@*****.**",
email=["*****@*****.**", "*****@*****.**"],
date_partition_parameter="submission_date",
depends_on_past=False,
arguments=["--schema_update_option=ALLOW_FIELD_ADDITION"],
dag=dag,
)
wait_for_copy_deduplicate_all = ExternalTaskCompletedSensor(
task_id="wait_for_copy_deduplicate_all",
external_dag_id="copy_deduplicate",
external_task_id="copy_deduplicate_all",
execution_delta=datetime.timedelta(seconds=7200),
check_existence=True,
mode="reschedule",
pool="DATA_ENG_EXTERNALTASKSENSOR",
)
contextual_services_derived__event_aggregates__v1.set_upstream(
wait_for_copy_deduplicate_all)
wait_for_search_terms_derived__suggest_impression_sanitized__v1 = (
ExternalTaskCompletedSensor(
task_id=
"wait_for_search_terms_derived__suggest_impression_sanitized__v1",
external_dag_id="bqetl_search_terms_daily",
external_task_id=
"search_terms_derived__suggest_impression_sanitized__v1",
check_existence=True,
destination_table="core_clients_last_seen_v1",
dataset_id="telemetry_derived",
project_id="moz-fx-data-shared-prod",
owner="*****@*****.**",
email=["*****@*****.**", "*****@*****.**"],
date_partition_parameter="submission_date",
depends_on_past=True,
priority_weight=70,
dag=dag,
)
wait_for_copy_deduplicate_all = ExternalTaskCompletedSensor(
task_id="wait_for_copy_deduplicate_all",
external_dag_id="copy_deduplicate",
external_task_id="copy_deduplicate_all",
execution_delta=datetime.timedelta(seconds=3600),
check_existence=True,
mode="reschedule",
pool="DATA_ENG_EXTERNALTASKSENSOR",
)
telemetry_derived__core_clients_daily__v1.set_upstream(
wait_for_copy_deduplicate_all
)
wait_for_telemetry_derived__core_clients_first_seen__v1 = (
ExternalTaskCompletedSensor(
task_id="wait_for_telemetry_derived__core_clients_first_seen__v1",
external_dag_id="copy_deduplicate",
external_task_id="telemetry_derived__core_clients_first_seen__v1",
execution_delta=datetime.timedelta(seconds=3600),
check_existence=True,
email=[
"*****@*****.**",
"*****@*****.**",
"*****@*****.**",
"*****@*****.**",
],
date_partition_parameter="submission_date",
depends_on_past=False,
dag=dag,
)
wait_for_telemetry_derived__clients_last_seen__v1 = ExternalTaskCompletedSensor(
task_id="wait_for_telemetry_derived__clients_last_seen__v1",
external_dag_id="bqetl_main_summary",
external_task_id="telemetry_derived__clients_last_seen__v1",
execution_delta=datetime.timedelta(seconds=7200),
check_existence=True,
mode="reschedule",
pool="DATA_ENG_EXTERNALTASKSENSOR",
)
search_derived__desktop_search_aggregates_by_userstate__v1.set_upstream(
wait_for_telemetry_derived__clients_last_seen__v1)
wait_for_search_derived__search_aggregates__v8 = ExternalTaskCompletedSensor(
task_id="wait_for_search_derived__search_aggregates__v8",
external_dag_id="bqetl_search",
external_task_id="search_derived__search_aggregates__v8",
execution_delta=datetime.timedelta(seconds=3600),
check_existence=True,
mode="reschedule",
name="export_public_data_json_telemetry_derived__ssl_ratios__v1",
arguments=["script/publish_public_data_json"] + [
"--query_file=sql/moz-fx-data-shared-prod/telemetry_derived/ssl_ratios_v1/query.sql"
] + ["--destination_table=ssl_ratios${{ds_nodash}}"] +
["--dataset_id=telemetry_derived"] +
["--project_id=moz-fx-data-shared-prod"] +
["--parameter=submission_date:DATE:{{ds}}"],
image=docker_image,
dag=dag,
)
wait_for_mozregression_aggregates__v1 = ExternalTaskCompletedSensor(
task_id="wait_for_mozregression_aggregates__v1",
external_dag_id="bqetl_internal_tooling",
external_task_id="mozregression_aggregates__v1",
execution_delta=datetime.timedelta(seconds=3600),
check_existence=True,
mode="reschedule",
failed_states=[State.FAILED, State.UPSTREAM_FAILED, State.SKIPPED],
pool="DATA_ENG_EXTERNALTASKSENSOR",
)
export_public_data_json_mozregression_aggregates__v1.set_upstream(
wait_for_mozregression_aggregates__v1)
wait_for_telemetry_derived__ssl_ratios__v1 = ExternalTaskCompletedSensor(
task_id="wait_for_telemetry_derived__ssl_ratios__v1",
external_dag_id="bqetl_ssl_ratios",
external_task_id="telemetry_derived__ssl_ratios__v1",
execution_delta=datetime.timedelta(seconds=10800),
check_existence=True,
mode="reschedule",
)
mozilla_vpn_derived__funnel_fxa_login_to_protected__v1.set_upstream(
mozilla_vpn_derived__protected__v1
)
mozilla_vpn_derived__funnel_fxa_login_to_protected__v1.set_upstream(
mozilla_vpn_derived__users__v1
)
wait_for_firefox_accounts_derived__fxa_auth_events__v1 = (
ExternalTaskCompletedSensor(
task_id="wait_for_firefox_accounts_derived__fxa_auth_events__v1",
external_dag_id="bqetl_fxa_events",
external_task_id="firefox_accounts_derived__fxa_auth_events__v1",
execution_delta=datetime.timedelta(seconds=900),
check_existence=True,
mode="reschedule",
pool="DATA_ENG_EXTERNALTASKSENSOR",
)
)
mozilla_vpn_derived__login_flows__v1.set_upstream(
wait_for_firefox_accounts_derived__fxa_auth_events__v1
)
wait_for_firefox_accounts_derived__fxa_content_events__v1 = (
ExternalTaskCompletedSensor(
task_id="wait_for_firefox_accounts_derived__fxa_content_events__v1",
external_dag_id="bqetl_fxa_events",
external_task_id="firefox_accounts_derived__fxa_content_events__v1",
execution_delta=datetime.timedelta(seconds=900),
"*****@*****.**",
"*****@*****.**",
"*****@*****.**",
"*****@*****.**",
"*****@*****.**",
],
date_partition_parameter="submission_date",
depends_on_past=False,
dag=dag,
)
wait_for_bq_main_events = ExternalTaskCompletedSensor(
task_id="wait_for_bq_main_events",
external_dag_id="copy_deduplicate",
external_task_id="bq_main_events",
execution_delta=datetime.timedelta(seconds=14400),
check_existence=True,
mode="reschedule",
pool="DATA_ENG_EXTERNALTASKSENSOR",
)
telemetry_derived__feature_usage__v2.set_upstream(wait_for_bq_main_events)
wait_for_copy_deduplicate_all = ExternalTaskCompletedSensor(
task_id="wait_for_copy_deduplicate_all",
external_dag_id="copy_deduplicate",
external_task_id="copy_deduplicate_all",
execution_delta=datetime.timedelta(seconds=14400),
check_existence=True,
mode="reschedule",
pool="DATA_ENG_EXTERNALTASKSENSOR",
)
dataset_id="telemetry_derived",
project_id="moz-fx-data-shared-prod",
owner="*****@*****.**",
email=[
"*****@*****.**", "*****@*****.**", "*****@*****.**"
],
date_partition_parameter="submission_date",
depends_on_past=False,
dag=dag,
)
wait_for_bq_main_events = ExternalTaskCompletedSensor(
task_id="wait_for_bq_main_events",
external_dag_id="copy_deduplicate",
external_task_id="bq_main_events",
execution_delta=datetime.timedelta(seconds=10800),
check_existence=True,
mode="reschedule",
pool="DATA_ENG_EXTERNALTASKSENSOR",
)
telemetry_derived__adm_engagements_daily__v1.set_upstream(
wait_for_bq_main_events)
wait_for_event_events = ExternalTaskCompletedSensor(
task_id="wait_for_event_events",
external_dag_id="copy_deduplicate",
external_task_id="event_events",
execution_delta=datetime.timedelta(seconds=10800),
check_existence=True,
mode="reschedule",
pool="DATA_ENG_EXTERNALTASKSENSOR",
