"*****@*****.**", "*****@*****.**", "*****@*****.**", ], date_partition_parameter="submission_date", depends_on_past=False, dag=dag, ) search_derived__search_aggregates__v8.set_upstream( search_derived__search_clients_daily__v8) wait_for_telemetry_derived__clients_daily_joined__v1 = ExternalTaskCompletedSensor( task_id="wait_for_telemetry_derived__clients_daily_joined__v1", external_dag_id="bqetl_main_summary", external_task_id="telemetry_derived__clients_daily_joined__v1", execution_delta=datetime.timedelta(seconds=3600), check_existence=True, mode="reschedule", pool="DATA_ENG_EXTERNALTASKSENSOR", ) search_derived__search_clients_daily__v8.set_upstream( wait_for_telemetry_derived__clients_daily_joined__v1) search_derived__search_clients_last_seen__v1.set_upstream( search_derived__search_clients_daily__v8) search_derived__search_metric_contribution__v1.set_upstream( search_derived__search_clients_daily__v8)
task_id="telemetry_derived__smoot_usage_nondesktop_compressed__v2", destination_table="smoot_usage_nondesktop_compressed_v2", dataset_id="telemetry_derived", project_id="moz-fx-data-shared-prod", owner="*****@*****.**", email=["*****@*****.**", "*****@*****.**"], date_partition_parameter="submission_date", depends_on_past=False, dag=dag, ) wait_for_telemetry_derived__clients_last_seen__v1 = ExternalTaskCompletedSensor( task_id="wait_for_telemetry_derived__clients_last_seen__v1", external_dag_id="bqetl_main_summary", external_task_id="telemetry_derived__clients_last_seen__v1", execution_delta=datetime.timedelta(seconds=3600), check_existence=True, mode="reschedule", pool="DATA_ENG_EXTERNALTASKSENSOR", ) telemetry_derived__smoot_usage_desktop__v2.set_upstream( wait_for_telemetry_derived__clients_last_seen__v1 ) telemetry_derived__smoot_usage_desktop_compressed__v2.set_upstream( telemetry_derived__smoot_usage_desktop__v2 ) wait_for_firefox_accounts_derived__fxa_users_last_seen__v1 = ( ExternalTaskCompletedSensor(
task_id="monitoring_derived__telemetry_missing_columns__v3", destination_table="telemetry_missing_columns_v3", dataset_id="monitoring_derived", project_id="moz-fx-data-shared-prod", owner="*****@*****.**", email=["*****@*****.**", "*****@*****.**"], date_partition_parameter="submission_date", depends_on_past=False, dag=dag, ) wait_for_copy_deduplicate_all = ExternalTaskCompletedSensor( task_id="wait_for_copy_deduplicate_all", external_dag_id="copy_deduplicate", external_task_id="copy_deduplicate_all", execution_delta=datetime.timedelta(seconds=3600), check_existence=True, mode="reschedule", pool="DATA_ENG_EXTERNALTASKSENSOR", ) monitoring_derived__average_ping_sizes__v1.set_upstream( wait_for_copy_deduplicate_all ) monitoring_derived__average_ping_sizes__v1.set_upstream( monitoring_derived__stable_table_sizes__v1 ) wait_for_copy_deduplicate_main_ping = ExternalTaskCompletedSensor( task_id="wait_for_copy_deduplicate_main_ping",
"*****@*****.**", ], date_partition_parameter="submission_date", depends_on_past=False, dag=dag, ) experiment_enrollment_daily_active_population.set_upstream( telemetry_derived__experiments_daily_active_clients__v1 ) wait_for_bq_main_events = ExternalTaskCompletedSensor( task_id="wait_for_bq_main_events", external_dag_id="copy_deduplicate", external_task_id="bq_main_events", execution_delta=datetime.timedelta(seconds=7200), check_existence=True, mode="reschedule", pool="DATA_ENG_EXTERNALTASKSENSOR", ) telemetry_derived__experiment_enrollment_aggregates__v1.set_upstream( wait_for_bq_main_events ) wait_for_copy_deduplicate_all = ExternalTaskCompletedSensor( task_id="wait_for_copy_deduplicate_all", external_dag_id="copy_deduplicate", external_task_id="copy_deduplicate_all", execution_delta=datetime.timedelta(seconds=7200), check_existence=True, mode="reschedule",
mozilla_vpn_derived__site_metrics_summary__v1 = bigquery_etl_query( task_id="mozilla_vpn_derived__site_metrics_summary__v1", destination_table="site_metrics_summary_v1", dataset_id="mozilla_vpn_derived", project_id="moz-fx-data-shared-prod", owner="*****@*****.**", email=["*****@*****.**", "*****@*****.**"], date_partition_parameter="date", depends_on_past=False, dag=dag, ) wait_for_mozilla_vpn_derived__all_subscriptions__v1 = ExternalTaskCompletedSensor( task_id="wait_for_mozilla_vpn_derived__all_subscriptions__v1", external_dag_id="bqetl_subplat", external_task_id="mozilla_vpn_derived__all_subscriptions__v1", execution_delta=datetime.timedelta(seconds=47700), check_existence=True, mode="reschedule", pool="DATA_ENG_EXTERNALTASKSENSOR", ) mozilla_vpn_derived__funnel_ga_to_subscriptions__v1.set_upstream( wait_for_mozilla_vpn_derived__all_subscriptions__v1) mozilla_vpn_derived__funnel_ga_to_subscriptions__v1.set_upstream( mozilla_vpn_derived__site_metrics_summary__v1) mozilla_vpn_derived__site_metrics_summary__v1.set_upstream( mozilla_vpn_derived__site_metrics_empty_check__v1)
destination_table="event_aggregates_v1", dataset_id="contextual_services_derived", project_id="moz-fx-data-shared-prod", owner="*****@*****.**", email=["*****@*****.**", "*****@*****.**"], date_partition_parameter="submission_date", depends_on_past=False, arguments=["--schema_update_option=ALLOW_FIELD_ADDITION"], dag=dag, ) wait_for_copy_deduplicate_all = ExternalTaskCompletedSensor( task_id="wait_for_copy_deduplicate_all", external_dag_id="copy_deduplicate", external_task_id="copy_deduplicate_all", execution_delta=datetime.timedelta(seconds=7200), check_existence=True, mode="reschedule", pool="DATA_ENG_EXTERNALTASKSENSOR", ) contextual_services_derived__event_aggregates__v1.set_upstream( wait_for_copy_deduplicate_all) wait_for_search_terms_derived__suggest_impression_sanitized__v1 = ( ExternalTaskCompletedSensor( task_id= "wait_for_search_terms_derived__suggest_impression_sanitized__v1", external_dag_id="bqetl_search_terms_daily", external_task_id= "search_terms_derived__suggest_impression_sanitized__v1", check_existence=True,
destination_table="core_clients_last_seen_v1", dataset_id="telemetry_derived", project_id="moz-fx-data-shared-prod", owner="*****@*****.**", email=["*****@*****.**", "*****@*****.**"], date_partition_parameter="submission_date", depends_on_past=True, priority_weight=70, dag=dag, ) wait_for_copy_deduplicate_all = ExternalTaskCompletedSensor( task_id="wait_for_copy_deduplicate_all", external_dag_id="copy_deduplicate", external_task_id="copy_deduplicate_all", execution_delta=datetime.timedelta(seconds=3600), check_existence=True, mode="reschedule", pool="DATA_ENG_EXTERNALTASKSENSOR", ) telemetry_derived__core_clients_daily__v1.set_upstream( wait_for_copy_deduplicate_all ) wait_for_telemetry_derived__core_clients_first_seen__v1 = ( ExternalTaskCompletedSensor( task_id="wait_for_telemetry_derived__core_clients_first_seen__v1", external_dag_id="copy_deduplicate", external_task_id="telemetry_derived__core_clients_first_seen__v1", execution_delta=datetime.timedelta(seconds=3600), check_existence=True,
email=[ "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", ], date_partition_parameter="submission_date", depends_on_past=False, dag=dag, ) wait_for_telemetry_derived__clients_last_seen__v1 = ExternalTaskCompletedSensor( task_id="wait_for_telemetry_derived__clients_last_seen__v1", external_dag_id="bqetl_main_summary", external_task_id="telemetry_derived__clients_last_seen__v1", execution_delta=datetime.timedelta(seconds=7200), check_existence=True, mode="reschedule", pool="DATA_ENG_EXTERNALTASKSENSOR", ) search_derived__desktop_search_aggregates_by_userstate__v1.set_upstream( wait_for_telemetry_derived__clients_last_seen__v1) wait_for_search_derived__search_aggregates__v8 = ExternalTaskCompletedSensor( task_id="wait_for_search_derived__search_aggregates__v8", external_dag_id="bqetl_search", external_task_id="search_derived__search_aggregates__v8", execution_delta=datetime.timedelta(seconds=3600), check_existence=True, mode="reschedule",
name="export_public_data_json_telemetry_derived__ssl_ratios__v1", arguments=["script/publish_public_data_json"] + [ "--query_file=sql/moz-fx-data-shared-prod/telemetry_derived/ssl_ratios_v1/query.sql" ] + ["--destination_table=ssl_ratios${{ds_nodash}}"] + ["--dataset_id=telemetry_derived"] + ["--project_id=moz-fx-data-shared-prod"] + ["--parameter=submission_date:DATE:{{ds}}"], image=docker_image, dag=dag, ) wait_for_mozregression_aggregates__v1 = ExternalTaskCompletedSensor( task_id="wait_for_mozregression_aggregates__v1", external_dag_id="bqetl_internal_tooling", external_task_id="mozregression_aggregates__v1", execution_delta=datetime.timedelta(seconds=3600), check_existence=True, mode="reschedule", failed_states=[State.FAILED, State.UPSTREAM_FAILED, State.SKIPPED], pool="DATA_ENG_EXTERNALTASKSENSOR", ) export_public_data_json_mozregression_aggregates__v1.set_upstream( wait_for_mozregression_aggregates__v1) wait_for_telemetry_derived__ssl_ratios__v1 = ExternalTaskCompletedSensor( task_id="wait_for_telemetry_derived__ssl_ratios__v1", external_dag_id="bqetl_ssl_ratios", external_task_id="telemetry_derived__ssl_ratios__v1", execution_delta=datetime.timedelta(seconds=10800), check_existence=True, mode="reschedule",
) mozilla_vpn_derived__funnel_fxa_login_to_protected__v1.set_upstream( mozilla_vpn_derived__protected__v1 ) mozilla_vpn_derived__funnel_fxa_login_to_protected__v1.set_upstream( mozilla_vpn_derived__users__v1 ) wait_for_firefox_accounts_derived__fxa_auth_events__v1 = ( ExternalTaskCompletedSensor( task_id="wait_for_firefox_accounts_derived__fxa_auth_events__v1", external_dag_id="bqetl_fxa_events", external_task_id="firefox_accounts_derived__fxa_auth_events__v1", execution_delta=datetime.timedelta(seconds=900), check_existence=True, mode="reschedule", pool="DATA_ENG_EXTERNALTASKSENSOR", ) ) mozilla_vpn_derived__login_flows__v1.set_upstream( wait_for_firefox_accounts_derived__fxa_auth_events__v1 ) wait_for_firefox_accounts_derived__fxa_content_events__v1 = ( ExternalTaskCompletedSensor( task_id="wait_for_firefox_accounts_derived__fxa_content_events__v1", external_dag_id="bqetl_fxa_events", external_task_id="firefox_accounts_derived__fxa_content_events__v1", execution_delta=datetime.timedelta(seconds=900),
"*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", ], date_partition_parameter="submission_date", depends_on_past=False, dag=dag, ) wait_for_bq_main_events = ExternalTaskCompletedSensor( task_id="wait_for_bq_main_events", external_dag_id="copy_deduplicate", external_task_id="bq_main_events", execution_delta=datetime.timedelta(seconds=14400), check_existence=True, mode="reschedule", pool="DATA_ENG_EXTERNALTASKSENSOR", ) telemetry_derived__feature_usage__v2.set_upstream(wait_for_bq_main_events) wait_for_copy_deduplicate_all = ExternalTaskCompletedSensor( task_id="wait_for_copy_deduplicate_all", external_dag_id="copy_deduplicate", external_task_id="copy_deduplicate_all", execution_delta=datetime.timedelta(seconds=14400), check_existence=True, mode="reschedule", pool="DATA_ENG_EXTERNALTASKSENSOR", )
dataset_id="telemetry_derived", project_id="moz-fx-data-shared-prod", owner="*****@*****.**", email=[ "*****@*****.**", "*****@*****.**", "*****@*****.**" ], date_partition_parameter="submission_date", depends_on_past=False, dag=dag, ) wait_for_bq_main_events = ExternalTaskCompletedSensor( task_id="wait_for_bq_main_events", external_dag_id="copy_deduplicate", external_task_id="bq_main_events", execution_delta=datetime.timedelta(seconds=10800), check_existence=True, mode="reschedule", pool="DATA_ENG_EXTERNALTASKSENSOR", ) telemetry_derived__adm_engagements_daily__v1.set_upstream( wait_for_bq_main_events) wait_for_event_events = ExternalTaskCompletedSensor( task_id="wait_for_event_events", external_dag_id="copy_deduplicate", external_task_id="event_events", execution_delta=datetime.timedelta(seconds=10800), check_existence=True, mode="reschedule", pool="DATA_ENG_EXTERNALTASKSENSOR",