"input_event_response_coalesced_ms_main_above_250", "input_event_response_coalesced_ms_main_above_2500", "input_event_response_coalesced_ms_content_above_150", "input_event_response_coalesced_ms_content_above_250", "input_event_response_coalesced_ms_content_above_2500", "ghost_windows_main_above_1", "ghost_windows_content_above_1", ] main_summary_export = SubDagOperator( subdag=export_to_parquet( table="moz-fx-data-shared-prod:telemetry_derived.main_summary_v4${{ds_nodash}}", static_partitions=["submission_date_s3={{ds_nodash}}"], arguments=[ "--partition-by=sample_id", "--replace='{{ds_nodash}}' AS submission_date", "--maps-from-entries", ] + main_summary_bigint_columns, parent_dag_name=dag.dag_id, dag_name="main_summary_export", default_args=default_args, num_workers=40), task_id="main_summary_export", executor=get_default_executor(), dag=dag) register_status(main_summary, "Main Summary", "A summary view of main pings.") addons = bigquery_etl_query( task_id="addons", destination_table="addons_v2", project_id="moz-fx-data-shared-prod",
task_id='clients_last_seen', bql='sql/clients_last_seen_v1.sql', destination_dataset_table='telemetry.clients_last_seen_v1${{ds_nodash}}', write_disposition='WRITE_TRUNCATE', use_legacy_sql=False, bigquery_conn_id="google_cloud_derived_datasets", depends_on_past=True, start_date=datetime(2019, 4, 15), dag=dag, ) clients_last_seen_export = SubDagOperator( subdag=export_to_parquet( table="clients_last_seen_v1", arguments=["--submission-date={{ds}}"], parent_dag_name=dag.dag_id, dag_name="clients_last_seen_export", default_args=default_args, num_preemptible_workers=10), task_id="clients_last_seen_export", dag=dag) exact_mau_by_dimensions = BigQueryOperator( task_id='exact_mau_by_dimensions', bql='sql/firefox_desktop_exact_mau28_by_dimensions_v1.sql', destination_dataset_table='telemetry.firefox_desktop_exact_mau28_by_dimensions_v1${{ds_nodash}}', write_disposition='WRITE_TRUNCATE', use_legacy_sql=False, bigquery_conn_id="google_cloud_derived_datasets", dag=dag, )
email=[ "*****@*****.**", "*****@*****.**", "*****@*****.**" ], depends_on_past=True, start_date=datetime(2019, 4, 15), dag=dag) clients_last_seen_export = SubDagOperator(subdag=export_to_parquet( table="clients_last_seen_v1", arguments=[ "--dataset=telemetry_derived", "--submission-date={{ds}}", "--destination-table=clients_last_seen_v1", "--select", "cast(log2(days_seen_bits & -days_seen_bits) as long) as days_since_seen", "cast(log2(days_visited_5_uri_bits & -days_visited_5_uri_bits) as long) as days_since_visited_5_uri", "cast(log2(days_opened_dev_tools_bits & -days_opened_dev_tools_bits) as long) as days_since_opened_dev_tools", "cast(log2(days_created_profile_bits & -days_created_profile_bits) as long) as days_since_created_profile", "*" ], parent_dag_name=dag.dag_id, dag_name="clients_last_seen_export", default_args=default_args, num_preemptible_workers=10), task_id="clients_last_seen_export", dag=dag) exact_mau_by_dimensions = bigquery_etl_query( task_id="exact_mau_by_dimensions", destination_table="firefox_desktop_exact_mau28_by_dimensions_v1", dataset_id="telemetry", owner="*****@*****.**",
"input_event_response_coalesced_ms_main_above_250", "input_event_response_coalesced_ms_main_above_2500", "input_event_response_coalesced_ms_content_above_150", "input_event_response_coalesced_ms_content_above_250", "input_event_response_coalesced_ms_content_above_2500", "ghost_windows_main_above_1", "ghost_windows_content_above_1", ] main_summary_export = SubDagOperator(subdag=export_to_parquet( table= "moz-fx-data-shared-prod:telemetry_derived.main_summary_v4${{ds_nodash}}", static_partitions=["submission_date_s3={{ds_nodash}}"], arguments=[ "--partition-by=sample_id", "--replace='{{ds_nodash}}' AS submission_date", "--maps-from-entries", ] + main_summary_bigint_columns, parent_dag_name=dag.dag_id, dag_name="main_summary_export", default_args=default_args, num_workers=40), task_id="main_summary_export", executor=GetDefaultExecutor(), dag=dag) register_status(main_summary, "Main Summary", "A summary view of main pings.") addons = bigquery_etl_query(task_id="addons", destination_table="addons_v2", dataset_id="telemetry_derived",
project_id="moz-fx-data-shared-prod", dataset_id="telemetry_derived", sql_file_path="sql/telemetry_derived/main_summary_v4/", multipart=True, owner="*****@*****.**", email=["*****@*****.**", "*****@*****.**"], start_date=datetime(2019, 10, 25), dag=dag) sql_main_summary_export = SubDagOperator(subdag=export_to_parquet( table= "moz-fx-data-shared-prod:telemetry_derived.main_summary_v4${{ds_nodash}}", destination_table="sql_main_summary_v4", static_partitions="submission_date_s3={{ds_nodash}}", arguments=[ "--partition-by=sample_id", "--replace='{{ds_nodash}}' AS submission_date", "--maps-from-entries", ], parent_dag_name=dag.dag_id, dag_name="sql_main_summary_export", default_args=default_args, num_workers=40), task_id="sql_main_summary_export", dag=dag) main_summary = MozDatabricksSubmitRunOperator( task_id="main_summary", job_name="Main Summary View", execution_timeout=timedelta(hours=6), email=[ "*****@*****.**", "*****@*****.**",
dataset_id="telemetry_derived", owner="*****@*****.**", email=["*****@*****.**", "*****@*****.**"]) event_events_export = SubDagOperator( subdag=export_to_parquet( table="moz-fx-data-shared-prod:telemetry_derived.event_events_v1${{ds_nodash}}", destination_table="events_v1", static_partitions=["submission_date_s3={{ds_nodash}}", "doc_type=event"], arguments=[ "--drop=submission_date", "--partition-by=doc_type", "--replace", "UNIX_TIMESTAMP(timestamp) AS timestamp", "CAST(sample_id AS STRING) AS sample_id", "UNIX_TIMESTAMP(session_start_time) AS session_start_time", "MAP_FROM_ARRAYS(experiments.key, experiments.value.branch) AS experiments", "MAP_FROM_ENTRIES(event_map_values) AS event_map_values", "--bigint-columns", "sample_id", "event_timestamp", ], s3_output_bucket="telemetry-parquet", parent_dag_name=dag.dag_id, dag_name="event_events_export", default_args=default_args, num_workers=10), task_id="event_events_export", executor=GetDefaultExecutor(), owner="*****@*****.**", email=["*****@*****.**", "*****@*****.**"], dag=dag)