addon_aggregates_bigquery_load.set_upstream(addon_aggregates) main_events.set_upstream(main_summary) main_events_bigquery_load.set_upstream(main_events) main_summary_experiments.set_upstream(main_summary) main_summary_experiments_bigquery_load.set_upstream(main_summary_experiments) experiments_aggregates_import.set_upstream(main_summary_experiments) search_dashboard.set_upstream(main_summary) search_dashboard_bigquery_load.set_upstream(search_dashboard) search_clients_daily.set_upstream(main_summary) search_clients_daily_bigquery_load.set_upstream(search_clients_daily) taar_dynamo.set_upstream(main_summary) taar_similarity.set_upstream(clients_daily_v6) clients_daily.set_upstream(main_summary) clients_daily_v6.set_upstream(main_summary) desktop_active_dau.set_upstream(clients_daily_v6) clients_daily_v6_bigquery_load.set_upstream(clients_daily_v6) clients_last_seen.set_upstream(clients_daily_v6_bigquery_load) clients_last_seen_export.set_upstream(clients_last_seen) exact_mau_by_dimensions.set_upstream(clients_last_seen) exact_mau_by_dimensions_export.set_upstream(exact_mau_by_dimensions) retention.set_upstream(main_summary) retention_bigquery_load.set_upstream(retention) client_count_daily_view.set_upstream(main_summary) desktop_dau.set_upstream(client_count_daily_view)
execution_delta=timedelta(
days=-7, hours=-1
), # main_summary waits one hour, execution date is beginning of the week
dag=taar_weekly,
)
taar_ensemble = MozDatabricksSubmitRunOperator(
task_id="taar_ensemble",
job_name="TAAR Ensemble Model",
owner="*****@*****.**",
email=["*****@*****.**", "*****@*****.**"],
execution_timeout=timedelta(hours=11),
instance_count=5,
instance_type="i3.2xlarge",
spot_bid_price_percent=100,
max_instance_count=60,
enable_autoscale=True,
pypi_libs=[
"mozilla-taar3==0.4.5",
"mozilla-srgutil==0.1.10",
"python-decouple==3.1",
],
env=mozetl_envvar("taar_ensemble", {"date": "{{ ds_nodash }}"}),
start_date=datetime(2019, 7, 14),
uri=
"https://raw.githubusercontent.com/mozilla/python_mozetl/master/bin/mozetl-databricks.py",
output_visibility="private",
)
taar_ensemble.set_upstream(wait_for_clients_daily)
main_summary_dataproc.set_upstream(main_ping_bigquery_export) main_ping_bigquery_export_delete.set_upstream(main_summary_dataproc) main_summary_dataproc_bigquery_load.set_upstream(main_summary_dataproc) main_summary_dataproc_s3_copy.set_upstream(main_summary_dataproc) engagement_ratio.set_upstream(main_summary) addons.set_upstream(main_summary) addons_bigquery_load.set_upstream(addons) addon_aggregates.set_upstream(addons) addon_aggregates_bigquery_load.set_upstream(addon_aggregates) main_events.set_upstream(main_summary) main_events_bigquery_load.set_upstream(main_events) main_summary_experiments.set_upstream(main_summary) main_summary_experiments_bigquery_load.set_upstream(main_summary_experiments) experiments_aggregates_import.set_upstream(main_summary_experiments) search_dashboard.set_upstream(main_summary) search_dashboard_bigquery_load.set_upstream(search_dashboard) search_clients_daily.set_upstream(main_summary) search_clients_daily_bigquery_load.set_upstream(search_clients_daily) taar_dynamo.set_upstream(main_summary) taar_similarity.set_upstream(clients_daily_v6) clients_daily.set_upstream(main_summary) clients_daily_v6.set_upstream(main_summary) desktop_active_dau.set_upstream(clients_daily_v6) clients_daily_v6_bigquery_load.set_upstream(clients_daily_v6)
)
addons_daily = MozDatabricksSubmitRunOperator(
task_id="addons_daily",
job_name="Addons Daily",
execution_timeout=timedelta(hours=4),
instance_count=10,
owner="*****@*****.**",
email=[
"*****@*****.**",
"*****@*****.**",
"*****@*****.**",
"*****@*****.**",
"*****@*****.**",
],
env=mozetl_envvar(
"addons_report",
{
"date": "{{ ds_nodash }}",
"deploy_environment": "{{ task.__class__.deploy_environment }}",
},
other={
"MOZETL_GIT_PATH": "https://github.com/mozilla/addons_daily.git",
"MOZETL_EXTERNAL_MODULE": "addons_daily",
},
),
dag=dag,
)
addons_daily.set_upstream(wait_for_search_clients_daily)
engagement_ratio.set_upstream(main_summary) addons.set_upstream(main_summary) addon_aggregates.set_upstream(addons) main_events.set_upstream(main_summary) main_summary_experiments.set_upstream(main_summary) experiments_aggregates_import.set_upstream(main_summary_experiments) search_dashboard.set_upstream(main_summary) search_clients_daily.set_upstream(main_summary) taar_dynamo.set_upstream(main_summary) taar_similarity.set_upstream(clients_daily_v6) clients_daily.set_upstream(main_summary) clients_daily_v6.set_upstream(main_summary) desktop_active_dau.set_upstream(clients_daily_v6) retention.set_upstream(main_summary) client_count_daily_view.set_upstream(main_summary) desktop_dau.set_upstream(client_count_daily_view) main_summary_glue.set_upstream(main_summary) taar_locale_job.set_upstream(clients_daily_v6) taar_collaborative_recommender.set_upstream(clients_daily_v6)
main_summary_export.set_upstream(main_summary)
clients_daily.set_upstream(main_summary)
clients_daily_export.set_upstream(clients_daily)
addons.set_upstream(copy_deduplicate_main_ping)
addons_export.set_upstream(addons)
addon_aggregates.set_upstream(copy_deduplicate_main_ping)
addon_aggregates_export.set_upstream(addon_aggregates)
main_summary_experiments.set_upstream(main_summary)
main_summary_experiments.set_upstream(
main_summary_experiments_get_experiment_list)
main_summary_experiments_export.set_upstream(main_summary_experiments)
taar_dynamo.set_upstream(main_summary_export)
taar_similarity.set_upstream(clients_daily_export)
clients_last_seen.set_upstream(clients_daily)
clients_last_seen_export.set_upstream(clients_last_seen)
exact_mau_by_dimensions.set_upstream(clients_last_seen)
exact_mau_by_dimensions_export.set_upstream(exact_mau_by_dimensions)
exact_mau_by_client_count_dimensions.set_upstream(clients_last_seen)
smoot_usage_desktop_v2.set_upstream(clients_last_seen)
simpleprophet_forecasts_desktop.set_upstream(exact_mau_by_dimensions)
devtools_panel_usage.set_upstream(clients_daily)
main_summary_glue.set_upstream(main_summary_export)
taar_locale_job.set_upstream(clients_daily_export)
taar_collaborative_recommender.set_upstream(clients_daily_export)