def hacker_news_staging():
return [
build_schedule_from_partitioned_job(download_staging_job),
make_slack_on_failure_sensor(base_url="my_staging_dagit_url.com"),
make_hn_tables_updated_sensor(story_recommender_staging_job),
make_hn_tables_updated_sensor(dbt_staging_job),
]
def test_job_scheduled_partitions():
@graph
def my_graph():
pass
@daily_partitioned_config(start_date="2021-09-01")
def daily_schedule_config(_start, _end):
return {}
my_job = my_graph.to_job(config=daily_schedule_config)
my_schedule = build_schedule_from_partitioned_job(my_job)
@repository
def schedule_repo():
return [my_schedule]
@repository
def job_repo():
return [my_job]
@repository
def schedule_job_repo():
return [my_job, my_schedule]
assert len(schedule_repo.partition_set_defs) == 1
assert schedule_repo.get_partition_set_def("my_graph_partition_set")
assert len(job_repo.partition_set_defs) == 1
assert job_repo.get_partition_set_def("my_graph_partition_set")
assert len(schedule_job_repo.partition_set_defs) == 1
assert schedule_job_repo.get_partition_set_def("my_graph_partition_set")
assert len(schedule_job_repo.job_names) == 1
def monthly_materialization_schedule():
@monthly_partitioned_config(start_date=datetime.datetime(2021, 1, 1),
timezone=_toys_tz_info())
def monthly_materialization_config(_start, _end):
return {}
return build_schedule_from_partitioned_job(
many_events.to_job("many_events_monthly",
config=monthly_materialization_config), )
def unreliable_job_test_schedule():
@weekly_partitioned_config(start_date="2020-01-05",
timezone=_toys_tz_info())
def unreliable_config(_start, _end):
return {}
unreliable_weekly_job = unreliable.to_job(config=unreliable_config)
unreliable_weekly_schedule = build_schedule_from_partitioned_job(
unreliable_weekly_job)
return unreliable_weekly_schedule
def longitudinal_schedule():
from .longitudinal import longitudinal_job
@daily_partitioned_config(start_date="2020-01-01",
timezone=_toys_tz_info())
def longitudinal_config(start, _end):
return {
"ops": {
op.name: {
"config": {
"partition": start.to_date_string()
}
}
for op in longitudinal_job.solids
}
}
longitudinal_job = longitudinal.to_job(config=longitudinal_config)
return build_schedule_from_partitioned_job(longitudinal_job)
"""isort:skip_file"""
from .partitioned_job import my_partitioned_config
# start_marker
from dagster import build_schedule_from_partitioned_job, job
@job(config=my_partitioned_config)
def do_stuff_partitioned():
...
do_stuff_partitioned_schedule = build_schedule_from_partitioned_job(
do_stuff_partitioned, )
# end_marker
@op(config_schema={"date": str})
def do_something_with_config(context):
return context.op_config["date"]
@daily_partitioned_config(start_date="2020-01-01")
def do_it_all_config(start, _end):
return {
"solids": {
"do_something_with_config": {
"config": {
"date": str(start)
}
}
}
}
@job(config=do_it_all_config)
def do_it_all():
do_something_with_config()
do_it_all_schedule = build_schedule_from_partitioned_job(do_it_all)
@repository
def do_it_all_repo():
return [do_it_all_schedule]