def define_schedules():
integer_partition_set = PartitionSetDefinition(
name="scheduled_integer_partitions",
pipeline_name="no_config_pipeline",
partition_fn=lambda: [Partition(x) for x in range(1, 10)],
tags_fn_for_partition=lambda _partition: {"test": "1234"},
)
no_config_pipeline_hourly_schedule = ScheduleDefinition(
name="no_config_pipeline_hourly_schedule",
cron_schedule="0 0 * * *",
pipeline_name="no_config_pipeline",
)
no_config_pipeline_hourly_schedule_with_config_fn = ScheduleDefinition(
name="no_config_pipeline_hourly_schedule_with_config_fn",
cron_schedule="0 0 * * *",
pipeline_name="no_config_pipeline",
)
no_config_should_execute = ScheduleDefinition(
name="no_config_should_execute",
cron_schedule="0 0 * * *",
pipeline_name="no_config_pipeline",
should_execute=lambda _context: False,
)
dynamic_config = ScheduleDefinition(
name="dynamic_config",
cron_schedule="0 0 * * *",
pipeline_name="no_config_pipeline",
)
partition_based = integer_partition_set.create_schedule_definition(
schedule_name="partition_based",
cron_schedule="0 0 * * *",
partition_selector=last_empty_partition,
)
@daily_schedule(
pipeline_name="no_config_pipeline",
start_date=today_at_midnight().subtract(days=1),
execution_time=(datetime.datetime.now() +
datetime.timedelta(hours=2)).time(),
)
def partition_based_decorator(_date):
return {}
@daily_schedule(
pipeline_name="no_config_pipeline",
start_date=today_at_midnight().subtract(days=1),
execution_time=(datetime.datetime.now() +
datetime.timedelta(hours=2)).time(),
default_status=DefaultScheduleStatus.RUNNING,
)
def running_in_code_schedule(_date):
return {}
@daily_schedule(
pipeline_name="multi_mode_with_loggers",
start_date=today_at_midnight().subtract(days=1),
execution_time=(datetime.datetime.now() +
datetime.timedelta(hours=2)).time(),
mode="foo_mode",
)
def partition_based_multi_mode_decorator(_date):
return {}
@hourly_schedule(
pipeline_name="no_config_chain_pipeline",
start_date=today_at_midnight().subtract(days=1),
execution_time=(datetime.datetime.now() +
datetime.timedelta(hours=2)).time(),
solid_selection=["return_foo"],
)
def solid_selection_hourly_decorator(_date):
return {}
@daily_schedule(
pipeline_name="no_config_chain_pipeline",
start_date=today_at_midnight().subtract(days=2),
execution_time=(datetime.datetime.now() +
datetime.timedelta(hours=3)).time(),
solid_selection=["return_foo"],
)
def solid_selection_daily_decorator(_date):
return {}
@monthly_schedule(
pipeline_name="no_config_chain_pipeline",
start_date=(today_at_midnight().subtract(days=100)).replace(day=1),
execution_time=(datetime.datetime.now() +
datetime.timedelta(hours=4)).time(),
solid_selection=["return_foo"],
)
def solid_selection_monthly_decorator(_date):
return {}
@weekly_schedule(
pipeline_name="no_config_chain_pipeline",
start_date=today_at_midnight().subtract(days=50),
execution_time=(datetime.datetime.now() +
datetime.timedelta(hours=5)).time(),
solid_selection=["return_foo"],
)
def solid_selection_weekly_decorator(_date):
return {}
# Schedules for testing the user error boundary
@daily_schedule(
pipeline_name="no_config_pipeline",
start_date=today_at_midnight().subtract(days=1),
should_execute=lambda _: asdf, # pylint: disable=undefined-variable
)
def should_execute_error_schedule(_date):
return {}
@daily_schedule(
pipeline_name="no_config_pipeline",
start_date=today_at_midnight().subtract(days=1),
tags_fn_for_date=lambda _: asdf, # pylint: disable=undefined-variable
)
def tags_error_schedule(_date):
return {}
@daily_schedule(
pipeline_name="no_config_pipeline",
start_date=today_at_midnight().subtract(days=1),
)
def run_config_error_schedule(_date):
return asdf # pylint: disable=undefined-variable
@daily_schedule(
pipeline_name="no_config_pipeline",
start_date=today_at_midnight("US/Central") -
datetime.timedelta(days=1),
execution_timezone="US/Central",
)
def timezone_schedule(_date):
return {}
tagged_pipeline_schedule = ScheduleDefinition(
name="tagged_pipeline_schedule",
cron_schedule="0 0 * * *",
pipeline_name="tagged_pipeline",
)
tagged_pipeline_override_schedule = ScheduleDefinition(
name="tagged_pipeline_override_schedule",
cron_schedule="0 0 * * *",
pipeline_name="tagged_pipeline",
tags={"foo": "notbar"},
)
invalid_config_schedule = ScheduleDefinition(
name="invalid_config_schedule",
cron_schedule="0 0 * * *",
pipeline_name="pipeline_with_enum_config",
run_config={"solids": {
"takes_an_enum": {
"config": "invalid"
}
}},
)
return [
run_config_error_schedule,
no_config_pipeline_hourly_schedule,
no_config_pipeline_hourly_schedule_with_config_fn,
no_config_should_execute,
dynamic_config,
partition_based,
partition_based_decorator,
partition_based_multi_mode_decorator,
solid_selection_hourly_decorator,
solid_selection_daily_decorator,
solid_selection_monthly_decorator,
solid_selection_weekly_decorator,
should_execute_error_schedule,
tagged_pipeline_schedule,
tagged_pipeline_override_schedule,
tags_error_schedule,
timezone_schedule,
invalid_config_schedule,
running_in_code_schedule,
]
from dagster.core.definitions.reconstructable import ReconstructableRepository
from dagster.core.instance import DagsterInstance
from dagster.core.origin import RepositoryGrpcServerOrigin
from dagster.core.scheduler import (
ScheduleTickStatus,
ScheduledExecutionFailed,
ScheduledExecutionSkipped,
ScheduledExecutionSuccess,
)
from dagster.core.telemetry import get_dir_from_dagster_home
from dagster.core.test_utils import instance_for_test, today_at_midnight
from dagster.core.types.loadable_target_origin import LoadableTargetOrigin
from dagster.grpc.server import GrpcServerProcess
from dagster.utils import find_free_port
_COUPLE_DAYS_AGO = today_at_midnight() - datetime.timedelta(days=2)
def _throw(_context):
raise Exception("bananas")
def _never(_context):
return False
@solid(config_schema={"work_amt": str})
def the_solid(context):
return "0.8.0 was {} of work".format(context.solid_config["work_amt"])
ScheduledExecutionResult,
ScheduledExecutionSkipped,
ScheduledExecutionSuccess,
)
from dagster.core.scheduler.job import JobTickStatus
from dagster.core.storage.pipeline_run import PipelineRunStatus
from dagster.core.telemetry import get_dir_from_dagster_home
from dagster.core.test_utils import instance_for_test, today_at_midnight
from dagster.core.types.loadable_target_origin import LoadableTargetOrigin
from dagster.grpc.server import GrpcServerProcess
from dagster.serdes.ipc import IPCErrorMessage, read_unary_response
from dagster.seven import xplat_shlex_split
from dagster.utils import find_free_port
from dagster.utils.temp_file import get_temp_file_name
_COUPLE_DAYS_AGO = today_at_midnight().subtract(days=2)
def _throw(_context):
raise Exception("bananas")
def _never(_context):
return False
@solid(config_schema={"work_amt": str})
def the_solid(context):
return "0.8.0 was {} of work".format(context.solid_config["work_amt"])
@lambda_solid(input_defs=[InputDefinition("num", Int)],
output_def=OutputDefinition(Int))
def add_one(num):
return num + 1
@lambda_solid(input_defs=[InputDefinition("num", Int)],
output_def=OutputDefinition(Int))
def mult_two(num):
return num * 2
@pipeline
def math():
return mult_two(num=add_one())
@daily_schedule(
pipeline_name="math",
start_date=today_at_midnight(),
)
def my_schedule(_):
return {"solids": {"mult_two": {"inputs": {"num": {"value": 2}}}}}
@repository
def test_repository():
return [math]
