def define_schedules(): integer_partition_set = PartitionSetDefinition( name="scheduled_integer_partitions", pipeline_name="no_config_pipeline", partition_fn=lambda: [Partition(x) for x in range(1, 10)], tags_fn_for_partition=lambda _partition: {"test": "1234"}, ) no_config_pipeline_hourly_schedule = ScheduleDefinition( name="no_config_pipeline_hourly_schedule", cron_schedule="0 0 * * *", pipeline_name="no_config_pipeline", ) no_config_pipeline_hourly_schedule_with_config_fn = ScheduleDefinition( name="no_config_pipeline_hourly_schedule_with_config_fn", cron_schedule="0 0 * * *", pipeline_name="no_config_pipeline", ) no_config_should_execute = ScheduleDefinition( name="no_config_should_execute", cron_schedule="0 0 * * *", pipeline_name="no_config_pipeline", should_execute=lambda _context: False, ) dynamic_config = ScheduleDefinition( name="dynamic_config", cron_schedule="0 0 * * *", pipeline_name="no_config_pipeline", ) partition_based = integer_partition_set.create_schedule_definition( schedule_name="partition_based", cron_schedule="0 0 * * *", partition_selector=last_empty_partition, ) @daily_schedule( pipeline_name="no_config_pipeline", start_date=today_at_midnight().subtract(days=1), execution_time=(datetime.datetime.now() + datetime.timedelta(hours=2)).time(), ) def partition_based_decorator(_date): return {} @daily_schedule( pipeline_name="no_config_pipeline", start_date=today_at_midnight().subtract(days=1), execution_time=(datetime.datetime.now() + datetime.timedelta(hours=2)).time(), default_status=DefaultScheduleStatus.RUNNING, ) def running_in_code_schedule(_date): return {} @daily_schedule( pipeline_name="multi_mode_with_loggers", start_date=today_at_midnight().subtract(days=1), execution_time=(datetime.datetime.now() + datetime.timedelta(hours=2)).time(), mode="foo_mode", ) def partition_based_multi_mode_decorator(_date): return {} @hourly_schedule( pipeline_name="no_config_chain_pipeline", start_date=today_at_midnight().subtract(days=1), execution_time=(datetime.datetime.now() + datetime.timedelta(hours=2)).time(), solid_selection=["return_foo"], ) def solid_selection_hourly_decorator(_date): return {} @daily_schedule( pipeline_name="no_config_chain_pipeline", start_date=today_at_midnight().subtract(days=2), execution_time=(datetime.datetime.now() + datetime.timedelta(hours=3)).time(), solid_selection=["return_foo"], ) def solid_selection_daily_decorator(_date): return {} @monthly_schedule( pipeline_name="no_config_chain_pipeline", start_date=(today_at_midnight().subtract(days=100)).replace(day=1), execution_time=(datetime.datetime.now() + datetime.timedelta(hours=4)).time(), solid_selection=["return_foo"], ) def solid_selection_monthly_decorator(_date): return {} @weekly_schedule( pipeline_name="no_config_chain_pipeline", start_date=today_at_midnight().subtract(days=50), execution_time=(datetime.datetime.now() + datetime.timedelta(hours=5)).time(), solid_selection=["return_foo"], ) def solid_selection_weekly_decorator(_date): return {} # Schedules for testing the user error boundary @daily_schedule( pipeline_name="no_config_pipeline", start_date=today_at_midnight().subtract(days=1), should_execute=lambda _: asdf, # pylint: disable=undefined-variable ) def should_execute_error_schedule(_date): return {} @daily_schedule( pipeline_name="no_config_pipeline", start_date=today_at_midnight().subtract(days=1), tags_fn_for_date=lambda _: asdf, # pylint: disable=undefined-variable ) def tags_error_schedule(_date): return {} @daily_schedule( pipeline_name="no_config_pipeline", start_date=today_at_midnight().subtract(days=1), ) def run_config_error_schedule(_date): return asdf # pylint: disable=undefined-variable @daily_schedule( pipeline_name="no_config_pipeline", start_date=today_at_midnight("US/Central") - datetime.timedelta(days=1), execution_timezone="US/Central", ) def timezone_schedule(_date): return {} tagged_pipeline_schedule = ScheduleDefinition( name="tagged_pipeline_schedule", cron_schedule="0 0 * * *", pipeline_name="tagged_pipeline", ) tagged_pipeline_override_schedule = ScheduleDefinition( name="tagged_pipeline_override_schedule", cron_schedule="0 0 * * *", pipeline_name="tagged_pipeline", tags={"foo": "notbar"}, ) invalid_config_schedule = ScheduleDefinition( name="invalid_config_schedule", cron_schedule="0 0 * * *", pipeline_name="pipeline_with_enum_config", run_config={"solids": { "takes_an_enum": { "config": "invalid" } }}, ) return [ run_config_error_schedule, no_config_pipeline_hourly_schedule, no_config_pipeline_hourly_schedule_with_config_fn, no_config_should_execute, dynamic_config, partition_based, partition_based_decorator, partition_based_multi_mode_decorator, solid_selection_hourly_decorator, solid_selection_daily_decorator, solid_selection_monthly_decorator, solid_selection_weekly_decorator, should_execute_error_schedule, tagged_pipeline_schedule, tagged_pipeline_override_schedule, tags_error_schedule, timezone_schedule, invalid_config_schedule, running_in_code_schedule, ]
from dagster.core.definitions.reconstructable import ReconstructableRepository from dagster.core.instance import DagsterInstance from dagster.core.origin import RepositoryGrpcServerOrigin from dagster.core.scheduler import ( ScheduleTickStatus, ScheduledExecutionFailed, ScheduledExecutionSkipped, ScheduledExecutionSuccess, ) from dagster.core.telemetry import get_dir_from_dagster_home from dagster.core.test_utils import instance_for_test, today_at_midnight from dagster.core.types.loadable_target_origin import LoadableTargetOrigin from dagster.grpc.server import GrpcServerProcess from dagster.utils import find_free_port _COUPLE_DAYS_AGO = today_at_midnight() - datetime.timedelta(days=2) def _throw(_context): raise Exception("bananas") def _never(_context): return False @solid(config_schema={"work_amt": str}) def the_solid(context): return "0.8.0 was {} of work".format(context.solid_config["work_amt"])
ScheduledExecutionResult, ScheduledExecutionSkipped, ScheduledExecutionSuccess, ) from dagster.core.scheduler.job import JobTickStatus from dagster.core.storage.pipeline_run import PipelineRunStatus from dagster.core.telemetry import get_dir_from_dagster_home from dagster.core.test_utils import instance_for_test, today_at_midnight from dagster.core.types.loadable_target_origin import LoadableTargetOrigin from dagster.grpc.server import GrpcServerProcess from dagster.serdes.ipc import IPCErrorMessage, read_unary_response from dagster.seven import xplat_shlex_split from dagster.utils import find_free_port from dagster.utils.temp_file import get_temp_file_name _COUPLE_DAYS_AGO = today_at_midnight().subtract(days=2) def _throw(_context): raise Exception("bananas") def _never(_context): return False @solid(config_schema={"work_amt": str}) def the_solid(context): return "0.8.0 was {} of work".format(context.solid_config["work_amt"])
@lambda_solid(input_defs=[InputDefinition("num", Int)], output_def=OutputDefinition(Int)) def add_one(num): return num + 1 @lambda_solid(input_defs=[InputDefinition("num", Int)], output_def=OutputDefinition(Int)) def mult_two(num): return num * 2 @pipeline def math(): return mult_two(num=add_one()) @daily_schedule( pipeline_name="math", start_date=today_at_midnight(), ) def my_schedule(_): return {"solids": {"mult_two": {"inputs": {"num": {"value": 2}}}}} @repository def test_repository(): return [math]