def test_execute_schedule_on_celery_k8s( # pylint: disable=redefined-outer-name, disable=unused-argument
dagster_instance_for_daemon, helm_namespace_for_daemon):
schedule_name = "frequent_celery"
with get_test_project_external_schedule(
dagster_instance_for_daemon, schedule_name) as external_schedule:
reoriginated_schedule = ReOriginatedExternalScheduleForTest(
external_schedule)
dagster_instance_for_daemon.start_schedule(reoriginated_schedule)
scheduler_runs = dagster_instance_for_daemon.get_runs(
RunsFilter(
tags=PipelineRun.tags_for_schedule(reoriginated_schedule)))
assert len(scheduler_runs) == 0
try:
start_time = time.time()
while True:
schedule_runs = dagster_instance_for_daemon.get_runs(
RunsFilter(tags=PipelineRun.tags_for_schedule(
reoriginated_schedule)))
if len(schedule_runs) > 0:
break
if time.time() - start_time > 120:
raise Exception(
"Timed out waiting for schedule to start a run. "
"Check the dagster-daemon pod logs to see why it didn't start."
)
time.sleep(1)
continue
finally:
dagster_instance_for_daemon.stop_schedule(
reoriginated_schedule.get_external_origin_id(),
reoriginated_schedule.selector_id,
reoriginated_schedule,
)
last_run = schedule_runs[0]
finished_pipeline_run = poll_for_finished_run(
dagster_instance_for_daemon, last_run.run_id, timeout=120)
assert finished_pipeline_run.is_success
def test_queue_from_schedule_and_sensor(instance, foo_example_workspace, foo_example_repo):
external_schedule = foo_example_repo.get_external_schedule("always_run_schedule")
external_sensor = foo_example_repo.get_external_sensor("always_on_sensor")
external_pipeline = foo_example_repo.get_full_external_pipeline("foo_pipeline")
instance.start_schedule_and_update_storage_state(external_schedule)
instance.start_sensor(external_sensor)
with start_daemon(timeout=180):
run = create_run(instance, external_pipeline)
instance.submit_run(run.run_id, foo_example_workspace)
runs = [
poll_for_finished_run(instance, run.run_id),
poll_for_finished_run(instance, run_tags=PipelineRun.tags_for_sensor(external_sensor)),
poll_for_finished_run(
instance,
run_tags=PipelineRun.tags_for_schedule(external_schedule),
timeout=90,
),
]
for run in runs:
logs = instance.all_logs(run.run_id)
assert_events_in_order(
logs,
[
"PIPELINE_ENQUEUED",
"PIPELINE_DEQUEUED",
"PIPELINE_STARTING",
"PIPELINE_START",
"PIPELINE_SUCCESS",
],
)
def get_execution_data(
self, context: "ScheduleExecutionContext"
) -> List[Union[RunRequest, SkipReason]]:
check.inst_param(context, "context", ScheduleExecutionContext)
execution_fn = cast(Callable[[ScheduleExecutionContext], Any],
self._execution_fn)
result = list(ensure_gen(execution_fn(context)))
if not result:
return []
if len(result) == 1:
check.is_list(result, of_type=(RunRequest, SkipReason))
data = result[0]
if isinstance(data, SkipReason):
return result
check.inst(data, RunRequest)
return [
RunRequest(
run_key=data.run_key,
run_config=data.run_config,
tags=merge_dicts(data.tags,
PipelineRun.tags_for_schedule(self)),
)
]
check.is_list(result, of_type=RunRequest)
check.invariant(
not any(not data.run_key for data in result),
"Schedules that return multiple RunRequests must specify a run_key in each RunRequest",
)
# clone all the run requests with the required schedule tags
return [
RunRequest(
run_key=data.run_key,
run_config=data.run_config,
tags=merge_dicts(data.tags,
PipelineRun.tags_for_schedule(self)),
) for data in result
]
def get_tags(self, context):
check.inst_param(context, "context", ScheduleExecutionContext)
if self._tags:
tags = self._tags
check_tags(tags, "tags")
else:
tags = self._tags_fn(context)
# These tags are checked in _tags_fn_wrapper
tags = merge_dicts(tags, PipelineRun.tags_for_schedule(self))
return tags
def _get_existing_run_for_request(instance, external_schedule, schedule_time, run_request):
tags = merge_dicts(
PipelineRun.tags_for_schedule(external_schedule),
{SCHEDULED_EXECUTION_TIME_TAG: schedule_time.in_tz("UTC").isoformat(),},
)
if run_request.run_key:
tags[RUN_KEY_TAG] = run_request.run_key
runs_filter = PipelineRunsFilter(tags=tags)
existing_runs = instance.get_runs(runs_filter)
if not len(existing_runs):
return None
return existing_runs[0]
def test_queue_from_schedule_and_sensor(tmpdir, foo_example_repo):
dagster_home_path = tmpdir.strpath
with setup_instance(
dagster_home_path,
"""run_coordinator:
module: dagster.core.run_coordinator
class: QueuedRunCoordinator
config:
dequeue_interval_seconds: 1
""",
) as instance:
external_schedule = foo_example_repo.get_external_schedule(
"never_run_schedule")
external_sensor = foo_example_repo.get_external_sensor(
"never_on_sensor")
foo_pipeline_handle = PipelineHandle("foo_pipeline",
foo_example_repo.handle)
instance.start_schedule_and_update_storage_state(external_schedule)
instance.start_sensor(external_sensor)
with start_daemon(timeout=180):
run = create_run(instance, foo_pipeline_handle)
with external_pipeline_from_run(run) as external_pipeline:
instance.submit_run(run.run_id, external_pipeline)
runs = [
poll_for_finished_run(instance, run.run_id),
poll_for_finished_run(
instance,
run_tags=PipelineRun.tags_for_sensor(external_sensor)),
poll_for_finished_run(
instance,
run_tags=PipelineRun.tags_for_schedule(
external_schedule),
timeout=90,
),
]
for run in runs:
logs = instance.all_logs(run.run_id)
assert_events_in_order(
logs,
[
"PIPELINE_ENQUEUED",
"PIPELINE_DEQUEUED",
"PIPELINE_STARTING",
"PIPELINE_START",
"PIPELINE_SUCCESS",
],
)
def _schedule_run_at_time(
instance,
logger,
repo_location,
schedule_state,
schedule_time_utc,
tick_holder,
debug_crash_flags,
):
schedule_name = schedule_state.name
repo_dict = repo_location.get_repositories()
check.invariant(
len(repo_dict) == 1,
"Reconstructed repository location should have exactly one repository",
)
external_repo = next(iter(repo_dict.values()))
external_schedule = external_repo.get_external_schedule(schedule_name)
pipeline_selector = PipelineSelector(
location_name=repo_location.name,
repository_name=external_repo.name,
pipeline_name=external_schedule.pipeline_name,
solid_selection=external_schedule.solid_selection,
)
subset_pipeline_result = repo_location.get_subset_external_pipeline_result(
pipeline_selector)
external_pipeline = ExternalPipeline(
subset_pipeline_result.external_pipeline_data,
external_repo.handle,
)
# Rule out the case where the scheduler crashed between creating a run for this time
# and launching it
runs_filter = PipelineRunsFilter(tags=merge_dicts(
PipelineRun.tags_for_schedule(schedule_state),
{SCHEDULED_EXECUTION_TIME_TAG: schedule_time_utc.isoformat()},
))
existing_runs = instance.get_runs(runs_filter)
run_to_launch = None
if len(existing_runs):
check.invariant(len(existing_runs) == 1)
run = existing_runs[0]
if run.status != PipelineRunStatus.NOT_STARTED:
# A run already exists and was launched for this time period,
# but the scheduler must have crashed before the tick could be put
# into a SUCCESS state
logger.info(
"Run {run_id} already completed for this execution of {schedule_name}"
.format(run_id=run.run_id, schedule_name=schedule_state.name))
tick_holder.update_with_status(ScheduleTickStatus.SUCCESS,
run_id=run.run_id)
return
else:
logger.info(
"Run {run_id} already created for this execution of {schedule_name}"
.format(run_id=run.run_id, schedule_name=schedule_state.name))
run_to_launch = run
else:
run_to_launch = _create_scheduler_run(
instance,
logger,
schedule_time_utc,
repo_location,
external_repo,
external_schedule,
external_pipeline,
tick_holder,
)
_check_for_debug_crash(debug_crash_flags, "RUN_CREATED")
if not run_to_launch:
check.invariant(tick_holder.status != ScheduleTickStatus.STARTED
and tick_holder.status != ScheduleTickStatus.SUCCESS)
return
if run_to_launch.status != PipelineRunStatus.FAILURE:
try:
instance.launch_run(run_to_launch.run_id, external_pipeline)
logger.info(
"Completed scheduled launch of run {run_id} for {schedule_name}"
.format(run_id=run_to_launch.run_id,
schedule_name=schedule_name))
except Exception as e: # pylint: disable=broad-except
if not isinstance(e, KeyboardInterrupt):
error = serializable_error_info_from_exc_info(sys.exc_info())
instance.report_engine_event(
error.message,
run_to_launch,
EngineEventData.engine_error(error),
)
instance.report_run_failed(run_to_launch)
logger.error(
"Run {run_id} created successfully but failed to launch.".
format(run_id=run_to_launch.run_id))
_check_for_debug_crash(debug_crash_flags, "RUN_LAUNCHED")
tick_holder.update_with_status(ScheduleTickStatus.SUCCESS,
run_id=run_to_launch.run_id)
_check_for_debug_crash(debug_crash_flags, "TICK_SUCCESS")
def _schedule_run_at_time(
instance,
logger,
repo_location,
external_repo,
external_schedule,
schedule_time,
tick_holder,
debug_crash_flags,
):
schedule_name = external_schedule.name
pipeline_selector = PipelineSelector(
location_name=repo_location.name,
repository_name=external_repo.name,
pipeline_name=external_schedule.pipeline_name,
solid_selection=external_schedule.solid_selection,
)
subset_pipeline_result = repo_location.get_subset_external_pipeline_result(
pipeline_selector)
external_pipeline = ExternalPipeline(
subset_pipeline_result.external_pipeline_data,
external_repo.handle,
)
# Rule out the case where the scheduler crashed between creating a run for this time
# and launching it
runs_filter = PipelineRunsFilter(tags=merge_dicts(
PipelineRun.tags_for_schedule(external_schedule),
{SCHEDULED_EXECUTION_TIME_TAG: schedule_time.in_tz("UTC").isoformat()},
))
existing_runs = instance.get_runs(runs_filter)
run_to_launch = None
if len(existing_runs):
check.invariant(len(existing_runs) == 1)
run = existing_runs[0]
if run.status != PipelineRunStatus.NOT_STARTED:
# A run already exists and was launched for this time period,
# but the scheduler must have crashed before the tick could be put
# into a SUCCESS state
logger.info(
"Run {run_id} already completed for this execution of {schedule_name}"
.format(run_id=run.run_id, schedule_name=schedule_name))
tick_holder.update_with_status(JobTickStatus.SUCCESS,
run_id=run.run_id)
return
else:
logger.info(
"Run {run_id} already created for this execution of {schedule_name}"
.format(run_id=run.run_id, schedule_name=schedule_name))
run_to_launch = run
else:
run_to_launch = _create_scheduler_run(
instance,
logger,
schedule_time,
repo_location,
external_repo,
external_schedule,
external_pipeline,
tick_holder,
)
_check_for_debug_crash(debug_crash_flags, "RUN_CREATED")
if not run_to_launch:
check.invariant(tick_holder.status != JobTickStatus.STARTED
and tick_holder.status != JobTickStatus.SUCCESS)
return
if run_to_launch.status != PipelineRunStatus.FAILURE:
try:
instance.submit_run(run_to_launch.run_id, external_pipeline)
logger.info(
"Completed scheduled launch of run {run_id} for {schedule_name}"
.format(run_id=run_to_launch.run_id,
schedule_name=schedule_name))
except Exception: # pylint: disable=broad-except
logger.error(
"Run {run_id} created successfully but failed to launch.".
format(run_id=run_to_launch.run_id))
_check_for_debug_crash(debug_crash_flags, "RUN_LAUNCHED")
tick_holder.update_with_status(JobTickStatus.SUCCESS,
run_id=run_to_launch.run_id)
_check_for_debug_crash(debug_crash_flags, "TICK_SUCCESS")
def get_tags(self, context):
check.inst_param(context, "context", ScheduleExecutionContext)
tags = self._tags_fn(context)
return merge_dicts(tags, PipelineRun.tags_for_schedule(self))
