def attempt_to_launch_runs(self):
max_runs_to_launch = self._max_concurrent_runs - self._count_in_progress_runs(
)
# Possibly under 0 if runs were launched without queuing
if max_runs_to_launch <= 0:
return
runs = self._get_queued_runs(limit=max_runs_to_launch)
for run in runs:
with external_pipeline_from_run(run) as external_pipeline:
enqueued_event = DagsterEvent(
event_type_value=DagsterEventType.PIPELINE_DEQUEUED.value,
pipeline_name=run.pipeline_name,
)
event_record = DagsterEventRecord(
message="",
user_message="",
level=logging.INFO,
pipeline_name=run.pipeline_name,
run_id=run.run_id,
error_info=None,
timestamp=time.time(),
dagster_event=enqueued_event,
)
self._instance.handle_new_event(event_record)
self._instance.launch_run(run.run_id, external_pipeline)
def _dequeue_run(self, run):
with external_pipeline_from_run(run) as external_pipeline:
# double check that the run is still queued before dequeing
reloaded_run = self._instance.get_run_by_id(run.run_id)
if reloaded_run.status != PipelineRunStatus.QUEUED:
self._logger.info(
"Run {run_id} is now {status} instead of QUEUED, skipping".
format(run_id=reloaded_run.run_id,
status=reloaded_run.status))
return
dequeued_event = DagsterEvent(
event_type_value=DagsterEventType.PIPELINE_DEQUEUED.value,
pipeline_name=run.pipeline_name,
)
event_record = DagsterEventRecord(
message="",
user_message="",
level=logging.INFO,
pipeline_name=run.pipeline_name,
run_id=run.run_id,
error_info=None,
timestamp=time.time(),
dagster_event=dequeued_event,
)
self._instance.handle_new_event(event_record)
self._instance.launch_run(run.run_id, external_pipeline)
def test_queued_runs(tmpdir, foo_pipeline_handle):
dagster_home_path = tmpdir.strpath
with setup_instance(
dagster_home_path,
"""run_coordinator:
module: dagster.core.run_coordinator
class: QueuedRunCoordinator
config:
dequeue_interval_seconds: 1
""",
) as instance:
with start_daemon():
run = create_run(instance, foo_pipeline_handle)
with external_pipeline_from_run(run) as external_pipeline:
instance.submit_run(run.run_id, external_pipeline)
poll_for_finished_run(instance, run.run_id)
logs = instance.all_logs(run.run_id)
assert_events_in_order(
logs,
[
"PIPELINE_ENQUEUED",
"PIPELINE_DEQUEUED",
"PIPELINE_STARTING",
"PIPELINE_START",
"PIPELINE_SUCCESS",
],
)
def test_external_pipeline_from_run():
with instance_for_test() as instance:
with get_foo_pipeline_handle() as pipeline_handle:
run = create_run_for_test(
instance,
pipeline_name=pipeline_handle.pipeline_name,
external_pipeline_origin=pipeline_handle.get_external_origin(),
)
with external_pipeline_from_run(run) as external_pipeline:
assert external_pipeline.name == pipeline_handle.pipeline_name
def test_queue_from_schedule_and_sensor(tmpdir, foo_example_repo):
dagster_home_path = tmpdir.strpath
with setup_instance(
dagster_home_path,
"""run_coordinator:
module: dagster.core.run_coordinator
class: QueuedRunCoordinator
config:
dequeue_interval_seconds: 1
""",
) as instance:
external_schedule = foo_example_repo.get_external_schedule(
"never_run_schedule")
external_sensor = foo_example_repo.get_external_sensor(
"never_on_sensor")
foo_pipeline_handle = PipelineHandle("foo_pipeline",
foo_example_repo.handle)
instance.start_schedule_and_update_storage_state(external_schedule)
instance.start_sensor(external_sensor)
with start_daemon(timeout=180):
run = create_run(instance, foo_pipeline_handle)
with external_pipeline_from_run(run) as external_pipeline:
instance.submit_run(run.run_id, external_pipeline)
runs = [
poll_for_finished_run(instance, run.run_id),
poll_for_finished_run(
instance,
run_tags=PipelineRun.tags_for_sensor(external_sensor)),
poll_for_finished_run(
instance,
run_tags=PipelineRun.tags_for_schedule(
external_schedule),
timeout=90,
),
]
for run in runs:
logs = instance.all_logs(run.run_id)
assert_events_in_order(
logs,
[
"PIPELINE_ENQUEUED",
"PIPELINE_DEQUEUED",
"PIPELINE_STARTING",
"PIPELINE_START",
"PIPELINE_SUCCESS",
],
)
def test_queued_runs(tmpdir, foo_pipeline_handle):
dagster_home_path = tmpdir.strpath
setup_instance(dagster_home_path)
with DagsterInstance.get() as instance:
with start_daemon():
run = create_run(instance, foo_pipeline_handle)
with external_pipeline_from_run(run) as external_pipeline:
instance.submit_run(run.run_id, external_pipeline)
poll_for_finished_run(instance, run.run_id)
logs = instance.all_logs(run.run_id)
assert_events_in_order(
logs,
["PIPELINE_ENQUEUED", "PIPELINE_DEQUEUED", "PIPELINE_SUCCESS"],
)
def _dequeue_run(self, run):
with external_pipeline_from_run(run) as external_pipeline:
dequeued_event = DagsterEvent(
event_type_value=DagsterEventType.PIPELINE_DEQUEUED.value,
pipeline_name=run.pipeline_name,
)
event_record = DagsterEventRecord(
message="",
user_message="",
level=logging.INFO,
pipeline_name=run.pipeline_name,
run_id=run.run_id,
error_info=None,
timestamp=time.time(),
dagster_event=dequeued_event,
)
self._instance.handle_new_event(event_record)
self._instance.launch_run(run.run_id, external_pipeline)
def run_iteration(self):
in_progress = self._count_in_progress_runs()
max_runs_to_launch = self._max_concurrent_runs - in_progress
# Possibly under 0 if runs were launched without queuing
if max_runs_to_launch <= 0:
self._logger.info(
"{} runs are currently in progress. Maximum is {}, won't launch more.".format(
in_progress, self._max_concurrent_runs
)
)
return
queued_runs = self._get_queued_runs(limit=max_runs_to_launch)
if not queued_runs:
self._logger.info("Poll returned no queued runs.")
else:
self._logger.info("Retrieved {} queued runs to launch.".format(len(queued_runs)))
for run in queued_runs:
with external_pipeline_from_run(run) as external_pipeline:
enqueued_event = DagsterEvent(
event_type_value=DagsterEventType.PIPELINE_DEQUEUED.value,
pipeline_name=run.pipeline_name,
)
event_record = DagsterEventRecord(
message="",
user_message="",
level=logging.INFO,
pipeline_name=run.pipeline_name,
run_id=run.run_id,
error_info=None,
timestamp=time.time(),
dagster_event=enqueued_event,
)
self._instance.handle_new_event(event_record)
self._instance.launch_run(run.run_id, external_pipeline)
def call_submit_run(coodinator, run): # pylint: disable=redefined-outer-name
with external_pipeline_from_run(run) as external:
return coodinator.submit_run(run, external)