def test_get_required_daemon_types():
from dagster.daemon.daemon import (
SensorDaemon,
BackfillDaemon,
SchedulerDaemon,
MonitoringDaemon,
)
with instance_for_test() as instance:
assert instance.get_required_daemon_types() == [
SensorDaemon.daemon_type(),
BackfillDaemon.daemon_type(),
SchedulerDaemon.daemon_type(),
]
with instance_for_test(
overrides={
"run_launcher": {
"module": "dagster_tests.daemon_tests.test_monitoring_daemon",
"class": "TestRunLauncher",
},
"run_monitoring": {"enabled": True},
}
) as instance:
assert instance.get_required_daemon_types() == [
SensorDaemon.daemon_type(),
BackfillDaemon.daemon_type(),
SchedulerDaemon.daemon_type(),
MonitoringDaemon.daemon_type(),
]
def test_thread_die_daemon(monkeypatch):
with instance_for_test(overrides={}) as instance:
from dagster.daemon.daemon import SchedulerDaemon, SensorDaemon
iteration_ran = {"ran": False}
def run_loop_error(_, _instance, _workspace):
iteration_ran["ran"] = True
raise KeyboardInterrupt
yield # pylint: disable=unreachable
monkeypatch.setattr(SensorDaemon, "core_loop", run_loop_error)
heartbeat_interval_seconds = 1
init_time = pendulum.now("UTC")
with daemon_controller_from_instance(
instance,
workspace_load_target=EmptyWorkspaceTarget(),
heartbeat_interval_seconds=heartbeat_interval_seconds,
) as controller:
while True:
now = pendulum.now("UTC")
status = get_daemon_statuses(
instance,
[SchedulerDaemon.daemon_type()],
now.float_timestamp,
heartbeat_interval_seconds=heartbeat_interval_seconds,
)[SchedulerDaemon.daemon_type()]
if iteration_ran["ran"] and status.healthy:
try:
controller.check_daemon_threads(
) # Should eventually throw since the sensor thread is interrupted
except Exception as e:
assert (
"Stopping dagster-daemon process since the following threads are no longer running: ['SENSOR']"
in str(e))
break
if (now - init_time).total_seconds() > 20:
raise Exception(
"timed out waiting for check_daemons to fail")
time.sleep(0.5)
def required_daemons(instance):
"""
Return which daemon types are required by the instance
"""
daemons = [SensorDaemon.daemon_type()]
if isinstance(instance.scheduler, DagsterDaemonScheduler):
daemons.append(SchedulerDaemon.daemon_type())
if isinstance(instance.run_coordinator, QueuedRunCoordinator):
daemons.append(QueuedRunCoordinatorDaemon.daemon_type())
return daemons
def create_daemon_of_type(daemon_type):
if daemon_type == SchedulerDaemon.daemon_type():
return SchedulerDaemon.create_from_instance(DagsterInstance.get())
elif daemon_type == SensorDaemon.daemon_type():
return SensorDaemon.create_from_instance(DagsterInstance.get())
elif daemon_type == QueuedRunCoordinatorDaemon.daemon_type():
return QueuedRunCoordinatorDaemon.create_from_instance(
DagsterInstance.get())
else:
raise Exception("Unexpected daemon type {daemon_type}".format(
daemon_type=daemon_type))
def create_daemon_of_type(daemon_type, instance):
if daemon_type == SchedulerDaemon.daemon_type():
return SchedulerDaemon(
interval_seconds=DEFAULT_DAEMON_INTERVAL_SECONDS)
elif daemon_type == SensorDaemon.daemon_type():
return SensorDaemon(interval_seconds=DEFAULT_SENSOR_DAEMON_INTERVAL)
elif daemon_type == QueuedRunCoordinatorDaemon.daemon_type():
return QueuedRunCoordinatorDaemon(
interval_seconds=instance.run_coordinator.dequeue_interval_seconds)
elif daemon_type == BackfillDaemon.daemon_type():
return BackfillDaemon(interval_seconds=DEFAULT_DAEMON_INTERVAL_SECONDS)
else:
raise Exception(f"Unexpected daemon type {daemon_type}")
def create_daemons_from_instance(instance):
daemon_types = required_daemons(instance)
daemons = []
# Separate instance for each daemon since each is in its own thread
for daemon_type in daemon_types:
if daemon_type == SchedulerDaemon.daemon_type():
daemons.append(
SchedulerDaemon.create_from_instance(DagsterInstance.get()))
elif daemon_type == SensorDaemon.daemon_type():
daemons.append(
SensorDaemon.create_from_instance(DagsterInstance.get()))
elif daemon_type == QueuedRunCoordinatorDaemon.daemon_type():
daemons.append(
QueuedRunCoordinatorDaemon.create_from_instance(
DagsterInstance.get()))
else:
raise Exception("Unexpected daemon type {daemon_type}".format(
daemon_type=daemon_type))
return daemons
def test_thread_die_daemon(monkeypatch):
with instance_for_test(overrides={}) as instance:
from dagster.daemon.daemon import SchedulerDaemon, SensorDaemon
iteration_ran = {"ran": False}
def run_iteration_error(_, _instance, _daemon_shutdown_event,
_grpc_server_registry):
iteration_ran["ran"] = True
raise KeyboardInterrupt
yield # pylint: disable=unreachable
monkeypatch.setattr(SensorDaemon, "run_iteration", run_iteration_error)
init_time = pendulum.now("UTC")
with daemon_controller_from_instance(
instance, wait_for_processes_on_exit=True) as controller:
while True:
now = pendulum.now("UTC")
status = get_daemon_status(instance,
SchedulerDaemon.daemon_type(),
now.float_timestamp)
if iteration_ran["ran"] and status.healthy:
try:
controller.check_daemons(
) # Should eventually throw since the sensor thread is interrupted
except Exception as e: # pylint: disable=broad-except
assert (
"Stopping dagster-daemon process since the following threads are no longer sending heartbeats: ['SENSOR']"
in str(e))
break
if (now - init_time).total_seconds() > 20:
raise Exception(
"timed out waiting for check_daemons to fail")
time.sleep(0.5)
