def test_get_required_daemon_types():
from dagster.daemon.daemon import (
SensorDaemon,
BackfillDaemon,
SchedulerDaemon,
MonitoringDaemon,
)
with instance_for_test() as instance:
assert instance.get_required_daemon_types() == [
SensorDaemon.daemon_type(),
BackfillDaemon.daemon_type(),
SchedulerDaemon.daemon_type(),
]
with instance_for_test(
overrides={
"run_launcher": {
"module": "dagster_tests.daemon_tests.test_monitoring_daemon",
"class": "TestRunLauncher",
},
"run_monitoring": {"enabled": True},
}
) as instance:
assert instance.get_required_daemon_types() == [
SensorDaemon.daemon_type(),
BackfillDaemon.daemon_type(),
SchedulerDaemon.daemon_type(),
MonitoringDaemon.daemon_type(),
]
def create_daemon_of_type(daemon_type):
if daemon_type == SchedulerDaemon.daemon_type():
return SchedulerDaemon.create_from_instance(DagsterInstance.get())
elif daemon_type == SensorDaemon.daemon_type():
return SensorDaemon.create_from_instance(DagsterInstance.get())
elif daemon_type == QueuedRunCoordinatorDaemon.daemon_type():
return QueuedRunCoordinatorDaemon.create_from_instance(
DagsterInstance.get())
else:
raise Exception("Unexpected daemon type {daemon_type}".format(
daemon_type=daemon_type))
def create_daemon_of_type(daemon_type, instance):
if daemon_type == SchedulerDaemon.daemon_type():
return SchedulerDaemon(
interval_seconds=DEFAULT_DAEMON_INTERVAL_SECONDS)
elif daemon_type == SensorDaemon.daemon_type():
return SensorDaemon(interval_seconds=DEFAULT_SENSOR_DAEMON_INTERVAL)
elif daemon_type == QueuedRunCoordinatorDaemon.daemon_type():
return QueuedRunCoordinatorDaemon(
interval_seconds=instance.run_coordinator.dequeue_interval_seconds)
elif daemon_type == BackfillDaemon.daemon_type():
return BackfillDaemon(interval_seconds=DEFAULT_DAEMON_INTERVAL_SECONDS)
else:
raise Exception(f"Unexpected daemon type {daemon_type}")
def __init__(self, instance):
self._instance = instance
self._daemons = {}
self._logger = get_default_daemon_logger("dagster-daemon")
if isinstance(instance.scheduler, DagsterDaemonScheduler):
max_catchup_runs = instance.scheduler.max_catchup_runs
self._add_daemon(
SchedulerDaemon(instance,
interval_seconds=30,
max_catchup_runs=max_catchup_runs))
if isinstance(instance.run_coordinator, QueuedRunCoordinator):
max_concurrent_runs = instance.run_coordinator.max_concurrent_runs
dequeue_interval_seconds = instance.run_coordinator.dequeue_interval_seconds
self._add_daemon(
QueuedRunCoordinatorDaemon(
instance,
interval_seconds=dequeue_interval_seconds,
max_concurrent_runs=max_concurrent_runs,
))
if not self._daemons:
raise Exception("No daemons configured on the DagsterInstance")
self._logger.info(
"instance is configured with the following daemons: {}".format(
_sorted_quoted(
type(daemon).__name__ for daemon in self.daemons)))
def test_thread_die_daemon(monkeypatch):
with instance_for_test(overrides={}) as instance:
from dagster.daemon.daemon import SchedulerDaemon, SensorDaemon
iteration_ran = {"ran": False}
def run_loop_error(_, _instance, _workspace):
iteration_ran["ran"] = True
raise KeyboardInterrupt
yield # pylint: disable=unreachable
monkeypatch.setattr(SensorDaemon, "core_loop", run_loop_error)
heartbeat_interval_seconds = 1
init_time = pendulum.now("UTC")
with daemon_controller_from_instance(
instance,
workspace_load_target=EmptyWorkspaceTarget(),
heartbeat_interval_seconds=heartbeat_interval_seconds,
) as controller:
while True:
now = pendulum.now("UTC")
status = get_daemon_statuses(
instance,
[SchedulerDaemon.daemon_type()],
now.float_timestamp,
heartbeat_interval_seconds=heartbeat_interval_seconds,
)[SchedulerDaemon.daemon_type()]
if iteration_ran["ran"] and status.healthy:
try:
controller.check_daemon_threads(
) # Should eventually throw since the sensor thread is interrupted
except Exception as e:
assert (
"Stopping dagster-daemon process since the following threads are no longer running: ['SENSOR']"
in str(e))
break
if (now - init_time).total_seconds() > 20:
raise Exception(
"timed out waiting for check_daemons to fail")
time.sleep(0.5)
def required_daemons(instance):
"""
Return which daemon types are required by the instance
"""
daemons = [SensorDaemon.daemon_type()]
if isinstance(instance.scheduler, DagsterDaemonScheduler):
daemons.append(SchedulerDaemon.daemon_type())
if isinstance(instance.run_coordinator, QueuedRunCoordinator):
daemons.append(QueuedRunCoordinatorDaemon.daemon_type())
return daemons
def create_daemons_from_instance(instance):
daemon_types = required_daemons(instance)
daemons = []
# Separate instance for each daemon since each is in its own thread
for daemon_type in daemon_types:
if daemon_type == SchedulerDaemon.daemon_type():
daemons.append(
SchedulerDaemon.create_from_instance(DagsterInstance.get()))
elif daemon_type == SensorDaemon.daemon_type():
daemons.append(
SensorDaemon.create_from_instance(DagsterInstance.get()))
elif daemon_type == QueuedRunCoordinatorDaemon.daemon_type():
daemons.append(
QueuedRunCoordinatorDaemon.create_from_instance(
DagsterInstance.get()))
else:
raise Exception("Unexpected daemon type {daemon_type}".format(
daemon_type=daemon_type))
return daemons
def __init__(self, instance):
self._instance = instance
self._daemon_uuid = str(uuid.uuid4())
self._daemons = {}
self._last_heartbeat_times = {}
self._last_iteration_times = {}
self._last_iteration_exceptions = {}
self._current_iteration_exceptions = {}
self._logger = get_default_daemon_logger("dagster-daemon")
if isinstance(instance.scheduler, DagsterDaemonScheduler):
max_catchup_runs = instance.scheduler.max_catchup_runs
self._add_daemon(
SchedulerDaemon(
instance,
interval_seconds=DEFAULT_DAEMON_INTERVAL_SECONDS,
max_catchup_runs=max_catchup_runs,
)
)
self._add_daemon(SensorDaemon(instance, interval_seconds=SENSOR_DAEMON_INTERVAL,))
if isinstance(instance.run_coordinator, QueuedRunCoordinator):
max_concurrent_runs = instance.run_coordinator.max_concurrent_runs
tag_concurrency_limits = instance.run_coordinator.tag_concurrency_limits
self._add_daemon(
QueuedRunCoordinatorDaemon(
instance,
interval_seconds=instance.run_coordinator.dequeue_interval_seconds,
max_concurrent_runs=max_concurrent_runs,
tag_concurrency_limits=tag_concurrency_limits,
)
)
assert set(required_daemons(instance)) == self._daemons.keys()
if not self._daemons:
raise Exception("No daemons configured on the DagsterInstance")
self._logger.info(
"instance is configured with the following daemons: {}".format(
_sorted_quoted(type(daemon).__name__ for daemon in self.daemons)
)
)
def __init__(self, instance):
self._instance = instance
self._daemon_uuid = str(uuid.uuid4())
self._daemons = {}
self._last_heartbeat_time = None
self._logger = get_default_daemon_logger("dagster-daemon")
if isinstance(instance.scheduler, DagsterDaemonScheduler):
max_catchup_runs = instance.scheduler.max_catchup_runs
self._add_daemon(
SchedulerDaemon(
instance,
interval_seconds=self._get_interval_seconds(
instance, SchedulerDaemon.__name__),
max_catchup_runs=max_catchup_runs,
))
self._add_daemon(
SensorDaemon(
instance,
interval_seconds=self._get_interval_seconds(
instance, SensorDaemon.__name__),
))
if isinstance(instance.run_coordinator, QueuedRunCoordinator):
max_concurrent_runs = instance.run_coordinator.max_concurrent_runs
self._add_daemon(
QueuedRunCoordinatorDaemon(
instance,
interval_seconds=self._get_interval_seconds(
instance, QueuedRunCoordinatorDaemon.__name__),
max_concurrent_runs=max_concurrent_runs,
))
assert set(self._expected_daemons(instance)) == self._daemons.keys()
if not self._daemons:
raise Exception("No daemons configured on the DagsterInstance")
self._logger.info(
"instance is configured with the following daemons: {}".format(
_sorted_quoted(
type(daemon).__name__ for daemon in self.daemons)))
def test_thread_die_daemon(monkeypatch):
with instance_for_test(overrides={}) as instance:
from dagster.daemon.daemon import SchedulerDaemon, SensorDaemon
iteration_ran = {"ran": False}
def run_iteration_error(_, _instance, _daemon_shutdown_event,
_grpc_server_registry):
iteration_ran["ran"] = True
raise KeyboardInterrupt
yield # pylint: disable=unreachable
monkeypatch.setattr(SensorDaemon, "run_iteration", run_iteration_error)
init_time = pendulum.now("UTC")
with daemon_controller_from_instance(
instance, wait_for_processes_on_exit=True) as controller:
while True:
now = pendulum.now("UTC")
status = get_daemon_status(instance,
SchedulerDaemon.daemon_type(),
now.float_timestamp)
if iteration_ran["ran"] and status.healthy:
try:
controller.check_daemons(
) # Should eventually throw since the sensor thread is interrupted
except Exception as e: # pylint: disable=broad-except
assert (
"Stopping dagster-daemon process since the following threads are no longer sending heartbeats: ['SENSOR']"
in str(e))
break
if (now - init_time).total_seconds() > 20:
raise Exception(
"timed out waiting for check_daemons to fail")
time.sleep(0.5)
