def test_get_required_daemon_types(): from dagster.daemon.daemon import ( SensorDaemon, BackfillDaemon, SchedulerDaemon, MonitoringDaemon, ) with instance_for_test() as instance: assert instance.get_required_daemon_types() == [ SensorDaemon.daemon_type(), BackfillDaemon.daemon_type(), SchedulerDaemon.daemon_type(), ] with instance_for_test( overrides={ "run_launcher": { "module": "dagster_tests.daemon_tests.test_monitoring_daemon", "class": "TestRunLauncher", }, "run_monitoring": {"enabled": True}, } ) as instance: assert instance.get_required_daemon_types() == [ SensorDaemon.daemon_type(), BackfillDaemon.daemon_type(), SchedulerDaemon.daemon_type(), MonitoringDaemon.daemon_type(), ]
def test_thread_die_daemon(monkeypatch): with instance_for_test(overrides={}) as instance: from dagster.daemon.daemon import SchedulerDaemon, SensorDaemon iteration_ran = {"ran": False} def run_loop_error(_, _instance, _workspace): iteration_ran["ran"] = True raise KeyboardInterrupt yield # pylint: disable=unreachable monkeypatch.setattr(SensorDaemon, "core_loop", run_loop_error) heartbeat_interval_seconds = 1 init_time = pendulum.now("UTC") with daemon_controller_from_instance( instance, workspace_load_target=EmptyWorkspaceTarget(), heartbeat_interval_seconds=heartbeat_interval_seconds, ) as controller: while True: now = pendulum.now("UTC") status = get_daemon_statuses( instance, [SchedulerDaemon.daemon_type()], now.float_timestamp, heartbeat_interval_seconds=heartbeat_interval_seconds, )[SchedulerDaemon.daemon_type()] if iteration_ran["ran"] and status.healthy: try: controller.check_daemon_threads( ) # Should eventually throw since the sensor thread is interrupted except Exception as e: assert ( "Stopping dagster-daemon process since the following threads are no longer running: ['SENSOR']" in str(e)) break if (now - init_time).total_seconds() > 20: raise Exception( "timed out waiting for check_daemons to fail") time.sleep(0.5)
def required_daemons(instance): """ Return which daemon types are required by the instance """ daemons = [SensorDaemon.daemon_type()] if isinstance(instance.scheduler, DagsterDaemonScheduler): daemons.append(SchedulerDaemon.daemon_type()) if isinstance(instance.run_coordinator, QueuedRunCoordinator): daemons.append(QueuedRunCoordinatorDaemon.daemon_type()) return daemons
def create_daemon_of_type(daemon_type): if daemon_type == SchedulerDaemon.daemon_type(): return SchedulerDaemon.create_from_instance(DagsterInstance.get()) elif daemon_type == SensorDaemon.daemon_type(): return SensorDaemon.create_from_instance(DagsterInstance.get()) elif daemon_type == QueuedRunCoordinatorDaemon.daemon_type(): return QueuedRunCoordinatorDaemon.create_from_instance( DagsterInstance.get()) else: raise Exception("Unexpected daemon type {daemon_type}".format( daemon_type=daemon_type))
def create_daemon_of_type(daemon_type, instance): if daemon_type == SchedulerDaemon.daemon_type(): return SchedulerDaemon( interval_seconds=DEFAULT_DAEMON_INTERVAL_SECONDS) elif daemon_type == SensorDaemon.daemon_type(): return SensorDaemon(interval_seconds=DEFAULT_SENSOR_DAEMON_INTERVAL) elif daemon_type == QueuedRunCoordinatorDaemon.daemon_type(): return QueuedRunCoordinatorDaemon( interval_seconds=instance.run_coordinator.dequeue_interval_seconds) elif daemon_type == BackfillDaemon.daemon_type(): return BackfillDaemon(interval_seconds=DEFAULT_DAEMON_INTERVAL_SECONDS) else: raise Exception(f"Unexpected daemon type {daemon_type}")
def create_daemons_from_instance(instance): daemon_types = required_daemons(instance) daemons = [] # Separate instance for each daemon since each is in its own thread for daemon_type in daemon_types: if daemon_type == SchedulerDaemon.daemon_type(): daemons.append( SchedulerDaemon.create_from_instance(DagsterInstance.get())) elif daemon_type == SensorDaemon.daemon_type(): daemons.append( SensorDaemon.create_from_instance(DagsterInstance.get())) elif daemon_type == QueuedRunCoordinatorDaemon.daemon_type(): daemons.append( QueuedRunCoordinatorDaemon.create_from_instance( DagsterInstance.get())) else: raise Exception("Unexpected daemon type {daemon_type}".format( daemon_type=daemon_type)) return daemons
def test_thread_die_daemon(monkeypatch): with instance_for_test(overrides={}) as instance: from dagster.daemon.daemon import SchedulerDaemon, SensorDaemon iteration_ran = {"ran": False} def run_iteration_error(_, _instance, _daemon_shutdown_event, _grpc_server_registry): iteration_ran["ran"] = True raise KeyboardInterrupt yield # pylint: disable=unreachable monkeypatch.setattr(SensorDaemon, "run_iteration", run_iteration_error) init_time = pendulum.now("UTC") with daemon_controller_from_instance( instance, wait_for_processes_on_exit=True) as controller: while True: now = pendulum.now("UTC") status = get_daemon_status(instance, SchedulerDaemon.daemon_type(), now.float_timestamp) if iteration_ran["ran"] and status.healthy: try: controller.check_daemons( ) # Should eventually throw since the sensor thread is interrupted except Exception as e: # pylint: disable=broad-except assert ( "Stopping dagster-daemon process since the following threads are no longer sending heartbeats: ['SENSOR']" in str(e)) break if (now - init_time).total_seconds() > 20: raise Exception( "timed out waiting for check_daemons to fail") time.sleep(0.5)