def test_get_required_daemon_types(): from dagster.daemon.daemon import ( SensorDaemon, BackfillDaemon, SchedulerDaemon, MonitoringDaemon, ) with instance_for_test() as instance: assert instance.get_required_daemon_types() == [ SensorDaemon.daemon_type(), BackfillDaemon.daemon_type(), SchedulerDaemon.daemon_type(), ] with instance_for_test( overrides={ "run_launcher": { "module": "dagster_tests.daemon_tests.test_monitoring_daemon", "class": "TestRunLauncher", }, "run_monitoring": {"enabled": True}, } ) as instance: assert instance.get_required_daemon_types() == [ SensorDaemon.daemon_type(), BackfillDaemon.daemon_type(), SchedulerDaemon.daemon_type(), MonitoringDaemon.daemon_type(), ]
def create_daemon_of_type(daemon_type): if daemon_type == SchedulerDaemon.daemon_type(): return SchedulerDaemon.create_from_instance(DagsterInstance.get()) elif daemon_type == SensorDaemon.daemon_type(): return SensorDaemon.create_from_instance(DagsterInstance.get()) elif daemon_type == QueuedRunCoordinatorDaemon.daemon_type(): return QueuedRunCoordinatorDaemon.create_from_instance( DagsterInstance.get()) else: raise Exception("Unexpected daemon type {daemon_type}".format( daemon_type=daemon_type))
def create_daemon_of_type(daemon_type, instance): if daemon_type == SchedulerDaemon.daemon_type(): return SchedulerDaemon( interval_seconds=DEFAULT_DAEMON_INTERVAL_SECONDS) elif daemon_type == SensorDaemon.daemon_type(): return SensorDaemon(interval_seconds=DEFAULT_SENSOR_DAEMON_INTERVAL) elif daemon_type == QueuedRunCoordinatorDaemon.daemon_type(): return QueuedRunCoordinatorDaemon( interval_seconds=instance.run_coordinator.dequeue_interval_seconds) elif daemon_type == BackfillDaemon.daemon_type(): return BackfillDaemon(interval_seconds=DEFAULT_DAEMON_INTERVAL_SECONDS) else: raise Exception(f"Unexpected daemon type {daemon_type}")
def __init__(self, instance): self._instance = instance self._daemons = {} self._logger = get_default_daemon_logger("dagster-daemon") if isinstance(instance.scheduler, DagsterDaemonScheduler): max_catchup_runs = instance.scheduler.max_catchup_runs self._add_daemon( SchedulerDaemon(instance, interval_seconds=30, max_catchup_runs=max_catchup_runs)) if isinstance(instance.run_coordinator, QueuedRunCoordinator): max_concurrent_runs = instance.run_coordinator.max_concurrent_runs dequeue_interval_seconds = instance.run_coordinator.dequeue_interval_seconds self._add_daemon( QueuedRunCoordinatorDaemon( instance, interval_seconds=dequeue_interval_seconds, max_concurrent_runs=max_concurrent_runs, )) if not self._daemons: raise Exception("No daemons configured on the DagsterInstance") self._logger.info( "instance is configured with the following daemons: {}".format( _sorted_quoted( type(daemon).__name__ for daemon in self.daemons)))
def test_thread_die_daemon(monkeypatch): with instance_for_test(overrides={}) as instance: from dagster.daemon.daemon import SchedulerDaemon, SensorDaemon iteration_ran = {"ran": False} def run_loop_error(_, _instance, _workspace): iteration_ran["ran"] = True raise KeyboardInterrupt yield # pylint: disable=unreachable monkeypatch.setattr(SensorDaemon, "core_loop", run_loop_error) heartbeat_interval_seconds = 1 init_time = pendulum.now("UTC") with daemon_controller_from_instance( instance, workspace_load_target=EmptyWorkspaceTarget(), heartbeat_interval_seconds=heartbeat_interval_seconds, ) as controller: while True: now = pendulum.now("UTC") status = get_daemon_statuses( instance, [SchedulerDaemon.daemon_type()], now.float_timestamp, heartbeat_interval_seconds=heartbeat_interval_seconds, )[SchedulerDaemon.daemon_type()] if iteration_ran["ran"] and status.healthy: try: controller.check_daemon_threads( ) # Should eventually throw since the sensor thread is interrupted except Exception as e: assert ( "Stopping dagster-daemon process since the following threads are no longer running: ['SENSOR']" in str(e)) break if (now - init_time).total_seconds() > 20: raise Exception( "timed out waiting for check_daemons to fail") time.sleep(0.5)
def required_daemons(instance): """ Return which daemon types are required by the instance """ daemons = [SensorDaemon.daemon_type()] if isinstance(instance.scheduler, DagsterDaemonScheduler): daemons.append(SchedulerDaemon.daemon_type()) if isinstance(instance.run_coordinator, QueuedRunCoordinator): daemons.append(QueuedRunCoordinatorDaemon.daemon_type()) return daemons
def create_daemons_from_instance(instance): daemon_types = required_daemons(instance) daemons = [] # Separate instance for each daemon since each is in its own thread for daemon_type in daemon_types: if daemon_type == SchedulerDaemon.daemon_type(): daemons.append( SchedulerDaemon.create_from_instance(DagsterInstance.get())) elif daemon_type == SensorDaemon.daemon_type(): daemons.append( SensorDaemon.create_from_instance(DagsterInstance.get())) elif daemon_type == QueuedRunCoordinatorDaemon.daemon_type(): daemons.append( QueuedRunCoordinatorDaemon.create_from_instance( DagsterInstance.get())) else: raise Exception("Unexpected daemon type {daemon_type}".format( daemon_type=daemon_type)) return daemons
def __init__(self, instance): self._instance = instance self._daemon_uuid = str(uuid.uuid4()) self._daemons = {} self._last_heartbeat_times = {} self._last_iteration_times = {} self._last_iteration_exceptions = {} self._current_iteration_exceptions = {} self._logger = get_default_daemon_logger("dagster-daemon") if isinstance(instance.scheduler, DagsterDaemonScheduler): max_catchup_runs = instance.scheduler.max_catchup_runs self._add_daemon( SchedulerDaemon( instance, interval_seconds=DEFAULT_DAEMON_INTERVAL_SECONDS, max_catchup_runs=max_catchup_runs, ) ) self._add_daemon(SensorDaemon(instance, interval_seconds=SENSOR_DAEMON_INTERVAL,)) if isinstance(instance.run_coordinator, QueuedRunCoordinator): max_concurrent_runs = instance.run_coordinator.max_concurrent_runs tag_concurrency_limits = instance.run_coordinator.tag_concurrency_limits self._add_daemon( QueuedRunCoordinatorDaemon( instance, interval_seconds=instance.run_coordinator.dequeue_interval_seconds, max_concurrent_runs=max_concurrent_runs, tag_concurrency_limits=tag_concurrency_limits, ) ) assert set(required_daemons(instance)) == self._daemons.keys() if not self._daemons: raise Exception("No daemons configured on the DagsterInstance") self._logger.info( "instance is configured with the following daemons: {}".format( _sorted_quoted(type(daemon).__name__ for daemon in self.daemons) ) )
def __init__(self, instance): self._instance = instance self._daemon_uuid = str(uuid.uuid4()) self._daemons = {} self._last_heartbeat_time = None self._logger = get_default_daemon_logger("dagster-daemon") if isinstance(instance.scheduler, DagsterDaemonScheduler): max_catchup_runs = instance.scheduler.max_catchup_runs self._add_daemon( SchedulerDaemon( instance, interval_seconds=self._get_interval_seconds( instance, SchedulerDaemon.__name__), max_catchup_runs=max_catchup_runs, )) self._add_daemon( SensorDaemon( instance, interval_seconds=self._get_interval_seconds( instance, SensorDaemon.__name__), )) if isinstance(instance.run_coordinator, QueuedRunCoordinator): max_concurrent_runs = instance.run_coordinator.max_concurrent_runs self._add_daemon( QueuedRunCoordinatorDaemon( instance, interval_seconds=self._get_interval_seconds( instance, QueuedRunCoordinatorDaemon.__name__), max_concurrent_runs=max_concurrent_runs, )) assert set(self._expected_daemons(instance)) == self._daemons.keys() if not self._daemons: raise Exception("No daemons configured on the DagsterInstance") self._logger.info( "instance is configured with the following daemons: {}".format( _sorted_quoted( type(daemon).__name__ for daemon in self.daemons)))
def test_thread_die_daemon(monkeypatch): with instance_for_test(overrides={}) as instance: from dagster.daemon.daemon import SchedulerDaemon, SensorDaemon iteration_ran = {"ran": False} def run_iteration_error(_, _instance, _daemon_shutdown_event, _grpc_server_registry): iteration_ran["ran"] = True raise KeyboardInterrupt yield # pylint: disable=unreachable monkeypatch.setattr(SensorDaemon, "run_iteration", run_iteration_error) init_time = pendulum.now("UTC") with daemon_controller_from_instance( instance, wait_for_processes_on_exit=True) as controller: while True: now = pendulum.now("UTC") status = get_daemon_status(instance, SchedulerDaemon.daemon_type(), now.float_timestamp) if iteration_ran["ran"] and status.healthy: try: controller.check_daemons( ) # Should eventually throw since the sensor thread is interrupted except Exception as e: # pylint: disable=broad-except assert ( "Stopping dagster-daemon process since the following threads are no longer sending heartbeats: ['SENSOR']" in str(e)) break if (now - init_time).total_seconds() > 20: raise Exception( "timed out waiting for check_daemons to fail") time.sleep(0.5)