def __init__(self, instance):
self._instance = instance
self._daemons = {}
self._logger = get_default_daemon_logger("dagster-daemon")
if isinstance(instance.scheduler, DagsterDaemonScheduler):
max_catchup_runs = instance.scheduler.max_catchup_runs
self._add_daemon(
SchedulerDaemon(instance,
interval_seconds=30,
max_catchup_runs=max_catchup_runs))
if isinstance(instance.run_coordinator, QueuedRunCoordinator):
max_concurrent_runs = instance.run_coordinator.max_concurrent_runs
dequeue_interval_seconds = instance.run_coordinator.dequeue_interval_seconds
self._add_daemon(
QueuedRunCoordinatorDaemon(
instance,
interval_seconds=dequeue_interval_seconds,
max_concurrent_runs=max_concurrent_runs,
))
if not self._daemons:
raise Exception("No daemons configured on the DagsterInstance")
self._logger.info(
"instance is configured with the following daemons: {}".format(
_sorted_quoted(
type(daemon).__name__ for daemon in self.daemons)))
def __init__(self, instance, daemons):
self._daemon_uuid = str(uuid.uuid4())
self._daemons = {}
self._daemon_threads = {}
self._instance = check.inst_param(instance, "instance", DagsterInstance)
self._daemons = {
daemon.daemon_type(): daemon
for daemon in check.list_param(daemons, "daemons", of_type=DagsterDaemon)
}
if not self._daemons:
raise Exception("No daemons configured on the DagsterInstance")
self._daemon_shutdown_event = threading.Event()
self._logger = get_default_daemon_logger("dagster-daemon")
self._logger.info(
"instance is configured with the following daemons: {}".format(
_sorted_quoted(type(daemon).__name__ for daemon in self.daemons)
)
)
for daemon_type, daemon in self._daemons.items():
self._daemon_threads[daemon_type] = threading.Thread(
target=daemon.run_loop,
args=(self._daemon_uuid, self._daemon_shutdown_event),
name="dagster-daemon-{daemon_type}".format(daemon_type=daemon_type),
)
self._daemon_threads[daemon_type].start()
self._start_time = pendulum.now("UTC")
def __init__(self, instance):
self._instance = instance
self._daemon_uuid = str(uuid.uuid4())
self._daemons = {}
self._last_heartbeat_times = {}
self._last_iteration_times = {}
self._last_iteration_exceptions = {}
self._current_iteration_exceptions = {}
self._logger = get_default_daemon_logger("dagster-daemon")
if isinstance(instance.scheduler, DagsterDaemonScheduler):
max_catchup_runs = instance.scheduler.max_catchup_runs
self._add_daemon(
SchedulerDaemon(
instance,
interval_seconds=DEFAULT_DAEMON_INTERVAL_SECONDS,
max_catchup_runs=max_catchup_runs,
)
)
self._add_daemon(SensorDaemon(instance, interval_seconds=SENSOR_DAEMON_INTERVAL,))
if isinstance(instance.run_coordinator, QueuedRunCoordinator):
max_concurrent_runs = instance.run_coordinator.max_concurrent_runs
tag_concurrency_limits = instance.run_coordinator.tag_concurrency_limits
self._add_daemon(
QueuedRunCoordinatorDaemon(
instance,
interval_seconds=instance.run_coordinator.dequeue_interval_seconds,
max_concurrent_runs=max_concurrent_runs,
tag_concurrency_limits=tag_concurrency_limits,
)
)
assert set(required_daemons(instance)) == self._daemons.keys()
if not self._daemons:
raise Exception("No daemons configured on the DagsterInstance")
self._logger.info(
"instance is configured with the following daemons: {}".format(
_sorted_quoted(type(daemon).__name__ for daemon in self.daemons)
)
)
def __init__(self, instance):
self._instance = instance
self._daemon_uuid = str(uuid.uuid4())
self._daemons = {}
self._last_heartbeat_time = None
self._logger = get_default_daemon_logger("dagster-daemon")
if isinstance(instance.scheduler, DagsterDaemonScheduler):
max_catchup_runs = instance.scheduler.max_catchup_runs
self._add_daemon(
SchedulerDaemon(
instance,
interval_seconds=self._get_interval_seconds(
instance, SchedulerDaemon.__name__),
max_catchup_runs=max_catchup_runs,
))
self._add_daemon(
SensorDaemon(
instance,
interval_seconds=self._get_interval_seconds(
instance, SensorDaemon.__name__),
))
if isinstance(instance.run_coordinator, QueuedRunCoordinator):
max_concurrent_runs = instance.run_coordinator.max_concurrent_runs
self._add_daemon(
QueuedRunCoordinatorDaemon(
instance,
interval_seconds=self._get_interval_seconds(
instance, QueuedRunCoordinatorDaemon.__name__),
max_concurrent_runs=max_concurrent_runs,
))
assert set(self._expected_daemons(instance)) == self._daemons.keys()
if not self._daemons:
raise Exception("No daemons configured on the DagsterInstance")
self._logger.info(
"instance is configured with the following daemons: {}".format(
_sorted_quoted(
type(daemon).__name__ for daemon in self.daemons)))
def __init__(self, instance, daemons, grpc_server_registry):
self._daemon_uuid = str(uuid.uuid4())
self._daemons = {}
self._daemon_threads = {}
self._instance = check.inst_param(instance, "instance",
DagsterInstance)
self._daemons = {
daemon.daemon_type(): daemon
for daemon in check.list_param(
daemons, "daemons", of_type=DagsterDaemon)
}
self._grpc_server_registry = check.inst_param(grpc_server_registry,
"grpc_server_registry",
GrpcServerRegistry)
if not self._daemons:
raise Exception("No daemons configured on the DagsterInstance")
self._daemon_shutdown_event = threading.Event()
self._logger = get_default_daemon_logger("dagster-daemon")
self._logger.info(
"instance is configured with the following daemons: {}".format(
_sorted_quoted(
type(daemon).__name__ for daemon in self.daemons)))
for daemon_type, daemon in self._daemons.items():
self._daemon_threads[daemon_type] = threading.Thread(
target=daemon.run_loop,
args=(self._daemon_uuid, self._daemon_shutdown_event,
self._grpc_server_registry),
name="dagster-daemon-{daemon_type}".format(
daemon_type=daemon_type),
daemon=
True, # Individual daemons should not outlive controller process
)
self._daemon_threads[daemon_type].start()
self._start_time = pendulum.now("UTC")
def __init__(
self,
instance,
daemons,
gen_workspace,
heartbeat_interval_seconds=DEFAULT_HEARTBEAT_INTERVAL_SECONDS,
heartbeat_tolerance_seconds=DEFAULT_DAEMON_HEARTBEAT_TOLERANCE_SECONDS,
error_interval_seconds=DEFAULT_DAEMON_ERROR_INTERVAL_SECONDS,
):
self._daemon_uuid = str(uuid.uuid4())
self._daemons = {}
self._daemon_threads = {}
self._instance = check.inst_param(instance, "instance",
DagsterInstance)
self._daemons = {
daemon.daemon_type(): daemon
for daemon in check.list_param(
daemons, "daemons", of_type=DagsterDaemon)
}
self._gen_workspace = check.callable_param(gen_workspace,
"gen_workspace")
self._heartbeat_interval_seconds = check.numeric_param(
heartbeat_interval_seconds, "heartbeat_interval_seconds")
self._heartbeat_tolerance_seconds = check.numeric_param(
heartbeat_tolerance_seconds, "heartbeat_tolerance_seconds")
if not self._daemons:
raise Exception("No daemons configured on the DagsterInstance")
self._daemon_shutdown_event = threading.Event()
self._logger = get_default_daemon_logger("dagster-daemon")
self._logger.info(
"instance is configured with the following daemons: {}".format(
_sorted_quoted(
type(daemon).__name__ for daemon in self.daemons)))
self._last_healthy_heartbeat_times = {}
for daemon_type, daemon in self._daemons.items():
self._daemon_threads[daemon_type] = threading.Thread(
target=daemon.run_loop,
args=(
self._instance.get_ref(),
self._daemon_uuid,
self._daemon_shutdown_event,
gen_workspace,
heartbeat_interval_seconds,
error_interval_seconds,
),
name="dagster-daemon-{daemon_type}".format(
daemon_type=daemon_type),
daemon=
True, # Individual daemons should not outlive controller process
)
self._last_healthy_heartbeat_times[daemon_type] = time.time()
self._daemon_threads[daemon_type].start()
self._start_time = pendulum.now("UTC")