def required_daemons(instance):
"""
Return which daemon types are required by the instance
"""
daemons = [SensorDaemon.daemon_type()]
if isinstance(instance.scheduler, DagsterDaemonScheduler):
daemons.append(SchedulerDaemon.daemon_type())
if isinstance(instance.run_coordinator, QueuedRunCoordinator):
daemons.append(QueuedRunCoordinatorDaemon.daemon_type())
return daemons
def create_daemon_of_type(daemon_type):
if daemon_type == SchedulerDaemon.daemon_type():
return SchedulerDaemon.create_from_instance(DagsterInstance.get())
elif daemon_type == SensorDaemon.daemon_type():
return SensorDaemon.create_from_instance(DagsterInstance.get())
elif daemon_type == QueuedRunCoordinatorDaemon.daemon_type():
return QueuedRunCoordinatorDaemon.create_from_instance(
DagsterInstance.get())
else:
raise Exception("Unexpected daemon type {daemon_type}".format(
daemon_type=daemon_type))
def create_daemon_of_type(daemon_type, instance):
if daemon_type == SchedulerDaemon.daemon_type():
return SchedulerDaemon(
interval_seconds=DEFAULT_DAEMON_INTERVAL_SECONDS)
elif daemon_type == SensorDaemon.daemon_type():
return SensorDaemon(interval_seconds=DEFAULT_SENSOR_DAEMON_INTERVAL)
elif daemon_type == QueuedRunCoordinatorDaemon.daemon_type():
return QueuedRunCoordinatorDaemon(
interval_seconds=instance.run_coordinator.dequeue_interval_seconds)
elif daemon_type == BackfillDaemon.daemon_type():
return BackfillDaemon(interval_seconds=DEFAULT_DAEMON_INTERVAL_SECONDS)
else:
raise Exception(f"Unexpected daemon type {daemon_type}")
def create_daemons_from_instance(instance):
daemon_types = required_daemons(instance)
daemons = []
# Separate instance for each daemon since each is in its own thread
for daemon_type in daemon_types:
if daemon_type == SchedulerDaemon.daemon_type():
daemons.append(
SchedulerDaemon.create_from_instance(DagsterInstance.get()))
elif daemon_type == SensorDaemon.daemon_type():
daemons.append(
SensorDaemon.create_from_instance(DagsterInstance.get()))
elif daemon_type == QueuedRunCoordinatorDaemon.daemon_type():
daemons.append(
QueuedRunCoordinatorDaemon.create_from_instance(
DagsterInstance.get()))
else:
raise Exception("Unexpected daemon type {daemon_type}".format(
daemon_type=daemon_type))
return daemons