async def spawn_resource_daemons(
*,
settings: configuration.OperatorSettings,
handlers: Sequence[handlers_.ResourceSpawningHandler],
daemons: MutableMapping[containers.DaemonId, containers.Daemon],
cause: causation.ResourceSpawningCause,
memory: containers.ResourceMemory,
) -> Collection[float]:
"""
Ensure that all daemons are spawned for this individual resource.
This function can be called multiple times on multiple handling cycles
(though usually should be called on the first-seen occasion), so it must
be idempotent: not having duplicating side-effects on multiple calls.
"""
if memory.live_fresh_body is None: # for type-checking; "not None" is ensured in processing.
raise RuntimeError(
"A daemon is spawned with None as body. This is a bug. Please report."
)
for handler in handlers:
daemon_id = containers.DaemonId(handler.id)
if daemon_id not in daemons:
stopper = primitives.DaemonStopper()
daemon_cause = causation.DaemonCause(
resource=cause.resource,
logger=cause.logger,
body=memory.live_fresh_body,
memo=memory.user_data,
patch=patches.Patch(
), # not the same as the one-shot spawning patch!
stopper=stopper, # for checking (passed to kwargs)
)
daemon = containers.Daemon(
stopper=stopper, # for stopping (outside of causes)
handler=handler,
logger=logging_engine.LocalObjectLogger(body=cause.body,
settings=settings),
task=asyncio.create_task(
_runner(
settings=settings,
handler=handler,
cause=daemon_cause,
memory=memory,
)),
)
daemons[daemon_id] = daemon
return []
async def match_resource_daemons(
*,
settings: configuration.OperatorSettings,
handlers: Sequence[handlers_.ResourceSpawningHandler],
daemons: MutableMapping[containers.DaemonId, containers.Daemon],
) -> Collection[float]:
"""
Re-match the running daemons with the filters, and stop those mismatching.
Stopping can take few iterations, same as `stop_resource_daemons` would do.
"""
matching_daemon_ids = {containers.DaemonId(handler.id) for handler in handlers}
mismatching_daemons = {
daemon_id: daemon
for daemon_id, daemon in daemons.items()
if daemon_id not in matching_daemon_ids
}
delays = await stop_resource_daemons(
settings=settings,
daemons=mismatching_daemons,
reason=primitives.DaemonStoppingReason.FILTERS_MISMATCH,
)
return delays
async def _runner(
*,
settings: configuration.OperatorSettings,
daemons: MutableMapping[containers.DaemonId, containers.Daemon],
handler: handlers_.ResourceSpawningHandler,
memory: containers.ResourceMemory,
cause: causation.DaemonCause,
) -> None:
"""
Guard a running daemon during its life cycle.
Note: synchronous daemons are awaited to the exit and postpone cancellation.
The runner will not exit until the thread exits. See `invoke` for details.
"""
try:
if isinstance(handler, handlers_.ResourceDaemonHandler):
await _resource_daemon(settings=settings, handler=handler, cause=cause)
elif isinstance(handler, handlers_.ResourceTimerHandler):
await _resource_timer(settings=settings, handler=handler, cause=cause, memory=memory)
else:
raise RuntimeError("Cannot determine which task wrapper to use. This is a bug.")
finally:
# Prevent future re-spawns for those exited on their own, for no reason.
# Only the filter-mismatching daemons can be re-spawned on future events.
if cause.stopper.reason == primitives.DaemonStoppingReason.NONE:
memory.forever_stopped.add(handler.id)
# Save the memory by not remembering the exited daemons (they may be never re-spawned).
del daemons[containers.DaemonId(handler.id)]
# Whatever happened, make sure the sync threads of asyncio threaded executor are notified:
# in a hope that they will exit maybe some time later to free the OS/asyncio resources.
# A possible case: operator is exiting and cancelling all "hung" non-root tasks, etc.
cause.stopper.set(reason=primitives.DaemonStoppingReason.DONE)