async def test_stop_itself_is_cancelled(assert_logs, caplog, cancelled):
logger = logging.getLogger()
caplog.set_level(0)
task1 = create_task(simple())
task2 = create_task(stuck())
stask = create_task(stop([task1, task2], title='sample', logger=logger, interval=0.01, cancelled=cancelled))
with async_timeout.timeout(1):
done, pending = await asyncio.wait({stask}, timeout=0.011)
assert not done
assert task1.done()
assert not task2.done()
stask.cancel()
with async_timeout.timeout(1):
done, pending = await asyncio.wait({stask}, timeout=0.011)
assert done
assert task1.done()
assert not task2.done()
assert_logs([
r"Sample tasks are not stopped: (finishing|cancelling) normally; tasks left: \{<Task",
r"Sample tasks are not stopped: (cancelling|double-cancelling) at stopping; tasks left: \{<Task",
], prohibited=[
r"Sample tasks are stopped",
])
task2.cancel()
with async_timeout.timeout(1):
done, pending = await asyncio.wait({task1, task2})
assert done
assert task1.done()
assert task2.done()
async def test_stop_iteratively(assert_logs, caplog, cancelled):
logger = logging.getLogger()
caplog.set_level(0)
task1 = create_task(simple())
task2 = create_task(stuck())
stask = create_task(
stop([task1, task2],
title='sample',
logger=logger,
interval=0.01,
cancelled=cancelled))
async with async_timeout.timeout(1): # extra test safety
done, pending = await asyncio.wait({stask}, timeout=0.011)
assert not done
assert task1.done()
assert not task2.done()
task2.cancel()
async with async_timeout.timeout(1): # extra test safety
done, pending = await asyncio.wait({stask}, timeout=0.011)
assert done
assert task1.done()
assert task2.done()
assert_logs([
r"Sample tasks are not stopped: (finishing|cancelling) normally; tasks left: \{<Task",
r"Sample tasks are stopped: (finishing|cancelling) normally; tasks left: set\(\)",
])
async def test_stop_immediately_with_cancelling(assert_logs, caplog):
logger = logging.getLogger()
caplog.set_level(0)
task1 = create_task(simple())
task2 = create_task(simple())
with async_timeout.timeout(1):
done, pending = await stop([task1, task2], title='sample', logger=logger, cancelled=True)
assert done
assert not pending
assert_logs(["Sample tasks are stopped: cancelling normally"])
assert task1.cancelled()
assert task2.cancelled()
async def test_alltasks_exclusion():
flag = asyncio.Event()
task1 = create_task(flag.wait())
task2 = create_task(flag.wait())
done, pending = await asyncio.wait([task1, task2], timeout=0.01)
assert not done
tasks = await all_tasks(ignored=[task2])
assert task1 in tasks
assert task2 not in tasks
assert asyncio.current_task() not in tasks
flag.set()
await task1
await task2
async def _stop_flag_checker(
signal_flag: aiotasks.Future,
stop_flag: Optional[primitives.Flag],
) -> None:
"""
A top-level task for external stopping by setting a stop-flag. Once set,
this task will exit, and thus all other top-level tasks will be cancelled.
"""
# Selects the flags to be awaited (if set).
flags = []
if signal_flag is not None:
flags.append(signal_flag)
if stop_flag is not None:
flags.append(aiotasks.create_task(primitives.wait_flag(stop_flag),
name="stop-flag waiter"))
# Wait until one of the stoppers is set/raised.
try:
done, pending = await asyncio.wait(flags, return_when=asyncio.FIRST_COMPLETED)
future = done.pop()
result = await future
except asyncio.CancelledError:
pass # operator is stopping for any other reason
else:
if result is None:
logger.info("Stop-flag is raised. Operator is stopping.")
elif isinstance(result, signal.Signals):
logger.info("Signal %s is received. Operator is stopping.", result.name)
else:
logger.info("Stop-flag is set to %r. Operator is stopping.", result)
async def test_py37_create_task_accepts_name(mocker):
real_create_task = mocker.patch('asyncio.create_task')
coro = sample()
task = create_task(coro, name='unused')
assert real_create_task.called
assert task is real_create_task.return_value
await coro # to prevent "never awaited" errors
async def test_wait_with_timeout():
flag = asyncio.Event()
task = create_task(flag.wait())
done, pending = await wait([task], timeout=0.01)
assert not done
assert pending == {task}
flag.set()
await task
async def test_reraise_skips_cancellations():
task = create_task(asyncio.Event().wait())
done, pending = await asyncio.wait([task], timeout=0.01) # let it start
assert not done
task.cancel()
done, pending = await asyncio.wait([task], timeout=0.01) # let it react
assert done
await reraise([task])
async def spawn_resource_daemons(
*,
settings: configuration.OperatorSettings,
handlers: Sequence[handlers_.ResourceSpawningHandler],
daemons: MutableMapping[handlers_.HandlerId, containers.Daemon],
cause: causation.ResourceSpawningCause,
memory: containers.ResourceMemory,
) -> Collection[float]:
"""
Ensure that all daemons are spawned for this individual resource.
This function can be called multiple times on multiple handling cycles
(though usually should be called on the first-seen occasion), so it must
be idempotent: not having duplicating side-effects on multiple calls.
"""
if memory.live_fresh_body is None: # for type-checking; "not None" is ensured in processing.
raise RuntimeError(
"A daemon is spawned with None as body. This is a bug. Please report."
)
for handler in handlers:
if handler.id not in daemons:
stopper = primitives.DaemonStopper()
daemon_cause = causation.DaemonCause(
resource=cause.resource,
indices=cause.indices,
logger=cause.logger,
body=memory.live_fresh_body,
memo=memory.memo,
patch=patches.Patch(
), # not the same as the one-shot spawning patch!
stopper=stopper, # for checking (passed to kwargs)
)
daemon = containers.Daemon(
stopper=stopper, # for stopping (outside of causes)
handler=handler,
logger=loggers.LocalObjectLogger(body=cause.body,
settings=settings),
task=aiotasks.create_task(
_runner(
settings=settings,
daemons=daemons, # for self-garbage-collection
handler=handler,
cause=daemon_cause,
memory=memory,
),
name=f'runner of {handler.id}'
), # sometimes, daemons; sometimes, timers.
)
daemons[handler.id] = daemon
return []
async def streaming_watch(
*,
settings: configuration.OperatorSettings,
resource: resources.Resource,
namespace: Optional[str],
freeze_mode: Optional[primitives.Toggle] = None,
) -> AsyncIterator[bodies.RawEvent]:
# Prevent both watching and listing while the freeze mode is on, until it is off.
# Specifically, the watch-stream closes its connection once the freeze mode is on,
# so the while-true & for-event-in-stream cycles exit, and this coroutine is started
# again by the `infinite_stream()` (the watcher timeout is swallowed by the freeze time).
if freeze_mode is not None and freeze_mode.is_on():
logger.debug("Freezing the watch-stream for %r", resource)
await freeze_mode.wait_for_off()
logger.debug("Resuming the watch-stream for %r", resource)
# A stop-feature is a client-specific way of terminating the streaming HTTPS connection
# when a freeze-mode is turned on. The low-level API call attaches its `response.close()`
# to the future's callbacks, and a background task triggers it when the mode is turned on.
freeze_waiter: aiotasks.Future
if freeze_mode is not None:
freeze_waiter = aiotasks.create_task(
freeze_mode.wait_for_on(),
name=
f'freeze-waiter for {resource.name} @ {namespace or "cluster-wide"}'
)
else:
freeze_waiter = asyncio.Future() # a dummy just ot have it
try:
stream = continuous_watch(
settings=settings,
resource=resource,
namespace=namespace,
freeze_waiter=freeze_waiter,
)
async for raw_event in stream:
yield raw_event
finally:
with contextlib.suppress(asyncio.CancelledError):
freeze_waiter.cancel()
await freeze_waiter
async def daemon_killer(
*,
settings: configuration.OperatorSettings,
memories: containers.ResourceMemories,
) -> None:
"""
An operator's root task to kill the daemons on the operator's shutdown.
"""
# Sleep forever, or until cancelled, which happens when the operator begins its shutdown.
try:
await asyncio.Event().wait()
# Terminate all running daemons when the operator exits (and this task is cancelled).
finally:
tasks = [
aiotasks.create_task(name=f"stop daemon {daemon.handler.id}",
coro=stop_daemon(daemon=daemon,
settings=settings))
for memory in memories.iter_all_memories()
for daemon in memory.running_daemons.values()
]
await aiotasks.wait(tasks)
async def spawn_tasks(
*,
lifecycle: Optional[lifecycles.LifeCycleFn] = None,
indexers: Optional[indexing.OperatorIndexers] = None,
registry: Optional[registries.OperatorRegistry] = None,
settings: Optional[configuration.OperatorSettings] = None,
memories: Optional[containers.ResourceMemories] = None,
insights: Optional[references.Insights] = None,
identity: Optional[peering.Identity] = None,
standalone: Optional[bool] = None,
priority: Optional[int] = None,
peering_name: Optional[str] = None,
liveness_endpoint: Optional[str] = None,
clusterwide: bool = False,
namespaces: Collection[references.NamespacePattern] = (),
namespace: Optional[references.NamespacePattern] = None, # deprecated
stop_flag: Optional[primitives.Flag] = None,
ready_flag: Optional[primitives.Flag] = None,
vault: Optional[credentials.Vault] = None,
memo: Optional[ephemera.AnyMemo] = None,
_command: Optional[Coroutine[None, None, None]] = None,
) -> Collection[aiotasks.Task]:
"""
Spawn all the tasks needed to run the operator.
The tasks are properly inter-connected with the synchronisation primitives.
"""
loop = asyncio.get_running_loop()
if namespaces and namespace:
raise TypeError("Either namespaces= or namespace= can be passed. Got both.")
elif namespace:
warnings.warn("namespace= is deprecated; use namespaces=[...]", DeprecationWarning)
namespaces = [namespace]
if clusterwide and namespaces:
raise TypeError("The operator can be either cluster-wide or namespaced, not both.")
if not clusterwide and not namespaces:
warnings.warn("Absence of either namespaces or cluster-wide flag will become an error soon."
" For now, switching to the cluster-wide mode for backward compatibility.",
FutureWarning)
clusterwide = True
# All tasks of the operator are synced via these primitives and structures:
lifecycle = lifecycle if lifecycle is not None else lifecycles.get_default_lifecycle()
registry = registry if registry is not None else registries.get_default_registry()
settings = settings if settings is not None else configuration.OperatorSettings()
memories = memories if memories is not None else containers.ResourceMemories()
indexers = indexers if indexers is not None else indexing.OperatorIndexers()
insights = insights if insights is not None else references.Insights()
identity = identity if identity is not None else peering.detect_own_id(manual=False)
vault = vault if vault is not None else credentials.Vault()
memo = memo if memo is not None else ephemera.Memo()
event_queue: posting.K8sEventQueue = asyncio.Queue()
signal_flag: aiotasks.Future = asyncio.Future()
started_flag: asyncio.Event = asyncio.Event()
operator_paused = primitives.ToggleSet(any)
tasks: MutableSequence[aiotasks.Task] = []
# Map kwargs into the settings object.
settings.peering.clusterwide = clusterwide
if peering_name is not None:
settings.peering.mandatory = True
settings.peering.name = peering_name
if standalone is not None:
settings.peering.standalone = standalone
if priority is not None:
settings.peering.priority = priority
# Prepopulate indexers with empty indices -- to be available startup handlers.
indexers.ensure(registry._resource_indexing.get_all_handlers())
# Global credentials store for this operator, also for CRD-reading & peering mode detection.
auth.vault_var.set(vault)
# Special case: pass the settings container through the user-side handlers (no explicit args).
# Toolkits have to keep the original operator context somehow, and the only way is contextvars.
posting.settings_var.set(settings)
# Few common background forever-running infrastructural tasks (irregular root tasks).
tasks.append(aiotasks.create_task(
name="stop-flag checker",
coro=_stop_flag_checker(
signal_flag=signal_flag,
stop_flag=stop_flag)))
tasks.append(aiotasks.create_task(
name="ultimate termination",
coro=_ultimate_termination(
settings=settings,
stop_flag=stop_flag)))
tasks.append(aiotasks.create_task(
name="startup/cleanup activities",
coro=_startup_cleanup_activities(
root_tasks=tasks, # used as a "live" view, populated later.
ready_flag=ready_flag,
started_flag=started_flag,
registry=registry,
settings=settings,
indices=indexers.indices,
vault=vault,
memo=memo))) # to purge & finalize the caches in the end.
# Kill all the daemons gracefully when the operator exits (so that they are not "hung").
tasks.append(aiotasks.create_guarded_task(
name="daemon killer", flag=started_flag, logger=logger,
coro=daemons.daemon_killer(
settings=settings,
memories=memories,
operator_paused=operator_paused)))
# Keeping the credentials fresh and valid via the authentication handlers on demand.
tasks.append(aiotasks.create_guarded_task(
name="credentials retriever", flag=started_flag, logger=logger,
coro=activities.authenticator(
registry=registry,
settings=settings,
indices=indexers.indices,
vault=vault,
memo=memo)))
# K8s-event posting. Events are queued in-memory and posted in the background.
# NB: currently, it is a global task, but can be made per-resource or per-object.
tasks.append(aiotasks.create_guarded_task(
name="poster of events", flag=started_flag, logger=logger,
coro=posting.poster(
backbone=insights.backbone,
event_queue=event_queue)))
# Liveness probing -- so that Kubernetes would know that the operator is alive.
if liveness_endpoint:
tasks.append(aiotasks.create_guarded_task(
name="health reporter", flag=started_flag, logger=logger,
coro=probing.health_reporter(
registry=registry,
settings=settings,
endpoint=liveness_endpoint,
indices=indexers.indices,
memo=memo)))
# Permanent observation of what resource kinds and namespaces are available in the cluster.
# Spawn and cancel dimensional tasks as they come and go; dimensions = resources x namespaces.
tasks.append(aiotasks.create_guarded_task(
name="resource observer", flag=started_flag, logger=logger,
coro=observation.resource_observer(
insights=insights,
registry=registry,
settings=settings)))
tasks.append(aiotasks.create_guarded_task(
name="namespace observer", flag=started_flag, logger=logger,
coro=observation.namespace_observer(
clusterwide=clusterwide,
namespaces=namespaces,
insights=insights,
settings=settings)))
# Explicit command is a hack for the CLI to run coroutines in an operator-like environment.
# If not specified, then use the normal resource processing. It is not exposed publicly (yet).
if _command is not None:
tasks.append(aiotasks.create_guarded_task(
name="the command", flag=started_flag, logger=logger, finishable=True,
coro=_command))
else:
tasks.append(aiotasks.create_guarded_task(
name="multidimensional multitasker", flag=started_flag, logger=logger,
coro=orchestration.ochestrator(
settings=settings,
insights=insights,
identity=identity,
operator_paused=operator_paused,
processor=functools.partial(processing.process_resource_event,
lifecycle=lifecycle,
registry=registry,
settings=settings,
indexers=indexers,
memories=memories,
memobase=memo,
event_queue=event_queue))))
# Ensure that all guarded tasks got control for a moment to enter the guard.
await asyncio.sleep(0)
# On Ctrl+C or pod termination, cancel all tasks gracefully.
if threading.current_thread() is threading.main_thread():
# Handle NotImplementedError when ran on Windows since asyncio only supports Unix signals
try:
loop.add_signal_handler(signal.SIGINT, signal_flag.set_result, signal.SIGINT)
loop.add_signal_handler(signal.SIGTERM, signal_flag.set_result, signal.SIGTERM)
except NotImplementedError:
logger.warning("OS signals are ignored: can't add signal handler in Windows.")
else:
logger.warning("OS signals are ignored: running not in the main thread.")
return tasks
async def streaming_block(
*,
resource: resources.Resource,
namespace: Optional[str],
freeze_checker: Optional[primitives.ToggleSet],
) -> AsyncIterator[aiotasks.Future]:
"""
Block the execution until the freeze is off; signal when it is on again.
This prevents both watching and listing while the freeze mode is on,
until it is off. Specifically, the watch-stream closes its connection
once the freeze mode is on, so the while-true & for-event-in-stream cycles
exit, and the streaming coroutine is started again by `infinite_stream()`
(the watcher timeout is swallowed by the freeze time).
Returns a future (or a task) that is set when the freeze is turned on again.
A stop-future is a client-specific way of terminating the streaming HTTPS
connections when the freeze is turned back on. The low-level streaming API
call attaches its `response.close()` to the future's "done" callback,
so that the stream is closed once the freeze is turned back on.
Note: this routine belongs to watching and does not belong to peering.
The freeze can be managed in any other ways: as an imaginary edge case,
imagine a operator with UI with a "pause" button that freezes the operator.
"""
where = f'in {namespace!r}' if namespace is not None else 'cluster-wide'
# Block until unfrozen before even starting the API communication.
if freeze_checker is not None and freeze_checker.is_on():
names = {
toggle.name
for toggle in freeze_checker if toggle.is_on() and toggle.name
}
freezing_reason = f" (blockers: {', '.join(names)})" if names else ""
logger.debug(
f"Freezing the watch-stream for {resource} {where}{freezing_reason}."
)
await freeze_checker.wait_for(False)
names = {
toggle.name
for toggle in freeze_checker if toggle.is_on() and toggle.name
}
resuming_reason = f" (resolved: {', '.join(names)})" if names else ""
logger.debug(
f"Resuming the watch-stream for {resource} {where}{resuming_reason}."
)
# Create the signalling future that the freeze is on again.
freeze_waiter: aiotasks.Future
if freeze_checker is not None:
freeze_waiter = aiotasks.create_task(
freeze_checker.wait_for(True),
name=f"freeze-waiter for {resource}")
else:
freeze_waiter = asyncio.Future() # a dummy just to have it
# Go for the streaming with the prepared freezing/unfreezing setup.
try:
yield freeze_waiter
finally:
with contextlib.suppress(asyncio.CancelledError):
freeze_waiter.cancel()
await freeze_waiter
async def spawn_tasks(
*,
lifecycle: Optional[lifecycles.LifeCycleFn] = None,
registry: Optional[registries.OperatorRegistry] = None,
settings: Optional[configuration.OperatorSettings] = None,
memories: Optional[containers.ResourceMemories] = None,
standalone: Optional[bool] = None,
priority: Optional[int] = None,
peering_name: Optional[str] = None,
liveness_endpoint: Optional[str] = None,
namespace: Optional[str] = None,
stop_flag: Optional[primitives.Flag] = None,
ready_flag: Optional[primitives.Flag] = None,
vault: Optional[credentials.Vault] = None,
) -> Collection[aiotasks.Task]:
"""
Spawn all the tasks needed to run the operator.
The tasks are properly inter-connected with the synchronisation primitives.
"""
loop = asyncio.get_running_loop()
# The freezer and the registry are scoped to this whole task-set, to sync them all.
lifecycle = lifecycle if lifecycle is not None else lifecycles.get_default_lifecycle(
)
registry = registry if registry is not None else registries.get_default_registry(
)
settings = settings if settings is not None else configuration.OperatorSettings(
)
memories = memories if memories is not None else containers.ResourceMemories(
)
vault = vault if vault is not None else global_vault
vault = vault if vault is not None else credentials.Vault()
event_queue: posting.K8sEventQueue = asyncio.Queue()
freeze_name = f"{peering_name!r}@{namespace}" if namespace else f"cluster-wide {peering_name!r}"
freeze_checker = primitives.ToggleSet()
freeze_toggle = await freeze_checker.make_toggle(name=freeze_name)
signal_flag: aiotasks.Future = asyncio.Future()
started_flag: asyncio.Event = asyncio.Event()
tasks: MutableSequence[aiotasks.Task] = []
# Map kwargs into the settings object.
if peering_name is not None:
settings.peering.mandatory = True
settings.peering.name = peering_name
if standalone is not None:
settings.peering.standalone = standalone
if priority is not None:
settings.peering.priority = priority
# Global credentials store for this operator, also for CRD-reading & peering mode detection.
auth.vault_var.set(vault)
# Special case: pass the settings container through the user-side handlers (no explicit args).
# Toolkits have to keep the original operator context somehow, and the only way is contextvars.
posting.settings_var.set(settings)
# Few common background forever-running infrastructural tasks (irregular root tasks).
tasks.append(
aiotasks.create_task(name="stop-flag checker",
coro=_stop_flag_checker(signal_flag=signal_flag,
stop_flag=stop_flag)))
tasks.append(
aiotasks.create_task(name="ultimate termination",
coro=_ultimate_termination(settings=settings,
stop_flag=stop_flag)))
tasks.append(
aiotasks.create_task(
name="startup/cleanup activities",
coro=_startup_cleanup_activities(
root_tasks=tasks, # used as a "live" view, populated later.
ready_flag=ready_flag,
started_flag=started_flag,
registry=registry,
settings=settings,
vault=vault))) # to purge & finalize the caches in the end.
# Kill all the daemons gracefully when the operator exits (so that they are not "hung").
tasks.append(
aiotasks.create_guarded_task(
name="daemon killer",
flag=started_flag,
logger=logger,
coro=daemons.daemon_killer(settings=settings, memories=memories)))
# Keeping the credentials fresh and valid via the authentication handlers on demand.
tasks.append(
aiotasks.create_guarded_task(name="credentials retriever",
flag=started_flag,
logger=logger,
coro=activities.authenticator(
registry=registry,
settings=settings,
vault=vault)))
# K8s-event posting. Events are queued in-memory and posted in the background.
# NB: currently, it is a global task, but can be made per-resource or per-object.
tasks.append(
aiotasks.create_guarded_task(
name="poster of events",
flag=started_flag,
logger=logger,
coro=posting.poster(event_queue=event_queue)))
# Liveness probing -- so that Kubernetes would know that the operator is alive.
if liveness_endpoint:
tasks.append(
aiotasks.create_guarded_task(name="health reporter",
flag=started_flag,
logger=logger,
coro=probing.health_reporter(
registry=registry,
settings=settings,
endpoint=liveness_endpoint)))
# Monitor the peers, unless explicitly disabled.
if await peering.detect_presence(namespace=namespace, settings=settings):
identity = peering.detect_own_id(manual=False)
tasks.append(
aiotasks.create_guarded_task(name="peering keepalive",
flag=started_flag,
logger=logger,
coro=peering.keepalive(
namespace=namespace,
settings=settings,
identity=identity)))
tasks.append(
aiotasks.create_guarded_task(
name="watcher of peering",
flag=started_flag,
logger=logger,
coro=queueing.watcher(
namespace=namespace,
settings=settings,
resource=peering.guess_resource(namespace=namespace),
processor=functools.partial(peering.process_peering_event,
namespace=namespace,
settings=settings,
identity=identity,
freeze_toggle=freeze_toggle))))
# Resource event handling, only once for every known resource (de-duplicated).
for resource in registry.resources:
tasks.append(
aiotasks.create_guarded_task(
name=f"watcher of {resource.name}",
flag=started_flag,
logger=logger,
coro=queueing.watcher(namespace=namespace,
settings=settings,
resource=resource,
freeze_checker=freeze_checker,
processor=functools.partial(
processing.process_resource_event,
lifecycle=lifecycle,
registry=registry,
settings=settings,
memories=memories,
resource=resource,
event_queue=event_queue))))
# On Ctrl+C or pod termination, cancel all tasks gracefully.
if threading.current_thread() is threading.main_thread():
# Handle NotImplementedError when ran on Windows since asyncio only supports Unix signals
try:
loop.add_signal_handler(signal.SIGINT, signal_flag.set_result,
signal.SIGINT)
loop.add_signal_handler(signal.SIGTERM, signal_flag.set_result,
signal.SIGTERM)
except NotImplementedError:
logger.warning(
"OS signals are ignored: can't add signal handler in Windows.")
else:
logger.warning(
"OS signals are ignored: running not in the main thread.")
return tasks
async def test_reraise_escalates_errors():
task = create_task(fail("boo!"))
await asyncio.wait([task], timeout=0.01) # let it start & react
with pytest.raises(Error):
await reraise([task])
