async def test_infinite_watch_never_exits_normally(
settings, resource, stream, namespace, aresponses):
error = aresponses.Response(status=555, reason='stop-infinite-cycle')
stream.feed(
STREAM_WITH_ERROR_410GONE, # watching restarted
STREAM_WITH_UNKNOWN_EVENT, # event ignored
error, # to finally exit it somehow
namespace=namespace,
)
stream.close(namespace=namespace)
events = []
with pytest.raises(aiohttp.ClientResponseError) as e:
async for event in infinite_watch(settings=settings,
resource=resource,
namespace=namespace):
events.append(event)
assert e.value.status == 555
assert e.value.message == 'stop-infinite-cycle'
assert len(events) == 3
assert events[0]['object']['spec'] == 'a'
assert events[1]['object']['spec'] == 'a'
assert events[2]['object']['spec'] == 'b'
async def watcher(
namespace: Union[None, str],
resource: resources.Resource,
processor: WatchStreamProcessor,
freeze_mode: Optional[primitives.Toggle] = None,
) -> None:
"""
The watchers watches for the resource events via the API, and spawns the workers for every object.
All resources and objects are done in parallel, but one single object is handled sequentially
(otherwise, concurrent handling of multiple events of the same object could cause data damage).
The watcher is as non-blocking and async, as possible. It does neither call any external routines,
nor it makes the API calls via the sync libraries.
The watcher is generally a never-ending task (unless an error happens or it is cancelled).
The workers, on the other hand, are limited approximately to the life-time of an object's event.
"""
# All per-object workers are handled as fire-and-forget jobs via the scheduler,
# and communicated via the per-object event queues.
scheduler = await aiojobs.create_scheduler(
limit=config.WorkersConfig.queue_workers_limit)
streams: Streams = {}
try:
# Either use the existing object's queue, or create a new one together with the per-object job.
# "Fire-and-forget": we do not wait for the result; the job destroys itself when it is fully done.
stream = watching.infinite_watch(
resource=resource,
namespace=namespace,
freeze_mode=freeze_mode,
)
async for raw_event in stream:
key = cast(ObjectRef,
(resource, raw_event['object']['metadata']['uid']))
try:
streams[key].replenished.set(
) # interrupt current sleeps, if any.
await streams[key].watchevents.put(raw_event)
except KeyError:
streams[key] = Stream(watchevents=asyncio.Queue(),
replenished=asyncio.Event())
streams[key].replenished.set(
) # interrupt current sleeps, if any.
await streams[key].watchevents.put(raw_event)
await scheduler.spawn(
worker(processor=processor, streams=streams, key=key))
finally:
# Allow the existing workers to finish gracefully before killing them.
await _wait_for_depletion(scheduler=scheduler, streams=streams)
# Forcedly terminate all the fire-and-forget per-object jobs, of they are still running.
await asyncio.shield(scheduler.close())
async def test_infinite_watch_never_exits_normally(resource, stream):
stream.feed(
STREAM_WITH_ERROR_410GONE, # watching restarted
STREAM_WITH_UNKNOWN_EVENT, # event ignored
SampleException(), # to finally exit it somehow
)
events = []
with pytest.raises(SampleException):
async for event in infinite_watch(resource=resource, namespace=None):
events.append(event)
assert len(events) == 3
assert events[0]['object']['spec'] == 'a'
assert events[1]['object']['spec'] == 'a'
assert events[2]['object']['spec'] == 'b'
async def test_exception_escalates(settings, resource, stream, namespace,
enforced_session, mocker):
enforced_session.get = mocker.Mock(side_effect=SampleException())
stream.feed([], namespace=namespace)
stream.close(namespace=namespace)
events = []
with pytest.raises(SampleException):
async for event in infinite_watch(settings=settings,
resource=resource,
namespace=namespace,
_iterations=1):
events.append(event)
assert len(events) == 0
async def watcher(
namespace: Union[None, str],
resource: registries.Resource,
handler: Callable,
):
"""
The watchers watches for the resource events via the API, and spawns the handlers for every object.
All resources and objects are done in parallel, but one single object is handled sequentially
(otherwise, concurrent handling of multiple events of the same object could cause data damage).
The watcher is as non-blocking and async, as possible. It does neither call any external routines,
nor it makes the API calls via the sync libraries.
The watcher is generally a never-ending task (unless an error happens or it is cancelled).
The workers, on the other hand, are limited approximately to the life-time of an object's event.
"""
# All per-object workers are handled as fire-and-forget jobs via the scheduler,
# and communicated via the per-object event queues.
scheduler = await aiojobs.create_scheduler(
limit=config.WorkersConfig.queue_workers_limit)
queues = {}
try:
# Either use the existing object's queue, or create a new one together with the per-object job.
# "Fire-and-forget": we do not wait for the result; the job destroys itself when it is fully done.
async for event in watching.infinite_watch(resource=resource,
namespace=namespace):
key = (resource, event['object']['metadata']['uid'])
try:
await queues[key].put(event)
except KeyError:
queues[key] = asyncio.Queue()
await queues[key].put(event)
await scheduler.spawn(
worker(handler=handler, queues=queues, key=key))
# Allow the existing workers to finish gracefully before killing them.
await _wait_for_depletion(scheduler=scheduler, queues=queues)
finally:
# Forcedly terminate all the fire-and-forget per-object jobs, of they are still running.
await scheduler.close()
async def watcher(
*,
namespace: references.Namespace,
settings: configuration.OperatorSettings,
resource: references.Resource,
processor: WatchStreamProcessor,
operator_paused: Optional[
primitives.ToggleSet] = None, # None for tests & observation
operator_indexed: Optional[
primitives.ToggleSet] = None, # None for tests & observation
resource_indexed: Optional[
primitives.Toggle] = None, # None for tests & non-indexable
) -> None:
"""
The watchers watches for the resource events via the API, and spawns the workers for every object.
All resources and objects are done in parallel, but one single object is handled sequentially
(otherwise, concurrent handling of multiple events of the same object could cause data damage).
The watcher is as non-blocking and async, as possible. It does neither call any external routines,
nor it makes the API calls via the sync libraries.
The watcher is generally a never-ending task (unless an error happens or it is cancelled).
The workers, on the other hand, are limited approximately to the life-time of an object's event.
Watchers spend their time in the infinite watch stream, not in task waiting.
The only valid way for a worker to wake up the watcher is to cancel it:
this will terminate any i/o operation with `asyncio.CancelledError`, where
we can make a decision on whether it was a real cancellation, or our own.
"""
# In case of a failed worker, stop the watcher, and escalate to the operator to stop it.
watcher_task = asyncio.current_task()
worker_error: Optional[BaseException] = None
def exception_handler(scheduler: aiojobs.Scheduler,
context: _aiojobs_Context) -> None:
nonlocal worker_error
if worker_error is None:
worker_error = context['exception']
if watcher_task is not None: # never happens, but is needed for type-checking.
watcher_task.cancel()
# All per-object workers are handled as fire-and-forget jobs via the scheduler,
# and communicated via the per-object event queues.
scheduler: aiojobs.Scheduler
signaller = asyncio.Condition()
scheduler = await aiojobs.create_scheduler(
limit=settings.batching.worker_limit,
exception_handler=exception_handler)
streams: Streams = {}
try:
# Either use the existing object's queue, or create a new one together with the per-object job.
# "Fire-and-forget": we do not wait for the result; the job destroys itself when it is fully done.
stream = watching.infinite_watch(
settings=settings,
resource=resource,
namespace=namespace,
operator_paused=operator_paused,
)
async for raw_event in stream:
# If the listing is over (even if it was empty), the resource kind is pre-indexed.
# At this moment, only the individual workers/processors can block the global readiness.
if raw_event is watching.Bookmark.LISTED:
if operator_indexed is not None and resource_indexed is not None:
await operator_indexed.drop_toggle(resource_indexed)
# Whatever is bookmarked there, don't let it go to the multiplexer. Handle it above.
if isinstance(raw_event, watching.Bookmark):
continue
# Multiplex the raw events to per-resource workers/queues. Start the new ones if needed.
key: ObjectRef = (resource, get_uid(raw_event))
try:
# Feed the worker, as fast as possible, no extra activities.
streams[key].pressure.set(
) # interrupt current sleeps, if any.
await streams[key].backlog.put(raw_event)
except KeyError:
# Block the operator's readiness for individual resource's index handlers.
# But NOT when the readiness is already achieved once! After that, ignore it.
# NB: Strictly before the worker starts -- the processor can be too slow, too late.
resource_object_indexed: Optional[primitives.Toggle] = None
if operator_indexed is not None and operator_indexed.is_on():
operator_indexed = None
if operator_indexed is not None and resource_indexed is not None:
resource_object_indexed = await operator_indexed.make_toggle(
name=f"{key!r}")
# Start the worker, and feed it initially. Starting can be moderately slow.
streams[key] = Stream(backlog=asyncio.Queue(),
pressure=asyncio.Event())
streams[key].pressure.set(
) # interrupt current sleeps, if any.
await streams[key].backlog.put(raw_event)
await scheduler.spawn(
worker(
signaller=signaller,
resource_indexed=resource_object_indexed,
operator_indexed=operator_indexed,
processor=processor,
settings=settings,
streams=streams,
key=key,
))
except asyncio.CancelledError:
if worker_error is None:
raise
else:
raise RuntimeError(
"Event processing has failed with an unrecoverable error. "
"This seems to be a framework bug. "
"The operator will stop to prevent damage.") from worker_error
finally:
# Allow the existing workers to finish gracefully before killing them.
# Ensure the depletion is done even if the watcher is double-cancelled (e.g. in tests).
depletion_task = asyncio.create_task(
_wait_for_depletion(
signaller=signaller,
scheduler=scheduler,
streams=streams,
settings=settings,
))
while not depletion_task.done():
with contextlib.suppress(asyncio.CancelledError):
await asyncio.shield(depletion_task)
# Terminate all the fire-and-forget per-object jobs if they are still running.
# Ensure the scheduler is closed even if the watcher is double-cancelled (e.g. in tests).
closing_task = asyncio.create_task(scheduler.close())
while not closing_task.done():
with contextlib.suppress(asyncio.CancelledError):
await asyncio.shield(closing_task)
async def watcher(
*,
namespace: Union[None, str],
settings: configuration.OperatorSettings,
resource: resources.Resource,
processor: WatchStreamProcessor,
freeze_checker: Optional[primitives.ToggleSet] = None,
) -> None:
"""
The watchers watches for the resource events via the API, and spawns the workers for every object.
All resources and objects are done in parallel, but one single object is handled sequentially
(otherwise, concurrent handling of multiple events of the same object could cause data damage).
The watcher is as non-blocking and async, as possible. It does neither call any external routines,
nor it makes the API calls via the sync libraries.
The watcher is generally a never-ending task (unless an error happens or it is cancelled).
The workers, on the other hand, are limited approximately to the life-time of an object's event.
Watchers spend their time in the infinite watch stream, not in task waiting.
The only valid way for a worker to wake up the watcher is to cancel it:
this will terminate any i/o operation with `asyncio.CancelledError`, where
we can make a decision on whether it was a real cancellation, or our own.
"""
# In case of a failed worker, stop the watcher, and escalate to the operator to stop it.
watcher_task = asyncio.current_task()
worker_error: Optional[BaseException] = None
def exception_handler(scheduler: aiojobs.Scheduler, context: _aiojobs_Context) -> None:
nonlocal worker_error
if worker_error is None:
worker_error = context['exception']
if watcher_task is not None: # never happens, but is needed for type-checking.
watcher_task.cancel()
# All per-object workers are handled as fire-and-forget jobs via the scheduler,
# and communicated via the per-object event queues.
signaller = asyncio.Condition()
scheduler = await aiojobs.create_scheduler(limit=settings.batching.worker_limit,
exception_handler=exception_handler)
streams: Streams = {}
try:
# Either use the existing object's queue, or create a new one together with the per-object job.
# "Fire-and-forget": we do not wait for the result; the job destroys itself when it is fully done.
stream = watching.infinite_watch(
settings=settings,
resource=resource, namespace=namespace,
freeze_checker=freeze_checker,
)
async for raw_event in stream:
key: ObjectRef = (resource, get_uid(raw_event))
try:
streams[key].replenished.set() # interrupt current sleeps, if any.
await streams[key].watchevents.put(raw_event)
except KeyError:
streams[key] = Stream(watchevents=asyncio.Queue(), replenished=asyncio.Event())
streams[key].replenished.set() # interrupt current sleeps, if any.
await streams[key].watchevents.put(raw_event)
await scheduler.spawn(worker(
signaller=signaller,
processor=processor,
settings=settings,
streams=streams,
key=key,
))
except asyncio.CancelledError:
if worker_error is None:
raise
else:
raise RuntimeError("Event processing has failed with an unrecoverable error. "
"This seems to be a framework bug. "
"The operator will stop to prevent damage.") from worker_error
finally:
# Allow the existing workers to finish gracefully before killing them.
await _wait_for_depletion(
signaller=signaller,
scheduler=scheduler,
streams=streams,
settings=settings,
)
# Terminate all the fire-and-forget per-object jobs if they are still running.
await asyncio.shield(scheduler.close())