async def watch_objs(
*,
settings: configuration.OperatorSettings,
resource: references.Resource,
namespace: references.Namespace,
timeout: Optional[float] = None,
since: Optional[str] = None,
context: Optional[auth.APIContext] = None, # injected by the decorator
freeze_waiter: aiotasks.Future,
) -> AsyncIterator[bodies.RawInput]:
"""
Watch objects of a specific resource type.
The cluster-scoped call is used in two cases:
* The resource itself is cluster-scoped, and namespacing makes not sense.
* The operator serves all namespaces for the namespaced custom resource.
Otherwise, the namespace-scoped call is used:
* The resource is namespace-scoped AND operator is namespaced-restricted.
"""
if context is None:
raise RuntimeError("API instance is not injected by the decorator.")
params: Dict[str, str] = {}
params['watch'] = 'true'
if since is not None:
params['resourceVersion'] = since
if timeout is not None:
params['timeoutSeconds'] = str(timeout)
# Stream the parsed events from the response until it is closed server-side,
# or until it is closed client-side by the freeze-waiting future's callbacks.
try:
response = await context.session.get(
url=resource.get_url(server=context.server,
namespace=namespace,
params=params),
timeout=aiohttp.ClientTimeout(
total=settings.watching.client_timeout,
sock_connect=settings.watching.connect_timeout,
),
)
await errors.check_response(response)
response_close_callback = lambda _: response.close()
freeze_waiter.add_done_callback(response_close_callback)
try:
async with response:
async for line in _iter_jsonlines(response.content):
raw_input = cast(bodies.RawInput,
json.loads(line.decode("utf-8")))
yield raw_input
finally:
freeze_waiter.remove_done_callback(response_close_callback)
except (aiohttp.ClientConnectionError, aiohttp.ClientPayloadError,
asyncio.TimeoutError):
pass
async def continuous_watch(
*,
settings: configuration.OperatorSettings,
resource: resources.Resource,
namespace: Optional[str],
freeze_waiter: aiotasks.Future,
) -> AsyncIterator[bodies.RawEvent]:
# First, list the resources regularly, and get the list's resource version.
# Simulate the events with type "None" event - used in detection of causes.
items, resource_version = await fetching.list_objs_rv(resource=resource,
namespace=namespace)
for item in items:
yield {'type': None, 'object': item}
# Repeat through disconnects of the watch as long as the resource version is valid (no errors).
# The individual watching API calls are disconnected by timeout even if the stream is fine.
while not freeze_waiter.done():
# Then, watch the resources starting from the list's resource version.
stream = watch_objs(
settings=settings,
resource=resource,
namespace=namespace,
timeout=settings.watching.server_timeout,
since=resource_version,
freeze_waiter=freeze_waiter,
)
async for raw_input in stream:
raw_type = raw_input['type']
raw_object = raw_input['object']
# "410 Gone" is for the "resource version too old" error, we must restart watching.
# The resource versions are lost by k8s after few minutes (5, as per the official doc).
# The error occurs when there is nothing happening for few minutes. This is normal.
if raw_type == 'ERROR' and cast(bodies.RawError,
raw_object)['code'] == 410:
where = f'in {namespace!r}' if namespace is not None else 'cluster-wide'
logger.debug(
f"Restarting the watch-stream for {resource} {where}.")
return # out of the regular stream, to the infinite stream.
# Other watch errors should be fatal for the operator.
if raw_type == 'ERROR':
raise WatchingError(f"Error in the watch-stream: {raw_object}")
# Ensure that the event is something we understand and can handle.
if raw_type not in ['ADDED', 'MODIFIED', 'DELETED']:
logger.warning("Ignoring an unsupported event type: %r",
raw_input)
continue
# Keep the latest seen resource version for continuation of the stream on disconnects.
body = cast(bodies.RawBody, raw_object)
resource_version = body.get('metadata',
{}).get('resourceVersion',
resource_version)
# Yield normal events to the consumer. Errors are already filtered out.
yield cast(bodies.RawEvent, raw_input)