async def watch_objs( *, settings: configuration.OperatorSettings, resource: references.Resource, namespace: references.Namespace, timeout: Optional[float] = None, since: Optional[str] = None, context: Optional[auth.APIContext] = None, # injected by the decorator freeze_waiter: aiotasks.Future, ) -> AsyncIterator[bodies.RawInput]: """ Watch objects of a specific resource type. The cluster-scoped call is used in two cases: * The resource itself is cluster-scoped, and namespacing makes not sense. * The operator serves all namespaces for the namespaced custom resource. Otherwise, the namespace-scoped call is used: * The resource is namespace-scoped AND operator is namespaced-restricted. """ if context is None: raise RuntimeError("API instance is not injected by the decorator.") params: Dict[str, str] = {} params['watch'] = 'true' if since is not None: params['resourceVersion'] = since if timeout is not None: params['timeoutSeconds'] = str(timeout) # Stream the parsed events from the response until it is closed server-side, # or until it is closed client-side by the freeze-waiting future's callbacks. try: response = await context.session.get( url=resource.get_url(server=context.server, namespace=namespace, params=params), timeout=aiohttp.ClientTimeout( total=settings.watching.client_timeout, sock_connect=settings.watching.connect_timeout, ), ) await errors.check_response(response) response_close_callback = lambda _: response.close() freeze_waiter.add_done_callback(response_close_callback) try: async with response: async for line in _iter_jsonlines(response.content): raw_input = cast(bodies.RawInput, json.loads(line.decode("utf-8"))) yield raw_input finally: freeze_waiter.remove_done_callback(response_close_callback) except (aiohttp.ClientConnectionError, aiohttp.ClientPayloadError, asyncio.TimeoutError): pass
async def continuous_watch( *, settings: configuration.OperatorSettings, resource: resources.Resource, namespace: Optional[str], freeze_waiter: aiotasks.Future, ) -> AsyncIterator[bodies.RawEvent]: # First, list the resources regularly, and get the list's resource version. # Simulate the events with type "None" event - used in detection of causes. items, resource_version = await fetching.list_objs_rv(resource=resource, namespace=namespace) for item in items: yield {'type': None, 'object': item} # Repeat through disconnects of the watch as long as the resource version is valid (no errors). # The individual watching API calls are disconnected by timeout even if the stream is fine. while not freeze_waiter.done(): # Then, watch the resources starting from the list's resource version. stream = watch_objs( settings=settings, resource=resource, namespace=namespace, timeout=settings.watching.server_timeout, since=resource_version, freeze_waiter=freeze_waiter, ) async for raw_input in stream: raw_type = raw_input['type'] raw_object = raw_input['object'] # "410 Gone" is for the "resource version too old" error, we must restart watching. # The resource versions are lost by k8s after few minutes (5, as per the official doc). # The error occurs when there is nothing happening for few minutes. This is normal. if raw_type == 'ERROR' and cast(bodies.RawError, raw_object)['code'] == 410: where = f'in {namespace!r}' if namespace is not None else 'cluster-wide' logger.debug( f"Restarting the watch-stream for {resource} {where}.") return # out of the regular stream, to the infinite stream. # Other watch errors should be fatal for the operator. if raw_type == 'ERROR': raise WatchingError(f"Error in the watch-stream: {raw_object}") # Ensure that the event is something we understand and can handle. if raw_type not in ['ADDED', 'MODIFIED', 'DELETED']: logger.warning("Ignoring an unsupported event type: %r", raw_input) continue # Keep the latest seen resource version for continuation of the stream on disconnects. body = cast(bodies.RawBody, raw_object) resource_version = body.get('metadata', {}).get('resourceVersion', resource_version) # Yield normal events to the consumer. Errors are already filtered out. yield cast(bodies.RawEvent, raw_input)