async def touch(
*,
identity: Identity,
settings: configuration.OperatorSettings,
namespace: Optional[str],
lifetime: Optional[int] = None,
) -> None:
name = settings.peering.name
resource = guess_resource(namespace=namespace)
peer = Peer(
identity=identity,
priority=settings.peering.priority,
lifetime=settings.peering.lifetime if lifetime is None else lifetime,
)
patch = patches.Patch()
patch.update(
{'status': {
identity: None if peer.is_dead else peer.as_dict()
}})
rsp = await patching.patch_obj(resource=resource,
namespace=namespace,
name=name,
patch=patch)
if not settings.peering.stealth or rsp is None:
where = f"in {namespace!r}" if namespace else "cluster-wide"
result = "not found" if rsp is None else "ok"
logger.debug(f"Keep-alive in {name!r} {where}: {result}.")
async def index_resource(
*,
indexers: OperatorIndexers,
registry: registries.OperatorRegistry,
settings: configuration.OperatorSettings,
resource: references.Resource,
raw_event: bodies.RawEvent,
memory: containers.ResourceMemory,
logger: Union[logging.Logger, logging.LoggerAdapter],
body: bodies.Body,
) -> None:
"""
Populate the indices from the received event. Log but ignore all errors.
This is a lightweight and standalone process, which is executed before
any real handlers are invoked. Multi-step calls are also not supported.
If the handler fails, it fails and is never retried.
Note: K8s-event posting is skipped for `kopf.on.event` handlers,
as they should be silent. Still, the messages are logged normally.
"""
if not registry._resource_indexing.has_handlers(resource=resource):
pass
elif raw_event['type'] == 'DELETED':
# Do not index it if it is deleted. Just discard quickly (ASAP!).
indexers.discard(body=body)
else:
# Otherwise, go for full indexing with handlers invocation with all kwargs.
cause = causation.ResourceIndexingCause(
resource=resource,
indices=indexers.indices,
logger=logger,
patch=patches.Patch(), # NB: not applied. TODO: get rid of it!
body=body,
memo=memory.memo,
)
# Note: the indexing state contains only failures & retries. Successes will be re-executed.
indexing_handlers = registry._resource_indexing.get_handlers(
cause=cause)
state = memory.indexing_state
state = state if state is not None else states.State.from_scratch()
state = state.with_handlers(indexing_handlers)
outcomes = await handling.execute_handlers_once(
lifecycle=lifecycles.all_at_once,
settings=settings,
handlers=indexing_handlers,
cause=cause,
state=state,
default_errors=handlers.ErrorsMode.IGNORED,
)
indexers.replace(body=body, outcomes=outcomes)
# Remember only failures & retries. Omit successes -- let them be re-executed every time.
state = state.with_outcomes(outcomes).without_successes()
memory.indexing_state = state if state else None
async def apply_reaction_outcomes(
*,
resource: resources.Resource,
body: bodies.Body,
patch: patches.Patch,
delays: Collection[float],
logger: logging_engine.ObjectLogger,
replenished: asyncio.Event,
) -> None:
delay = min(delays) if delays else None
if patch:
logger.debug("Patching with: %r", patch)
await patching.patch_obj(resource=resource, patch=patch, body=body)
# Sleep strictly after patching, never before -- to keep the status proper.
# The patching above, if done, interrupts the sleep instantly, so we skip it at all.
# Note: a zero-second or negative sleep is still a sleep, it will trigger a dummy patch.
if delay and patch:
logger.debug(
f"Sleeping was skipped because of the patch, {delay} seconds left."
)
elif delay is None and not patch:
logger.debug(
f"Handling cycle is finished, waiting for new changes since now.")
elif delay is not None:
if delay > WAITING_KEEPALIVE_INTERVAL:
limit = WAITING_KEEPALIVE_INTERVAL
logger.debug(
f"Sleeping for {delay} (capped {limit}) seconds for the delayed handlers."
)
unslept_delay = await sleeping.sleep_or_wait(limit, replenished)
elif delay > 0:
logger.debug(
f"Sleeping for {delay} seconds for the delayed handlers.")
unslept_delay = await sleeping.sleep_or_wait(delay, replenished)
else:
unslept_delay = None # no need to sleep? means: slept in full.
if unslept_delay is not None:
logger.debug(
f"Sleeping was interrupted by new changes, {unslept_delay} seconds left."
)
else:
# Any unique always-changing value will work; not necessary a timestamp.
dummy_value = datetime.datetime.utcnow().isoformat()
dummy_patch = patches.Patch(
{'status': {
'kopf': {
'dummy': dummy_value
}
}})
logger.debug("Provoking reaction with: %r", dummy_patch)
await patching.patch_obj(resource=resource,
patch=dummy_patch,
body=body)
async def clean(
*,
peers: Iterable[Peer],
settings: configuration.OperatorSettings,
resource: references.Resource,
namespace: references.Namespace,
) -> None:
name = settings.peering.name
patch = patches.Patch()
patch.update({'status': {peer.identity: None for peer in peers}})
await patching.patch_obj(resource=resource, namespace=namespace, name=name, patch=patch)
def _create_watching_cause(
resource: resources_.Resource,
event: bodies.RawEvent,
) -> causation.ResourceWatchingCause:
return causation.detect_resource_watching_cause(
resource=resource,
raw_event=event,
patch=patches.Patch(), # unused
type=event['type'], # unused
body=bodies.Body(event['object']), # unused
raw=event, # unused
)
async def spawn_resource_daemons(
*,
settings: configuration.OperatorSettings,
handlers: Sequence[handlers_.ResourceSpawningHandler],
daemons: MutableMapping[handlers_.HandlerId, containers.Daemon],
cause: causation.ResourceSpawningCause,
memory: containers.ResourceMemory,
) -> Collection[float]:
"""
Ensure that all daemons are spawned for this individual resource.
This function can be called multiple times on multiple handling cycles
(though usually should be called on the first-seen occasion), so it must
be idempotent: not having duplicating side-effects on multiple calls.
"""
if memory.live_fresh_body is None: # for type-checking; "not None" is ensured in processing.
raise RuntimeError(
"A daemon is spawned with None as body. This is a bug. Please report."
)
for handler in handlers:
if handler.id not in daemons:
stopper = primitives.DaemonStopper()
daemon_cause = causation.DaemonCause(
resource=cause.resource,
indices=cause.indices,
logger=cause.logger,
body=memory.live_fresh_body,
memo=memory.memo,
patch=patches.Patch(
), # not the same as the one-shot spawning patch!
stopper=stopper, # for checking (passed to kwargs)
)
daemon = containers.Daemon(
stopper=stopper, # for stopping (outside of causes)
handler=handler,
logger=loggers.LocalObjectLogger(body=cause.body,
settings=settings),
task=aiotasks.create_task(
_runner(
settings=settings,
daemons=daemons, # for self-garbage-collection
handler=handler,
cause=daemon_cause,
memory=memory,
),
name=f'runner of {handler.id}'
), # sometimes, daemons; sometimes, timers.
)
daemons[handler.id] = daemon
return []
async def apply(
*,
settings: configuration.OperatorSettings,
resource: references.Resource,
body: bodies.Body,
patch: patches.Patch,
delays: Collection[float],
logger: loggers.ObjectLogger,
replenished: asyncio.Event,
) -> bool:
delay = min(delays) if delays else None
# Delete dummies on occasion, but don't trigger special patching for them [discussable].
if patch: # TODO: LATER: and the dummies are there (without additional methods?)
settings.persistence.progress_storage.touch(body=body, patch=patch, value=None)
# Actually patch if it was not empty originally or after the dummies removal.
await patch_and_check(resource=resource, patch=patch, body=body, logger=logger)
# Sleep strictly after patching, never before -- to keep the status proper.
# The patching above, if done, interrupts the sleep instantly, so we skip it at all.
# Note: a zero-second or negative sleep is still a sleep, it will trigger a dummy patch.
applied = False
if delay and patch:
logger.debug(f"Sleeping was skipped because of the patch, {delay} seconds left.")
elif delay is not None:
if delay > WAITING_KEEPALIVE_INTERVAL:
limit = WAITING_KEEPALIVE_INTERVAL
logger.debug(f"Sleeping for {delay} (capped {limit}) seconds for the delayed handlers.")
unslept_delay = await sleep_or_wait(limit, replenished)
elif delay > 0:
logger.debug(f"Sleeping for {delay} seconds for the delayed handlers.")
unslept_delay = await sleep_or_wait(delay, replenished)
else:
unslept_delay = None # no need to sleep? means: slept in full.
# Exclude cases when touching immediately after patching (including: ``delay == 0``).
if patch and not delay:
pass
elif unslept_delay is not None:
logger.debug(f"Sleeping was interrupted by new changes, {unslept_delay} seconds left.")
else:
# Any unique always-changing value will work; not necessary a timestamp.
value = datetime.datetime.utcnow().isoformat()
touch = patches.Patch()
settings.persistence.progress_storage.touch(body=body, patch=touch, value=value)
await patch_and_check(resource=resource, patch=touch, body=body, logger=logger)
elif not patch: # no patch/touch and no delay
applied = True
return applied
def iter_event_handlers(
self,
resource: resources_.Resource,
event: bodies.Event,
) -> Iterator[ResourceHandler]:
warnings.warn("registry.iter_event_handlers() is deprecated; "
"use registry.iter_resource_watching_handlers().", DeprecationWarning)
return self.iter_resource_watching_handlers(cause=causation.detect_resource_watching_cause(
resource=resource,
event=event,
patch=patches.Patch(), # unused
type=event['type'], # unused
body=event['object'], # unused
raw=event, # unused
logger=logging.Logger('kopf'), # unused
))
async def apply_peers(
peers: Iterable[Peer],
name: str,
namespace: Union[None, str],
legacy: bool = False,
) -> None:
"""
Apply the changes in the peers to the sync-object.
The dead peers are removed, the new or alive peers are stored.
Note: this does NOT change their `lastseen` field, so do it explicitly with ``touch()``.
"""
patch = patches.Patch()
patch.update({'status': {peer.id: None if peer.is_dead else peer.as_dict() for peer in peers}})
resource = (LEGACY_PEERING_RESOURCE if legacy else
CLUSTER_PEERING_RESOURCE if namespace is None else
NAMESPACED_PEERING_RESOURCE)
await patching.patch_obj(resource=resource, namespace=namespace, name=name, patch=patch)
async def serve_admission_request(
# Required for all webhook servers, meaningless without it:
request: reviews.Request,
*,
# Optional for webhook servers that can recognise this information:
headers: Optional[Mapping[str, str]] = None,
sslpeer: Optional[Mapping[str, Any]] = None,
webhook: Optional[ids.HandlerId] = None,
reason: Optional[
handlers.
WebhookType] = None, # TODO: undocumented: requires typing clarity!
# Injected by partial() from spawn_tasks():
settings: configuration.OperatorSettings,
memories: containers.ResourceMemories,
memobase: ephemera.AnyMemo,
registry: registries.OperatorRegistry,
insights: references.Insights,
indices: ephemera.Indices,
) -> reviews.Response:
"""
The actual and the only implementation of the `WebhookFn` protocol.
This function is passed to all webhook servers/tunnels to be called
whenever a new admission request is received.
Some parameters are provided by the framework itself via partial binding,
so that the resulting function matches the `WebhookFn` protocol. Other
parameters are passed by the webhook servers when they call the function.
"""
# Reconstruct the cause specially for web handlers.
resource = find_resource(request=request, insights=insights)
operation = request.get('request', {}).get('operation')
userinfo = request.get('request', {}).get('userInfo')
new_body = request.get('request', {}).get('object')
old_body = request.get('request', {}).get('oldObject')
raw_body = new_body if new_body is not None else old_body
if userinfo is None:
raise MissingDataError(
"User info is missing from the admission request.")
if raw_body is None:
raise MissingDataError(
"Either old or new object is missing from the admission request.")
memory = await memories.recall(raw_body,
memo=memobase,
ephemeral=operation == 'CREATE')
body = bodies.Body(raw_body)
patch = patches.Patch()
warnings: List[str] = []
cause = causation.ResourceWebhookCause(
resource=resource,
indices=indices,
logger=loggers.LocalObjectLogger(body=body, settings=settings),
patch=patch,
memo=memory.memo,
body=body,
userinfo=userinfo,
warnings=warnings,
operation=operation,
dryrun=bool(request.get('request', {}).get('dryRun')),
sslpeer=sslpeer if sslpeer is not None else
{}, # ensure a mapping even if not provided.
headers=headers if headers is not None else
{}, # ensure a mapping even if not provided.
webhook=webhook,
reason=reason,
)
# Retrieve the handlers to be executed; maybe only one if the webhook server provides a hint.
handlers_ = registry._resource_webhooks.get_handlers(cause)
state = states.State.from_scratch().with_handlers(handlers_)
outcomes = await handling.execute_handlers_once(
lifecycle=lifecycles.all_at_once,
settings=settings,
handlers=handlers_,
cause=cause,
state=state,
default_errors=handlers.ErrorsMode.PERMANENT,
)
# Construct the response as per Kubernetes's conventions and expectations.
response = build_response(
request=request,
outcomes=outcomes,
warnings=warnings,
jsonpatch=patch.as_json_patch(),
)
return response
async def configuration_manager(
*,
reason: handlers.WebhookType,
selector: references.Selector,
registry: registries.OperatorRegistry,
settings: configuration.OperatorSettings,
insights: references.Insights,
container: primitives.Container[reviews.WebhookClientConfig],
) -> None:
"""
Manage the webhook configurations dynamically.
This is one of an operator's root tasks that run forever.
If exited, the whole operator exits as by an error.
The manager waits for changes in one of these:
* Observed resources in the cluster (via insights).
* A new webhook client config yielded by the webhook server.
On either of these occasion, the manager rebuilds the webhook configuration
and applies it to the specified configuration resources in the cluster
(for which it needs some RBAC permissions).
Besides, it also creates an webhook configuration resource if it is absent.
"""
# Do nothing if not managed. The root task cannot be skipped from creation,
# since the managed mode is only set at the startup activities.
if settings.admission.managed is None:
await asyncio.Event().wait()
return
# Wait until the prerequisites for managing are available (scanned from the cluster).
await insights.ready_resources.wait()
resource = await insights.backbone.wait_for(selector)
all_handlers = registry._resource_webhooks.get_all_handlers()
all_handlers = [h for h in all_handlers if h.reason == reason]
# Optionally (if configured), pre-create the configuration objects if they are absent.
# Use the try-or-fail strategy instead of check-and-do -- to reduce the RBAC requirements.
try:
await creating.create_obj(resource=resource,
name=settings.admission.managed)
except errors.APIConflictError:
pass # exists already
except errors.APIForbiddenError:
logger.error(f"Not enough RBAC permissions to create a {resource}.")
raise
# Execute either when actually changed (yielded from the webhook server),
# or the condition is chain-notified (from the insights: on resources/namespaces revision).
# Ignore inconsistencies: they are expected -- the server fills the defaults.
client_config: Optional[reviews.WebhookClientConfig] = None
try:
async for client_config in container.as_changed():
logger.info(
f"Reconfiguring the {reason.value} webhook {settings.admission.managed}."
)
webhooks = build_webhooks(all_handlers,
resources=insights.resources,
name_suffix=settings.admission.managed,
client_config=client_config)
await patching.patch_obj(
resource=resource,
namespace=None,
name=settings.admission.managed,
patch=patches.Patch({'webhooks': webhooks}),
)
finally:
# Attempt to remove all managed webhooks, except for the strict ones.
if client_config is not None:
logger.info(
f"Cleaning up the admission webhook {settings.admission.managed}."
)
webhooks = build_webhooks(all_handlers,
resources=insights.resources,
name_suffix=settings.admission.managed,
client_config=client_config,
persistent_only=True)
await patching.patch_obj(
resource=resource,
namespace=None,
name=settings.admission.managed,
patch=patches.Patch({'webhooks': webhooks}),
)
async def resource_handler(
lifecycle: lifecycles.LifeCycleFn,
registry: registries.OperatorRegistry,
memories: containers.ResourceMemories,
resource: resources.Resource,
event: bodies.Event,
freeze: asyncio.Event,
replenished: asyncio.Event,
event_queue: posting.K8sEventQueue,
) -> None:
"""
Handle a single custom object low-level watch-event.
Convert the low-level events, as provided by the watching/queueing tasks,
to the high-level causes, and then call the cause-handling logic.
All the internally provoked changes are intercepted, do not create causes,
and therefore do not call the handling logic.
"""
body: bodies.Body = event['object']
patch: patches.Patch = patches.Patch()
delay: Optional[float] = None
# Each object has its own prefixed logger, to distinguish parallel handling.
logger = logging_engine.ObjectLogger(body=body)
posting.event_queue_loop_var.set(asyncio.get_running_loop())
posting.event_queue_var.set(event_queue) # till the end of this object's task.
# If the global freeze is set for the processing (i.e. other operator overrides), do nothing.
if freeze.is_set():
logger.debug("Ignoring the events due to freeze.")
return
# Recall what is stored about that object. Share it in little portions with the consumers.
# And immediately forget it if the object is deleted from the cluster (but keep in memory).
memory = await memories.recall(body, noticed_by_listing=event['type'] is None)
if event['type'] == 'DELETED':
await memories.forget(body)
# Invoke all silent spies. No causation, no progress storage is performed.
if registry.has_resource_watching_handlers(resource=resource):
resource_watching_cause = causation.detect_resource_watching_cause(
event=event,
resource=resource,
logger=logger,
patch=patch,
memo=memory.user_data,
)
await handle_resource_watching_cause(
lifecycle=lifecycles.all_at_once,
registry=registry,
memory=memory,
cause=resource_watching_cause,
)
# Object patch accumulator. Populated by the methods. Applied in the end of the handler.
# Detect the cause and handle it (or at least log this happened).
if registry.has_resource_changing_handlers(resource=resource):
extra_fields = registry.get_extra_fields(resource=resource)
old, new, diff = lastseen.get_essential_diffs(body=body, extra_fields=extra_fields)
resource_changing_cause = causation.detect_resource_changing_cause(
event=event,
resource=resource,
logger=logger,
patch=patch,
old=old,
new=new,
diff=diff,
memo=memory.user_data,
initial=memory.noticed_by_listing and not memory.fully_handled_once,
requires_finalizer=registry.requires_finalizer(resource=resource, body=body),
)
delay = await handle_resource_changing_cause(
lifecycle=lifecycle,
registry=registry,
memory=memory,
cause=resource_changing_cause,
)
# Whatever was done, apply the accumulated changes to the object.
# But only once, to reduce the number of API calls and the generated irrelevant events.
if patch:
logger.debug("Patching with: %r", patch)
await patching.patch_obj(resource=resource, patch=patch, body=body)
# Sleep strictly after patching, never before -- to keep the status proper.
# The patching above, if done, interrupts the sleep instantly, so we skip it at all.
if delay and patch:
logger.debug(f"Sleeping was skipped because of the patch, {delay} seconds left.")
elif delay:
logger.debug(f"Sleeping for {delay} seconds for the delayed handlers.")
unslept = await sleeping.sleep_or_wait(min(delay, WAITING_KEEPALIVE_INTERVAL), replenished)
if unslept is not None:
logger.debug(f"Sleeping was interrupted by new changes, {unslept} seconds left.")
else:
now = datetime.datetime.utcnow()
dummy = patches.Patch({'status': {'kopf': {'dummy': now.isoformat()}}})
logger.debug("Provoking reaction with: %r", dummy)
await patching.patch_obj(resource=resource, patch=dummy, body=body)
async def process_resource_event(
lifecycle: lifecycles.LifeCycleFn,
indexers: indexing.OperatorIndexers,
registry: registries.OperatorRegistry,
settings: configuration.OperatorSettings,
memories: containers.ResourceMemories,
memobase: ephemera.AnyMemo,
resource: references.Resource,
raw_event: bodies.RawEvent,
event_queue: posting.K8sEventQueue,
stream_pressure: Optional[asyncio.Event] = None, # None for tests
resource_indexed: Optional[
primitives.Toggle] = None, # None for tests & observation
operator_indexed: Optional[
primitives.ToggleSet] = None, # None for tests & observation
) -> None:
"""
Handle a single custom object low-level watch-event.
Convert the low-level events, as provided by the watching/queueing tasks,
to the high-level causes, and then call the cause-handling logic.
"""
# Recall what is stored about that object. Share it in little portions with the consumers.
# And immediately forget it if the object is deleted from the cluster (but keep in memory).
raw_type, raw_body = raw_event['type'], raw_event['object']
memory = await memories.recall(raw_body,
noticed_by_listing=raw_type is None,
memo=memobase)
if memory.live_fresh_body is not None:
memory.live_fresh_body._replace_with(raw_body)
if raw_type == 'DELETED':
await memories.forget(raw_body)
# Convert to a heavy mapping-view wrapper only now, when heavy processing begins.
# Raw-event streaming, queueing, and batching use regular lightweight dicts.
# Why here? 1. Before it splits into multiple causes & handlers for the same object's body;
# 2. After it is batched (queueing); 3. While the "raw" parsed JSON is still known;
# 4. Same as where a patch object of a similar wrapping semantics is created.
body = memory.live_fresh_body if memory.live_fresh_body is not None else bodies.Body(
raw_body)
patch = patches.Patch()
# Throttle the non-handler-related errors. The regular event watching/batching continues
# to prevent queue overfilling, but the processing is skipped (events are ignored).
# Choice of place: late enough to have a per-resource memory for a throttler; also, a logger.
# But early enough to catch environment errors from K8s API, and from most of the complex code.
async with effects.throttled(
throttler=memory.error_throttler,
logger=loggers.LocalObjectLogger(body=body, settings=settings),
delays=settings.batching.error_delays,
wakeup=stream_pressure,
) as should_run:
if should_run:
# Each object has its own prefixed logger, to distinguish parallel handling.
logger = loggers.ObjectLogger(body=body, settings=settings)
posting.event_queue_loop_var.set(asyncio.get_running_loop())
posting.event_queue_var.set(
event_queue) # till the end of this object's task.
# [Pre-]populate the indices. This must be lightweight.
await indexing.index_resource(
registry=registry,
indexers=indexers,
settings=settings,
resource=resource,
raw_event=raw_event,
body=body,
memory=memory,
logger=loggers.TerseObjectLogger(body=body, settings=settings),
)
# Wait for all other individual resources and all other resource kinds' lists to finish.
# If this one has changed while waiting for the global readiness, let it be reprocessed.
if operator_indexed is not None and resource_indexed is not None:
await operator_indexed.drop_toggle(resource_indexed)
if operator_indexed is not None:
await operator_indexed.wait_for(
True) # other resource kinds & objects.
if stream_pressure is not None and stream_pressure.is_set():
return
# Do the magic -- do the job.
delays, matched = await process_resource_causes(
lifecycle=lifecycle,
indexers=indexers,
registry=registry,
settings=settings,
resource=resource,
raw_event=raw_event,
body=body,
patch=patch,
memory=memory,
logger=logger,
)
# Whatever was done, apply the accumulated changes to the object, or sleep-n-touch for delays.
# But only once, to reduce the number of API calls and the generated irrelevant events.
# And only if the object is at least supposed to exist (not "GONE"), even if actually does not.
if raw_event['type'] != 'DELETED':
applied = await effects.apply(
settings=settings,
resource=resource,
body=body,
patch=patch,
logger=logger,
delays=delays,
stream_pressure=stream_pressure,
)
if applied and matched:
logger.debug(
f"Handling cycle is finished, waiting for new changes since now."
)
async def process_resource_event(
lifecycle: lifecycles.LifeCycleFn,
registry: registries.OperatorRegistry,
settings: configuration.OperatorSettings,
memories: containers.ResourceMemories,
resource: resources.Resource,
raw_event: bodies.RawEvent,
replenished: asyncio.Event,
event_queue: posting.K8sEventQueue,
) -> None:
"""
Handle a single custom object low-level watch-event.
Convert the low-level events, as provided by the watching/queueing tasks,
to the high-level causes, and then call the cause-handling logic.
All the internally provoked changes are intercepted, do not create causes,
and therefore do not call the handling logic.
"""
finalizer = settings.persistence.finalizer
# Recall what is stored about that object. Share it in little portions with the consumers.
# And immediately forget it if the object is deleted from the cluster (but keep in memory).
raw_type, raw_body = raw_event['type'], raw_event['object']
memory = await memories.recall(raw_body,
noticed_by_listing=raw_type is None)
if memory.live_fresh_body is not None:
memory.live_fresh_body._replace_with(raw_body)
if raw_type == 'DELETED':
await memories.forget(raw_body)
# Convert to a heavy mapping-view wrapper only now, when heavy processing begins.
# Raw-event streaming, queueing, and batching use regular lightweight dicts.
# Why here? 1. Before it splits into multiple causes & handlers for the same object's body;
# 2. After it is batched (queueing); 3. While the "raw" parsed JSON is still known;
# 4. Same as where a patch object of a similar wrapping semantics is created.
body = memory.live_fresh_body if memory.live_fresh_body is not None else bodies.Body(
raw_body)
patch = patches.Patch()
# Each object has its own prefixed logger, to distinguish parallel handling.
logger = logging_engine.ObjectLogger(body=body, settings=settings)
posting.event_queue_loop_var.set(asyncio.get_running_loop())
posting.event_queue_var.set(
event_queue) # till the end of this object's task.
extra_fields = registry.resource_changing_handlers[
resource].get_extra_fields()
old = settings.persistence.diffbase_storage.fetch(body=body)
new = settings.persistence.diffbase_storage.build(
body=body, extra_fields=extra_fields)
old = settings.persistence.progress_storage.clear(
essence=old) if old is not None else None
new = settings.persistence.progress_storage.clear(
essence=new) if new is not None else None
diff = diffs.diff(old, new)
# Detect what are we going to do on this processing cycle.
resource_watching_cause = causation.detect_resource_watching_cause(
raw_event=raw_event,
resource=resource,
logger=logger,
patch=patch,
body=body,
memo=memory.memo,
) if registry.resource_watching_handlers[resource] else None
resource_spawning_cause = causation.detect_resource_spawning_cause(
resource=resource,
logger=logger,
patch=patch,
body=body,
memo=memory.memo,
reset=bool(
diff), # only essential changes reset idling, not every event
) if registry.resource_spawning_handlers[resource] else None
resource_changing_cause = causation.detect_resource_changing_cause(
finalizer=finalizer,
raw_event=raw_event,
resource=resource,
logger=logger,
patch=patch,
body=body,
old=old,
new=new,
diff=diff,
memo=memory.memo,
initial=memory.noticed_by_listing and not memory.fully_handled_once,
) if registry.resource_changing_handlers[resource] else None
# Block the object from deletion if we have anything to do in its end of life:
# specifically, if there are daemons to kill or mandatory on-deletion handlers to call.
# The high-level handlers are prevented if this event cycle is dedicated to the finalizer.
# The low-level handlers (on-event spying & daemon spawning) are still executed asap.
deletion_is_ongoing = finalizers.is_deletion_ongoing(body=body)
deletion_is_blocked = finalizers.is_deletion_blocked(body=body,
finalizer=finalizer)
deletion_must_be_blocked = (
(resource_spawning_cause is not None
and registry.resource_spawning_handlers[resource].requires_finalizer(
cause=resource_spawning_cause,
excluded=memory.forever_stopped,
)) or
(resource_changing_cause is not None
and registry.resource_changing_handlers[resource].requires_finalizer(
cause=resource_changing_cause, )))
if deletion_must_be_blocked and not deletion_is_blocked and not deletion_is_ongoing:
logger.debug(
"Adding the finalizer, thus preventing the actual deletion.")
finalizers.block_deletion(body=body, patch=patch, finalizer=finalizer)
resource_changing_cause = None # prevent further high-level processing this time
if not deletion_must_be_blocked and deletion_is_blocked:
logger.debug(
"Removing the finalizer, as there are no handlers requiring it.")
finalizers.allow_deletion(body=body, patch=patch, finalizer=finalizer)
resource_changing_cause = None # prevent further high-level processing this time
# Invoke all the handlers that should or could be invoked at this processing cycle.
# The low-level spies go ASAP always. However, the daemons are spawned before the high-level
# handlers and killed after them: the daemons should live throughout the full object lifecycle.
if resource_watching_cause is not None:
await process_resource_watching_cause(
lifecycle=lifecycles.all_at_once,
registry=registry,
settings=settings,
cause=resource_watching_cause,
)
resource_spawning_delays: Collection[float] = []
if resource_spawning_cause is not None:
resource_spawning_delays = await process_resource_spawning_cause(
registry=registry,
settings=settings,
memory=memory,
cause=resource_spawning_cause,
)
resource_changing_delays: Collection[float] = []
if resource_changing_cause is not None:
resource_changing_delays = await process_resource_changing_cause(
lifecycle=lifecycle,
registry=registry,
settings=settings,
memory=memory,
cause=resource_changing_cause,
)
# Release the object if everything is done, and it is marked for deletion.
# But not when it has already gone.
if deletion_is_ongoing and deletion_is_blocked \
and not resource_spawning_delays \
and not resource_changing_delays:
logger.debug(
"Removing the finalizer, thus allowing the actual deletion.")
finalizers.allow_deletion(body=body, patch=patch, finalizer=finalizer)
# Whatever was done, apply the accumulated changes to the object, or sleep-n-touch for delays.
# But only once, to reduce the number of API calls and the generated irrelevant events.
# And only if the object is at least supposed to exist (not "GONE"), even if actually does not.
if raw_event['type'] != 'DELETED':
await apply_reaction_outcomes(
settings=settings,
resource=resource,
body=body,
patch=patch,
logger=logger,
delays=list(resource_spawning_delays) +
list(resource_changing_delays),
replenished=replenished,
)
async def process_resource_event(
lifecycle: lifecycles.LifeCycleFn,
registry: registries.OperatorRegistry,
memories: containers.ResourceMemories,
resource: resources.Resource,
raw_event: bodies.RawEvent,
replenished: asyncio.Event,
event_queue: posting.K8sEventQueue,
) -> None:
"""
Handle a single custom object low-level watch-event.
Convert the low-level events, as provided by the watching/queueing tasks,
to the high-level causes, and then call the cause-handling logic.
All the internally provoked changes are intercepted, do not create causes,
and therefore do not call the handling logic.
"""
# Convert to a heavy mapping-view wrapper only now, when heavy processing begins.
# Raw-event streaming, queueing, and batching use regular lightweight dicts.
# Why here? 1. Before it splits into multiple causes & handlers for the same object's body;
# 2. After it is batched (queueing); 3. While the "raw" parsed JSON is still known;
# 4. Same as where a patch object of a similar wrapping semantics is created.
body = bodies.Body(raw_event['object'])
patch = patches.Patch()
delay: Optional[float] = None
# Each object has its own prefixed logger, to distinguish parallel handling.
logger = logging_engine.ObjectLogger(body=body)
posting.event_queue_loop_var.set(asyncio.get_running_loop())
posting.event_queue_var.set(
event_queue) # till the end of this object's task.
# Recall what is stored about that object. Share it in little portions with the consumers.
# And immediately forget it if the object is deleted from the cluster (but keep in memory).
memory = await memories.recall(
body, noticed_by_listing=raw_event['type'] is None)
if raw_event['type'] == 'DELETED':
await memories.forget(body)
# Invoke all silent spies. No causation, no progress storage is performed.
if registry.resource_watching_handlers[resource]:
resource_watching_cause = causation.detect_resource_watching_cause(
raw_event=raw_event,
resource=resource,
logger=logger,
patch=patch,
body=body,
memo=memory.user_data,
)
await process_resource_watching_cause(
lifecycle=lifecycles.all_at_once,
registry=registry,
memory=memory,
cause=resource_watching_cause,
)
# Object patch accumulator. Populated by the methods. Applied in the end of the handler.
# Detect the cause and handle it (or at least log this happened).
if registry.resource_changing_handlers[resource]:
extra_fields = registry.resource_changing_handlers[
resource].get_extra_fields()
old, new, diff = lastseen.get_essential_diffs(
body=body, extra_fields=extra_fields)
resource_changing_cause = causation.detect_resource_changing_cause(
raw_event=raw_event,
resource=resource,
logger=logger,
patch=patch,
body=body,
old=old,
new=new,
diff=diff,
memo=memory.user_data,
initial=memory.noticed_by_listing
and not memory.fully_handled_once,
)
delay = await process_resource_changing_cause(
lifecycle=lifecycle,
registry=registry,
memory=memory,
cause=resource_changing_cause,
)
# Whatever was done, apply the accumulated changes to the object.
# But only once, to reduce the number of API calls and the generated irrelevant events.
if patch:
logger.debug("Patching with: %r", patch)
await patching.patch_obj(resource=resource, patch=patch, body=body)
# Sleep strictly after patching, never before -- to keep the status proper.
# The patching above, if done, interrupts the sleep instantly, so we skip it at all.
# Note: a zero-second or negative sleep is still a sleep, it will trigger a dummy patch.
if delay and patch:
logger.debug(
f"Sleeping was skipped because of the patch, {delay} seconds left."
)
elif delay is None and not patch:
logger.debug(
f"Handling cycle is finished, waiting for new changes since now.")
elif delay is not None:
if delay > 0:
logger.debug(
f"Sleeping for {delay} seconds for the delayed handlers.")
limited_delay = min(delay, handling.WAITING_KEEPALIVE_INTERVAL)
unslept_delay = await sleeping.sleep_or_wait(
limited_delay, replenished)
else:
unslept_delay = None # no need to sleep? means: slept in full.
if unslept_delay is not None:
logger.debug(
f"Sleeping was interrupted by new changes, {unslept_delay} seconds left."
)
else:
# Any unique always-changing value will work; not necessary a timestamp.
dummy_value = datetime.datetime.utcnow().isoformat()
dummy_patch = patches.Patch(
{'status': {
'kopf': {
'dummy': dummy_value
}
}})
logger.debug("Provoking reaction with: %r", dummy_patch)
await patching.patch_obj(resource=resource,
patch=dummy_patch,
body=body)