def detect_resource_changing_cause(
*,
raw_event: bodies.RawEvent,
body: bodies.Body,
old: Optional[bodies.BodyEssence] = None,
new: Optional[bodies.BodyEssence] = None,
diff: Optional[diffs.Diff] = None,
initial: bool = False,
**kwargs: Any,
) -> ResourceChangingCause:
"""
Detect the cause of the event to be handled.
This is a purely computational function with no side-effects.
The causes are then consumed by `custom_object_handler`,
which performs the actual handler invocation, logging, patching,
and other side-effects.
"""
# Put them back to the pass-through kwargs (to avoid code duplication).
kwargs.update(body=body, old=old, new=new, initial=initial)
if diff is not None:
kwargs.update(diff=diff)
# The object was really deleted from the cluster. But we do not care anymore.
if raw_event['type'] == 'DELETED':
return ResourceChangingCause(reason=handlers.Reason.GONE, **kwargs)
# The finalizer has been just removed. We are fully done.
if finalizers.is_deletion_ongoing(
body) and not finalizers.is_deletion_blocked(body):
return ResourceChangingCause(reason=handlers.Reason.FREE, **kwargs)
if finalizers.is_deletion_ongoing(body):
return ResourceChangingCause(reason=handlers.Reason.DELETE, **kwargs)
# For an object seen for the first time (i.e. just-created), call the creation handlers,
# then mark the state as if it was seen when the creation has finished.
# Creation never mixes with resuming, even if an object is detected on startup (first listing).
if old is None: # i.e. we have no essence stored
kwargs['initial'] = False
return ResourceChangingCause(reason=handlers.Reason.CREATE, **kwargs)
# Cases with no essence changes are usually ignored (NOOP). But for the not-yet-resumed objects,
# we simulate a fake cause to invoke the resuming handlers. For cases with the essence changes,
# the resuming handlers will be mixed-in to the regular cause handling ("cuckoo-style")
# due to the ``initial=True`` flag on the cause, regardless of the reason.
if not diff and initial:
return ResourceChangingCause(reason=handlers.Reason.RESUME, **kwargs)
# The previous step triggers one more patch operation without actual changes. Ignore it.
# Either the last-seen state or the status field has changed.
if not diff:
return ResourceChangingCause(reason=handlers.Reason.NOOP, **kwargs)
# And what is left, is the update operation on one of the useful fields of the existing object.
return ResourceChangingCause(reason=handlers.Reason.UPDATE, **kwargs)
async def process_resource_spawning_cause(
registry: registries.OperatorRegistry,
settings: configuration.OperatorSettings,
memory: containers.ResourceMemory,
cause: causation.ResourceSpawningCause,
) -> Collection[float]:
"""
Spawn/kill all the background tasks of a resource.
The spawning and killing happens in parallel with the resource-changing
handlers invocation (even if it takes few cycles). For this, the signal
to terminate is sent to the daemons immediately, but the actual check
of their shutdown is performed only when all the on-deletion handlers
have succeeded (or after they were invoked if they are optional;
or immediately if there were no on-deletion handlers to invoke at all).
The resource remains blocked by the finalizers until all the daemons exit
(except those marked as tolerating being orphaned).
"""
# Refresh the up-to-date body & essential timestamp for all the daemons/timers.
if memory.live_fresh_body is None:
memory.live_fresh_body = cause.body
if cause.reset:
memory.idle_reset_time = time.monotonic()
if finalizers.is_deletion_ongoing(cause.body):
stopping_delays = await daemons.stop_resource_daemons(
settings=settings,
daemons=memory.daemons,
)
return stopping_delays
else:
handlers = registry.resource_spawning_handlers[
cause.resource].get_handlers(
cause=cause,
excluded=memory.forever_stopped,
)
spawning_delays = await daemons.spawn_resource_daemons(
settings=settings,
daemons=memory.daemons,
cause=cause,
memory=memory,
handlers=handlers,
)
matching_delays = await daemons.match_resource_daemons(
settings=settings,
daemons=memory.daemons,
handlers=handlers,
)
return list(spawning_delays) + list(matching_delays)
def deleted(self) -> bool:
""" Used to conditionally skip/select the @on.resume handlers if the object is deleted. """
return finalizers.is_deletion_ongoing(self.body)
def test_is_deleted(expected, body):
result = is_deletion_ongoing(body=body)
assert result == expected
async def process_resource_causes(
lifecycle: lifecycles.LifeCycleFn,
indexers: indexing.OperatorIndexers,
registry: registries.OperatorRegistry,
settings: configuration.OperatorSettings,
resource: references.Resource,
raw_event: bodies.RawEvent,
body: bodies.Body,
patch: patches.Patch,
logger: loggers.ObjectLogger,
memory: containers.ResourceMemory,
) -> Tuple[Collection[float], bool]:
finalizer = settings.persistence.finalizer
extra_fields = (
# NB: indexing handlers are useless here, they are handled on their own.
registry._resource_watching.get_extra_fields(resource=resource)
| registry._resource_changing.get_extra_fields(resource=resource)
| registry._resource_spawning.get_extra_fields(resource=resource))
old = settings.persistence.diffbase_storage.fetch(body=body)
new = settings.persistence.diffbase_storage.build(
body=body, extra_fields=extra_fields)
old = settings.persistence.progress_storage.clear(
essence=old) if old is not None else None
new = settings.persistence.progress_storage.clear(
essence=new) if new is not None else None
diff = diffs.diff(old, new)
# Detect what are we going to do on this processing cycle.
resource_watching_cause = causation.detect_resource_watching_cause(
raw_event=raw_event,
resource=resource,
indices=indexers.indices,
logger=logger,
patch=patch,
body=body,
memo=memory.memo,
) if registry._resource_watching.has_handlers(resource=resource) else None
resource_spawning_cause = causation.detect_resource_spawning_cause(
resource=resource,
indices=indexers.indices,
logger=logger,
patch=patch,
body=body,
memo=memory.memo,
reset=bool(
diff), # only essential changes reset idling, not every event
) if registry._resource_spawning.has_handlers(resource=resource) else None
resource_changing_cause = causation.detect_resource_changing_cause(
finalizer=finalizer,
raw_event=raw_event,
resource=resource,
indices=indexers.indices,
logger=logger,
patch=patch,
body=body,
old=old,
new=new,
diff=diff,
memo=memory.memo,
initial=memory.noticed_by_listing and not memory.fully_handled_once,
) if registry._resource_changing.has_handlers(resource=resource) else None
# If there are any handlers for this resource kind in general, but not for this specific object
# due to filters, then be blind to it, store no state, and log nothing about the handling cycle.
if (resource_changing_cause is not None
and not registry._resource_changing.prematch(
cause=resource_changing_cause)):
resource_changing_cause = None
# Block the object from deletion if we have anything to do in its end of life:
# specifically, if there are daemons to kill or mandatory on-deletion handlers to call.
# The high-level handlers are prevented if this event cycle is dedicated to the finalizer.
# The low-level handlers (on-event spying & daemon spawning) are still executed asap.
deletion_is_ongoing = finalizers.is_deletion_ongoing(body=body)
deletion_is_blocked = finalizers.is_deletion_blocked(body=body,
finalizer=finalizer)
deletion_must_be_blocked = (
(resource_spawning_cause is not None
and registry._resource_spawning.requires_finalizer(
cause=resource_spawning_cause,
excluded=memory.forever_stopped,
)) or (resource_changing_cause is not None
and registry._resource_changing.requires_finalizer(
cause=resource_changing_cause, )))
if deletion_must_be_blocked and not deletion_is_blocked and not deletion_is_ongoing:
logger.debug(
"Adding the finalizer, thus preventing the actual deletion.")
finalizers.block_deletion(body=body, patch=patch, finalizer=finalizer)
resource_changing_cause = None # prevent further high-level processing this time
if not deletion_must_be_blocked and deletion_is_blocked:
logger.debug(
"Removing the finalizer, as there are no handlers requiring it.")
finalizers.allow_deletion(body=body, patch=patch, finalizer=finalizer)
resource_changing_cause = None # prevent further high-level processing this time
# Invoke all the handlers that should or could be invoked at this processing cycle.
# The low-level spies go ASAP always. However, the daemons are spawned before the high-level
# handlers and killed after them: the daemons should live throughout the full object lifecycle.
if resource_watching_cause is not None:
await process_resource_watching_cause(
lifecycle=lifecycles.all_at_once,
registry=registry,
settings=settings,
cause=resource_watching_cause,
)
resource_spawning_delays: Collection[float] = []
if resource_spawning_cause is not None:
resource_spawning_delays = await process_resource_spawning_cause(
registry=registry,
settings=settings,
memory=memory,
cause=resource_spawning_cause,
)
resource_changing_delays: Collection[float] = []
if resource_changing_cause is not None:
resource_changing_delays = await process_resource_changing_cause(
lifecycle=lifecycle,
registry=registry,
settings=settings,
memory=memory,
cause=resource_changing_cause,
)
# Release the object if everything is done, and it is marked for deletion.
# But not when it has already gone.
if deletion_is_ongoing and deletion_is_blocked \
and not resource_spawning_delays \
and not resource_changing_delays:
logger.debug(
"Removing the finalizer, thus allowing the actual deletion.")
finalizers.allow_deletion(body=body, patch=patch, finalizer=finalizer)
delays = list(resource_spawning_delays) + list(resource_changing_delays)
return (delays, resource_changing_cause is not None)
async def process_resource_event(
lifecycle: lifecycles.LifeCycleFn,
registry: registries.OperatorRegistry,
settings: configuration.OperatorSettings,
memories: containers.ResourceMemories,
resource: resources.Resource,
raw_event: bodies.RawEvent,
replenished: asyncio.Event,
event_queue: posting.K8sEventQueue,
) -> None:
"""
Handle a single custom object low-level watch-event.
Convert the low-level events, as provided by the watching/queueing tasks,
to the high-level causes, and then call the cause-handling logic.
All the internally provoked changes are intercepted, do not create causes,
and therefore do not call the handling logic.
"""
finalizer = settings.persistence.finalizer
# Recall what is stored about that object. Share it in little portions with the consumers.
# And immediately forget it if the object is deleted from the cluster (but keep in memory).
raw_type, raw_body = raw_event['type'], raw_event['object']
memory = await memories.recall(raw_body,
noticed_by_listing=raw_type is None)
if memory.live_fresh_body is not None:
memory.live_fresh_body._replace_with(raw_body)
if raw_type == 'DELETED':
await memories.forget(raw_body)
# Convert to a heavy mapping-view wrapper only now, when heavy processing begins.
# Raw-event streaming, queueing, and batching use regular lightweight dicts.
# Why here? 1. Before it splits into multiple causes & handlers for the same object's body;
# 2. After it is batched (queueing); 3. While the "raw" parsed JSON is still known;
# 4. Same as where a patch object of a similar wrapping semantics is created.
body = memory.live_fresh_body if memory.live_fresh_body is not None else bodies.Body(
raw_body)
patch = patches.Patch()
# Each object has its own prefixed logger, to distinguish parallel handling.
logger = logging_engine.ObjectLogger(body=body, settings=settings)
posting.event_queue_loop_var.set(asyncio.get_running_loop())
posting.event_queue_var.set(
event_queue) # till the end of this object's task.
extra_fields = registry.resource_changing_handlers[
resource].get_extra_fields()
old = settings.persistence.diffbase_storage.fetch(body=body)
new = settings.persistence.diffbase_storage.build(
body=body, extra_fields=extra_fields)
old = settings.persistence.progress_storage.clear(
essence=old) if old is not None else None
new = settings.persistence.progress_storage.clear(
essence=new) if new is not None else None
diff = diffs.diff(old, new)
# Detect what are we going to do on this processing cycle.
resource_watching_cause = causation.detect_resource_watching_cause(
raw_event=raw_event,
resource=resource,
logger=logger,
patch=patch,
body=body,
memo=memory.memo,
) if registry.resource_watching_handlers[resource] else None
resource_spawning_cause = causation.detect_resource_spawning_cause(
resource=resource,
logger=logger,
patch=patch,
body=body,
memo=memory.memo,
reset=bool(
diff), # only essential changes reset idling, not every event
) if registry.resource_spawning_handlers[resource] else None
resource_changing_cause = causation.detect_resource_changing_cause(
finalizer=finalizer,
raw_event=raw_event,
resource=resource,
logger=logger,
patch=patch,
body=body,
old=old,
new=new,
diff=diff,
memo=memory.memo,
initial=memory.noticed_by_listing and not memory.fully_handled_once,
) if registry.resource_changing_handlers[resource] else None
# Block the object from deletion if we have anything to do in its end of life:
# specifically, if there are daemons to kill or mandatory on-deletion handlers to call.
# The high-level handlers are prevented if this event cycle is dedicated to the finalizer.
# The low-level handlers (on-event spying & daemon spawning) are still executed asap.
deletion_is_ongoing = finalizers.is_deletion_ongoing(body=body)
deletion_is_blocked = finalizers.is_deletion_blocked(body=body,
finalizer=finalizer)
deletion_must_be_blocked = (
(resource_spawning_cause is not None
and registry.resource_spawning_handlers[resource].requires_finalizer(
cause=resource_spawning_cause,
excluded=memory.forever_stopped,
)) or
(resource_changing_cause is not None
and registry.resource_changing_handlers[resource].requires_finalizer(
cause=resource_changing_cause, )))
if deletion_must_be_blocked and not deletion_is_blocked and not deletion_is_ongoing:
logger.debug(
"Adding the finalizer, thus preventing the actual deletion.")
finalizers.block_deletion(body=body, patch=patch, finalizer=finalizer)
resource_changing_cause = None # prevent further high-level processing this time
if not deletion_must_be_blocked and deletion_is_blocked:
logger.debug(
"Removing the finalizer, as there are no handlers requiring it.")
finalizers.allow_deletion(body=body, patch=patch, finalizer=finalizer)
resource_changing_cause = None # prevent further high-level processing this time
# Invoke all the handlers that should or could be invoked at this processing cycle.
# The low-level spies go ASAP always. However, the daemons are spawned before the high-level
# handlers and killed after them: the daemons should live throughout the full object lifecycle.
if resource_watching_cause is not None:
await process_resource_watching_cause(
lifecycle=lifecycles.all_at_once,
registry=registry,
settings=settings,
cause=resource_watching_cause,
)
resource_spawning_delays: Collection[float] = []
if resource_spawning_cause is not None:
resource_spawning_delays = await process_resource_spawning_cause(
registry=registry,
settings=settings,
memory=memory,
cause=resource_spawning_cause,
)
resource_changing_delays: Collection[float] = []
if resource_changing_cause is not None:
resource_changing_delays = await process_resource_changing_cause(
lifecycle=lifecycle,
registry=registry,
settings=settings,
memory=memory,
cause=resource_changing_cause,
)
# Release the object if everything is done, and it is marked for deletion.
# But not when it has already gone.
if deletion_is_ongoing and deletion_is_blocked \
and not resource_spawning_delays \
and not resource_changing_delays:
logger.debug(
"Removing the finalizer, thus allowing the actual deletion.")
finalizers.allow_deletion(body=body, patch=patch, finalizer=finalizer)
# Whatever was done, apply the accumulated changes to the object, or sleep-n-touch for delays.
# But only once, to reduce the number of API calls and the generated irrelevant events.
# And only if the object is at least supposed to exist (not "GONE"), even if actually does not.
if raw_event['type'] != 'DELETED':
await apply_reaction_outcomes(
settings=settings,
resource=resource,
body=body,
patch=patch,
logger=logger,
delays=list(resource_spawning_delays) +
list(resource_changing_delays),
replenished=replenished,
)
