def test_custom_mappings_are_recursed():
class SampleMapping(collections.abc.Mapping):
def __init__(self, data=(), **kwargs) -> None:
super().__init__()
self._items = dict(data, **kwargs)
def __len__(self) -> int:
return len(self._items)
def __iter__(self):
return iter(self._items)
def __getitem__(self, item: str) -> str:
return self._items[item]
class MappingA(SampleMapping):
pass
class MappingB(SampleMapping):
pass
a = MappingA(a=100, b=200)
b = MappingB(b=300, c=400)
d = diff(a, b)
assert (DiffOperation.REMOVE, ('a', ), 100, None) in d
assert (DiffOperation.CHANGE, ('b', ), 200, 300) in d
assert (DiffOperation.ADD, ('c', ), None, 400) in d
assert (DiffOperation.CHANGE, (), a, b) not in d
def test_dicts_different_items_handled():
body_before_storage_size_update = {'spec': {'items': ['task1', 'task2']}}
body_after_storage_size_update = {'spec': {'items': ['task3', 'task4']}}
d = diff(body_before_storage_size_update, body_after_storage_size_update)
assert d == (('change', ('spec', 'items'), ['task1',
'task2'], ['task3',
'task4']), )
def test_admission_kwargs(resource, attr):
body = {'metadata': {'uid': 'uid1', 'name': 'name1', 'namespace': 'ns1',
'labels': {'l1': 'v1'}, 'annotations': {'a1': 'v1'}},
'spec': {'field': 'value'},
'status': {'info': 'payload'}}
cause = WebhookCause(
logger=logging.getLogger('kopf.test.fake.logger'),
indices=OperatorIndexers().indices,
resource=resource,
patch=Patch(),
memo=Memo(),
body=Body(body),
dryrun=False,
headers={'k1': 'v1'},
sslpeer={'k2': 'v2'},
userinfo={'k3': 'v3'},
warnings=['w1'],
webhook=None,
reason=None,
operation=None,
subresource=None,
new=BodyEssence(body),
old=None,
diff=diffs.diff(BodyEssence(body), None),
)
kwargs = getattr(cause, attr) # cause.kwargs / cause.sync_kwargs / cause.async_kwargs
assert set(kwargs) == {'logger', 'resource',
'dryrun', 'headers', 'sslpeer', 'userinfo', 'warnings', 'subresource',
'patch', 'memo',
'body', 'spec', 'status', 'meta', 'uid', 'name', 'namespace',
'labels', 'annotations', 'old', 'new', 'diff', 'operation'}
assert kwargs['resource'] is cause.resource
assert kwargs['logger'] is cause.logger
assert kwargs['dryrun'] is cause.dryrun
assert kwargs['headers'] is cause.headers
assert kwargs['sslpeer'] is cause.sslpeer
assert kwargs['userinfo'] is cause.userinfo
assert kwargs['warnings'] is cause.warnings
assert kwargs['patch'] is cause.patch
assert kwargs['memo'] is cause.memo
assert kwargs['body'] is cause.body
assert kwargs['spec'] is cause.body.spec
assert kwargs['meta'] is cause.body.metadata
assert kwargs['status'] is cause.body.status
assert kwargs['labels'] is cause.body.metadata.labels
assert kwargs['annotations'] is cause.body.metadata.annotations
assert kwargs['uid'] == cause.body.metadata.uid
assert kwargs['name'] == cause.body.metadata.name
assert kwargs['namespace'] == cause.body.metadata.namespace
assert kwargs['operation'] == cause.operation
assert kwargs['new'] == cause.new
assert kwargs['old'] == cause.old
assert kwargs['diff'] == cause.diff
async def patch_and_check(
*,
settings: configuration.OperatorSettings,
resource: references.Resource,
body: bodies.Body,
patch: patches.Patch,
logger: typedefs.Logger,
) -> None:
"""
Apply a patch and verify that it is applied correctly.
The inconsistencies are checked only against what was in the patch.
Other unexpected changes in the body are ignored, including the system
fields, such as generations, resource versions, and other unrelated fields,
such as other statuses, spec, labels, annotations, etc.
Selected false-positive inconsistencies are explicitly ignored
for K8s-managed fields, such as finalizers, labels or annotations:
whenever an empty list/dict is stored, such fields are completely removed.
For normal fields (e.g. in spec/status), an empty list/dict is still
a value and is persisted in the object and matches with the patch.
"""
if patch:
logger.debug(f"Patching with: {patch!r}")
resulting_body = await patching.patch_obj(
settings=settings,
resource=resource,
namespace=body.metadata.namespace,
name=body.metadata.name,
patch=patch,
logger=logger,
)
inconsistencies = diffs.diff(patch,
resulting_body,
scope=diffs.DiffScope.LEFT)
inconsistencies = diffs.Diff(
diffs.DiffItem(op, field, old, new)
for op, field, old, new in inconsistencies
if old or new or field not in KNOWN_INCONSISTENCIES)
if resulting_body is None:
logger.debug(
f"Patching was skipped: the object does not exist anymore.")
elif inconsistencies:
logger.warning(
f"Patching failed with inconsistencies: {inconsistencies}")
def test_scalars_unequal(scope):
a = 100
b = 200
d = diff(a, b, scope=scope)
assert d == (('change', (), 100, 200), )
def test_dicts_with_keys_added_and_noticed(scope):
a = {'hello': 'world'}
b = {'hello': 'world', 'key': 'val'}
d = diff(a, b, scope=scope)
assert d == (('add', ('key', ), None, 'val'), )
def test_dicts_with_keys_added_but_ignored(scope):
a = {'hello': 'world'}
b = {'hello': 'world', 'key': 'val'}
d = diff(a, b, scope=scope)
assert d == ()
def test_none_for_old(scope):
a = None
b = object()
d = diff(a, b, scope=scope)
assert d == (('add', (), None, b), )
def test_dicts_equal(scope):
a = {'hello': 'world', 'key': 'val'}
b = {'key': 'val', 'hello': 'world'}
d = diff(a, b, scope=scope)
assert d == ()
def test_lists_equal(scope):
a = [100, 200, 300]
b = [100, 200, 300]
d = diff(a, b, scope=scope)
assert d == ()
def test_lists_unequal(scope):
a = [100, 200, 300]
b = [100, 666, 300]
d = diff(a, b, scope=scope)
assert d == (('change', (), [100, 200, 300], [100, 666, 300]), )
def test_dicts_with_keys_removed_and_noticed(scope):
a = {'hello': 'world', 'key': 'val'}
b = {'hello': 'world'}
d = diff(a, b, scope=scope)
assert d == (('remove', ('key', ), 'val', None), )
def test_strings_unequal(scope):
a = 'hello'
b = 'world'
d = diff(a, b, scope=scope)
assert d == (('change', (), 'hello', 'world'), )
def test_scalars_equal(scope):
a = 100
b = 100
d = diff(a, b, scope=scope)
assert d == ()
async def serve_admission_request(
# Required for all webhook servers, meaningless without it:
request: reviews.Request,
*,
# Optional for webhook servers that can recognise this information:
headers: Optional[Mapping[str, str]] = None,
sslpeer: Optional[Mapping[str, Any]] = None,
webhook: Optional[ids.HandlerId] = None,
reason: Optional[
causes.
WebhookType] = None, # TODO: undocumented: requires typing clarity!
# Injected by partial() from spawn_tasks():
settings: configuration.OperatorSettings,
memories: MemoGetter,
memobase: ephemera.AnyMemo,
registry: registries.OperatorRegistry,
insights: references.Insights,
indices: ephemera.Indices,
) -> reviews.Response:
"""
The actual and the only implementation of the `WebhookFn` protocol.
This function is passed to all webhook servers/tunnels to be called
whenever a new admission request is received.
Some parameters are provided by the framework itself via partial binding,
so that the resulting function matches the `WebhookFn` protocol. Other
parameters are passed by the webhook servers when they call the function.
"""
# Reconstruct the cause specially for web handlers.
resource = find_resource(request=request, insights=insights)
subresource = request.get('request', {}).get('subResource')
operation = request.get('request', {}).get('operation')
userinfo = request.get('request', {}).get('userInfo')
new_body = request.get('request', {}).get('object')
old_body = request.get('request', {}).get('oldObject')
raw_body = new_body if new_body is not None else old_body
if userinfo is None:
raise MissingDataError(
"User info is missing from the admission request.")
if raw_body is None:
raise MissingDataError(
"Either old or new object is missing from the admission request.")
memo = await memories.recall_memo(raw_body,
memobase=memobase,
ephemeral=operation == 'CREATE')
body = bodies.Body(raw_body)
old = bodies.Body(old_body) if old_body is not None else None
new = bodies.Body(new_body) if new_body is not None else None
diff = diffs.diff(old, new)
patch = patches.Patch(body=raw_body)
warnings: List[str] = []
cause = causes.WebhookCause(
resource=resource,
indices=indices,
logger=loggers.LocalObjectLogger(body=body, settings=settings),
patch=patch,
memo=memo,
body=body,
userinfo=userinfo,
warnings=warnings,
operation=operation,
subresource=subresource,
dryrun=bool(request.get('request', {}).get('dryRun')),
sslpeer=sslpeer if sslpeer is not None else
{}, # ensure a mapping even if not provided.
headers=headers if headers is not None else
{}, # ensure a mapping even if not provided.
webhook=webhook,
reason=reason,
old=old,
new=new,
diff=diff,
)
# Retrieve the handlers to be executed; maybe only one if the webhook server provides a hint.
handlers_ = registry._webhooks.get_handlers(cause)
state = progression.State.from_scratch().with_handlers(handlers_)
outcomes = await execution.execute_handlers_once(
lifecycle=lifecycles.all_at_once,
settings=settings,
handlers=handlers_,
cause=cause,
state=state,
default_errors=execution.ErrorsMode.PERMANENT,
)
# Construct the response as per Kubernetes's conventions and expectations.
response = build_response(
request=request,
outcomes=outcomes,
warnings=warnings,
jsonpatch=patch.as_json_patch(),
)
return response
def test_nones_for_both(scope):
a = None
b = None
d = diff(a, b, scope=scope)
assert d == ()
def test_dicts_with_subkeys_changed(scope):
a = {'main': {'hello': 'world', 'key': 'old'}}
b = {'main': {'hello': 'world', 'key': 'new'}}
d = diff(a, b, scope=scope)
assert d == (('change', ('main', 'key'), 'old', 'new'), )
def test_dicts_updating_storage_size():
body_before_storage_size_update = {'spec': {'size': '42G'}}
body_after_storage_size_update = {'spec': {'size': '76G'}}
d = diff(body_before_storage_size_update, body_after_storage_size_update)
assert d == (('change', ('spec', 'size'), '42G', '76G'), )
def test_dicts_adding_label():
body_before_labelling = {'metadata': {}}
body_after_labelling = {'metadata': {'labels': 'LABEL'}}
d = diff(body_before_labelling, body_after_labelling)
assert d == (('add', ('metadata', 'labels'), None, 'LABEL'), )
def test_none_for_new(scope):
a = object()
b = None
d = diff(a, b, scope=scope)
assert d == (('remove', (), a, None), )
async def process_resource_causes(
lifecycle: execution.LifeCycleFn,
indexers: indexing.OperatorIndexers,
registry: registries.OperatorRegistry,
settings: configuration.OperatorSettings,
resource: references.Resource,
raw_event: bodies.RawEvent,
body: bodies.Body,
patch: patches.Patch,
memory: inventory.ResourceMemory,
local_logger: loggers.ObjectLogger,
event_logger: loggers.ObjectLogger,
) -> Tuple[Collection[float], bool]:
finalizer = settings.persistence.finalizer
extra_fields = (
# NB: indexing handlers are useless here, they are handled on their own.
registry._watching.get_extra_fields(resource=resource) |
registry._changing.get_extra_fields(resource=resource) |
registry._spawning.get_extra_fields(resource=resource))
old = settings.persistence.diffbase_storage.fetch(body=body)
new = settings.persistence.diffbase_storage.build(body=body, extra_fields=extra_fields)
old = settings.persistence.progress_storage.clear(essence=old) if old is not None else None
new = settings.persistence.progress_storage.clear(essence=new) if new is not None else None
diff = diffs.diff(old, new)
# Detect what are we going to do on this processing cycle.
watching_cause = causes.detect_watching_cause(
raw_event=raw_event,
resource=resource,
indices=indexers.indices,
logger=local_logger,
patch=patch,
body=body,
memo=memory.memo,
) if registry._watching.has_handlers(resource=resource) else None
spawning_cause = causes.detect_spawning_cause(
resource=resource,
indices=indexers.indices,
logger=event_logger,
patch=patch,
body=body,
memo=memory.memo,
reset=bool(diff), # only essential changes reset idling, not every event
) if registry._spawning.has_handlers(resource=resource) else None
changing_cause = causes.detect_changing_cause(
finalizer=finalizer,
raw_event=raw_event,
resource=resource,
indices=indexers.indices,
logger=event_logger,
patch=patch,
body=body,
old=old,
new=new,
diff=diff,
memo=memory.memo,
initial=memory.noticed_by_listing and not memory.fully_handled_once,
) if registry._changing.has_handlers(resource=resource) else None
# If there are any handlers for this resource kind in general, but not for this specific object
# due to filters, then be blind to it, store no state, and log nothing about the handling cycle.
if changing_cause is not None and not registry._changing.prematch(cause=changing_cause):
changing_cause = None
# Block the object from deletion if we have anything to do in its end of life:
# specifically, if there are daemons to kill or mandatory on-deletion handlers to call.
# The high-level handlers are prevented if this event cycle is dedicated to the finalizer.
# The low-level handlers (on-event spying & daemon spawning) are still executed asap.
deletion_is_ongoing = finalizers.is_deletion_ongoing(body=body)
deletion_is_blocked = finalizers.is_deletion_blocked(body=body, finalizer=finalizer)
deletion_must_be_blocked = (
(spawning_cause is not None and
registry._spawning.requires_finalizer(
cause=spawning_cause,
excluded=memory.daemons_memory.forever_stopped,
))
or
(changing_cause is not None and
registry._changing.requires_finalizer(
cause=changing_cause,
)))
if deletion_must_be_blocked and not deletion_is_blocked and not deletion_is_ongoing:
local_logger.debug("Adding the finalizer, thus preventing the actual deletion.")
finalizers.block_deletion(body=body, patch=patch, finalizer=finalizer)
changing_cause = None # prevent further high-level processing this time
if not deletion_must_be_blocked and deletion_is_blocked:
local_logger.debug("Removing the finalizer, as there are no handlers requiring it.")
finalizers.allow_deletion(body=body, patch=patch, finalizer=finalizer)
changing_cause = None # prevent further high-level processing this time
# Invoke all the handlers that should or could be invoked at this processing cycle.
# The low-level spies go ASAP always. However, the daemons are spawned before the high-level
# handlers and killed after them: the daemons should live throughout the full object lifecycle.
if watching_cause is not None:
await process_watching_cause(
lifecycle=lifecycles.all_at_once,
registry=registry,
settings=settings,
cause=watching_cause,
)
spawning_delays: Collection[float] = []
if spawning_cause is not None:
spawning_delays = await process_spawning_cause(
registry=registry,
settings=settings,
memory=memory,
cause=spawning_cause,
)
changing_delays: Collection[float] = []
if changing_cause is not None:
changing_delays = await process_changing_cause(
lifecycle=lifecycle,
registry=registry,
settings=settings,
memory=memory,
cause=changing_cause,
)
# Release the object if everything is done, and it is marked for deletion.
# But not when it has already gone.
if deletion_is_ongoing and deletion_is_blocked and not spawning_delays and not changing_delays:
local_logger.debug("Removing the finalizer, thus allowing the actual deletion.")
finalizers.allow_deletion(body=body, patch=patch, finalizer=finalizer)
delays = list(spawning_delays) + list(changing_delays)
return (delays, changing_cause is not None)
def test_strings_equal(scope):
a = 'hello'
b = 'hello'
d = diff(a, b, scope=scope)
assert d == ()
