async def test_new_resources_and_namespaces_spawn_new_tasks( settings, ensemble: Ensemble, insights: Insights, peering_resource): settings.peering.namespaced = peering_resource.namespaced r1 = Resource(group='group1', version='version1', plural='plural1', namespaced=True) r2 = Resource(group='group2', version='version2', plural='plural2', namespaced=True) insights.resources.add(r1) insights.resources.add(r2) insights.namespaces.add('ns1') insights.namespaces.add('ns2') r1ns1 = EnsembleKey(resource=r1, namespace='ns1') r1ns2 = EnsembleKey(resource=r1, namespace='ns2') r2ns1 = EnsembleKey(resource=r2, namespace='ns1') r2ns2 = EnsembleKey(resource=r2, namespace='ns2') peer1 = EnsembleKey(resource=peering_resource, namespace='ns1') peer2 = EnsembleKey(resource=peering_resource, namespace='ns2') await adjust_tasks( processor=processor, identity=Identity('...'), settings=settings, insights=insights, ensemble=ensemble, ) assert set(ensemble.watcher_tasks) == {r1ns1, r1ns2, r2ns1, r2ns2} assert set(ensemble.peering_tasks) == {peer1, peer2} assert set(ensemble.pinging_tasks) == {peer1, peer2} assert set(ensemble.conflicts_found) == {peer1, peer2}
def test_url_for_a_list_of_clusterscoped_corev1_subresources_in_a_namespace(): resource = Resource('', 'v1', 'plural', namespaced=False) with pytest.raises(ValueError) as err: resource.get_url(namespace='ns', subresource='status') assert str(err.value) in { "Specific namespaces are not supported for cluster-scoped resources.", "Subresources can be used only with specific resources by their name.", }
async def insights(settings, resource): val_resource = Resource('admissionregistration.k8s.io', 'v1', 'validatingwebhookconfigurations') mut_resource = Resource('admissionregistration.k8s.io', 'v1', 'mutatingwebhookconfigurations') insights = Insights() insights.resources.add(resource) await insights.backbone.fill(resources=[val_resource, mut_resource]) insights.ready_resources.set() return insights
async def test_gone_resources_and_namespaces_stop_running_tasks( settings, ensemble: Ensemble, insights: Insights, peering_resource): settings.peering.namespaced = peering_resource.namespaced r1 = Resource(group='group1', version='version1', plural='plural1', namespaced=True) r2 = Resource(group='group2', version='version2', plural='plural2', namespaced=True) insights.watched_resources.add(r1) insights.watched_resources.add(r2) insights.namespaces.add('ns1') insights.namespaces.add('ns2') r1ns1 = EnsembleKey(resource=r1, namespace='ns1') r1ns2 = EnsembleKey(resource=r1, namespace='ns2') r2ns1 = EnsembleKey(resource=r2, namespace='ns1') r2ns2 = EnsembleKey(resource=r2, namespace='ns2') peerns = peering_resource.namespaced peer1 = EnsembleKey(resource=peering_resource, namespace='ns1' if peerns else None) await adjust_tasks( # initialisation processor=processor, identity=Identity('...'), settings=settings, insights=insights, ensemble=ensemble) r1ns2_task = ensemble.watcher_tasks[r1ns2] r2ns1_task = ensemble.watcher_tasks[r2ns1] r2ns2_task = ensemble.watcher_tasks[r2ns2] insights.watched_resources.discard(r2) insights.namespaces.discard('ns2') await adjust_tasks( # action-under-test processor=processor, identity=Identity('...'), settings=settings, insights=insights, ensemble=ensemble, ) assert set(ensemble.watcher_tasks) == {r1ns1} assert set(ensemble.peering_tasks) == {peer1} assert set(ensemble.pinging_tasks) == {peer1} assert set(ensemble.conflicts_found) == {peer1} assert r1ns2_task.cancelled() assert r2ns1_task.cancelled() assert r2ns2_task.cancelled()
def test_no_ambiguity_in_generic_selector(registry, decorator, caplog, assert_logs, insights): r1 = Resource(group='g1', version='v1', plural='plural', verbs=VERBS) r2 = Resource(group='g2', version='v2', plural='plural', verbs=VERBS) @decorator(EVERYTHING) def fn(**_): ... revise_resources(registry=registry, insights=insights, group=None, resources=[r1, r2]) assert insights.watched_resources == {r1, r2} assert_logs([], prohibited=[r"Ambiguous resources will not be served"])
async def test_cluster_tasks_continue_running_on_namespace_deletion( settings, ensemble: Ensemble, insights: Insights, cluster_peering_resource): settings.peering.namespaced = cluster_peering_resource.namespaced r1 = Resource(group='group1', version='version1', plural='plural1', namespaced=True) r2 = Resource(group='group2', version='version2', plural='plural2', namespaced=True) insights.watched_resources.add(r1) insights.watched_resources.add(r2) insights.namespaces.add(None) r1nsN = EnsembleKey(resource=r1, namespace=None) r2nsN = EnsembleKey(resource=r2, namespace=None) peerN = EnsembleKey(resource=cluster_peering_resource, namespace=None) await adjust_tasks( # initialisation processor=processor, identity=Identity('...'), settings=settings, insights=insights, ensemble=ensemble) r1nsN_task = ensemble.watcher_tasks[r1nsN] r2nsN_task = ensemble.watcher_tasks[r2nsN] insights.namespaces.discard(None) await adjust_tasks( # action-under-test processor=processor, identity=Identity('...'), settings=settings, insights=insights, ensemble=ensemble, ) assert set(ensemble.watcher_tasks) == {r1nsN, r2nsN} assert set(ensemble.peering_tasks) == {peerN} assert set(ensemble.pinging_tasks) == {peerN} assert set(ensemble.conflicts_found) == {peerN} assert not r1nsN_task.cancelled() assert not r2nsN_task.cancelled() assert not r1nsN_task.done() assert not r2nsN_task.done()
async def watch_objs( *, settings: configuration.OperatorSettings, resource: references.Resource, namespace: references.Namespace, timeout: Optional[float] = None, since: Optional[str] = None, context: Optional[auth.APIContext] = None, # injected by the decorator operator_pause_waiter: aiotasks.Future, ) -> AsyncIterator[bodies.RawInput]: """ Watch objects of a specific resource type. The cluster-scoped call is used in two cases: * The resource itself is cluster-scoped, and namespacing makes not sense. * The operator serves all namespaces for the namespaced custom resource. Otherwise, the namespace-scoped call is used: * The resource is namespace-scoped AND operator is namespaced-restricted. """ if context is None: raise RuntimeError("API instance is not injected by the decorator.") params: Dict[str, str] = {} params['watch'] = 'true' if since is not None: params['resourceVersion'] = since if timeout is not None: params['timeoutSeconds'] = str(timeout) # Stream the parsed events from the response until it is closed server-side, # or until it is closed client-side by the pause-waiting future's callbacks. try: response = await context.session.get( url=resource.get_url(server=context.server, namespace=namespace, params=params), timeout=aiohttp.ClientTimeout( total=settings.watching.client_timeout, sock_connect=settings.watching.connect_timeout, ), ) await errors.check_response(response) response_close_callback = lambda _: response.close() operator_pause_waiter.add_done_callback(response_close_callback) try: async with response: async for line in _iter_jsonlines(response.content): raw_input = cast(bodies.RawInput, json.loads(line.decode("utf-8"))) yield raw_input finally: operator_pause_waiter.remove_done_callback(response_close_callback) except (aiohttp.ClientConnectionError, aiohttp.ClientPayloadError, asyncio.TimeoutError): pass
async def create_obj( *, resource: references.Resource, namespace: references.Namespace = None, name: Optional[str] = None, body: Optional[bodies.RawBody] = None, context: Optional[auth.APIContext] = None, # injected by the decorator ) -> Optional[bodies.RawBody]: """ Create a resource. """ if context is None: raise RuntimeError("API instance is not injected by the decorator.") body = body if body is not None else {} if namespace is not None: body.setdefault('metadata', {}).setdefault('namespace', namespace) if name is not None: body.setdefault('metadata', {}).setdefault('name', name) namespace = cast(references.Namespace, body.get('metadata', {}).get('namespace')) response = await context.session.post( url=resource.get_url(server=context.server, namespace=namespace), json=body, ) created_body: bodies.RawBody = await errors.parse_response(response) return created_body
def test_creation_with_all_kwargs(): resource = Resource( group='group', version='version', plural='plural', kind='kind', singular='singular', shortcuts=['shortcut1', 'shortcut2'], categories=['category1', 'category2'], subresources=['sub1', 'sub2'], namespaced=True, preferred=True, verbs=['verb1', 'verb2'], ) assert resource.group == 'group' assert resource.version == 'version' assert resource.plural == 'plural' assert resource.kind == 'kind' assert resource.singular == 'singular' assert resource.shortcuts == ['shortcut1', 'shortcut2'] assert resource.categories == ['category1', 'category2'] assert resource.subresources == ['sub1', 'sub2'] assert resource.namespaced == True assert resource.preferred == True assert resource.verbs == ['verb1', 'verb2']
def resource(): return Resource( group='group1', version='version1', preferred=True, plural='plural1', singular='singular1', kind='kind1', shortcuts=['shortcut1', 'shortcut2'], categories=['category1', 'category2'], )
async def create_obj( *, settings: configuration.OperatorSettings, resource: references.Resource, namespace: references.Namespace = None, name: Optional[str] = None, body: Optional[bodies.RawBody] = None, logger: typedefs.Logger, ) -> Optional[bodies.RawBody]: """ Create a resource. """ body = body if body is not None else {} if namespace is not None: body.setdefault('metadata', {}).setdefault('namespace', namespace) if name is not None: body.setdefault('metadata', {}).setdefault('name', name) namespace = cast(references.Namespace, body.get('metadata', {}).get('namespace')) created_body: bodies.RawBody = await api.post( url=resource.get_url(namespace=namespace), payload=body, logger=logger, settings=settings, ) return created_body
async def test_followups_for_deletion_of_resource(settings, registry, apis_mock, group1_empty_mock, timer, etype, insights, insights_resources): e1 = RawEvent(type=etype, object=RawBody(spec={'group': 'group1'})) r1 = Resource(group='group1', version='version1', plural='plural1') insights_resources.add(r1) async def delayed_injection(delay: float): await asyncio.sleep(delay) await process_discovered_resource_event(insights=insights, raw_event=e1, registry=registry, settings=settings) task = asyncio.create_task(delayed_injection(0.1)) with timer: async with insights.revised: await insights.revised.wait() await task assert 0.1 < timer.seconds < 1.0 assert not insights_resources assert apis_mock.called assert group1_empty_mock.called
def test_ambiguity_in_specific_selectors(registry, decorator, caplog, assert_logs, insights): r1 = Resource(group='g1', version='v1', plural='plural', verbs=VERBS) r2 = Resource(group='g2', version='v2', plural='plural', verbs=VERBS) @decorator(plural='plural') def fn(**_): ... revise_resources(registry=registry, insights=insights, group=None, resources=[r1, r2]) assert not insights.watched_resources assert not insights.webhook_resources assert_logs([r"Ambiguous resources will not be served"])
async def test_waiting_for_preexisting_resources_ends_instantly(timer): resource = Resource('', 'v1', 'namespaces') backbone = Backbone() await backbone.fill(resources=[resource]) with timer: found_resource = await backbone.wait_for(NAMESPACES) assert timer.seconds < 0.1 assert found_resource == resource
def test_corev1_overrides_ambuigity(registry, decorator, caplog, assert_logs, insights): r1 = Resource(group='', version='v1', plural='pods', verbs=VERBS) r2 = Resource(group='metrics.k8s.io', version='v1', plural='pods', verbs=VERBS) @decorator(plural='pods') def fn(**_): ... revise_resources(registry=registry, insights=insights, group=None, resources=[r1, r2]) assert insights.watched_resources == {r1} assert_logs([], prohibited=[r"Ambiguous resources will not be served"])
def test_replacing_a_new_group(registry, insights, insights_resources): r1 = Resource(group='group1', version='version1', plural='plural1', verbs=VERBS) r2 = Resource(group='group2', version='version2', plural='plural2', verbs=VERBS) revise_resources(registry=registry, insights=insights, group=None, resources=[r1]) revise_resources(registry=registry, insights=insights, group='group2', resources=[r2]) assert insights_resources == {r1, r2}
def test_replacing_all_insights(registry): r1 = Resource(group='group1', version='version1', plural='plural1', verbs=VERBS) r2 = Resource(group='group2', version='version2', plural='plural2', verbs=VERBS) insights = Insights() revise_resources(registry=registry, insights=insights, group=None, resources=[r1]) revise_resources(registry=registry, insights=insights, group=None, resources=[r2]) assert insights.resources == {r2}
def test_initial_population(registry, insights, insights_resources): r1 = Resource(group='group1', version='version1', plural='plural1', verbs=VERBS) revise_resources(registry=registry, insights=insights, group=None, resources=[r1]) assert insights_resources == {r1}
def test_catchall_versions_are_ignored_for_nonpreferred_resources(): resource = Resource( group='group1', version='version1', preferred=False, plural='plural1', singular='singular1', kind='kind1', shortcuts=['shortcut1', 'shortcut2'], categories=['category1', 'category2'], ) selector = Selector(EVERYTHING) matches = selector.check(resource) assert not matches
def test_selectors_with_no_resources(registry, decorator, caplog, assert_logs, insights): r1 = Resource(group='group1', version='version1', plural='plural1', verbs=VERBS) r2 = Resource(group='group2', version='version2', plural='plural2', verbs=VERBS) @decorator(plural='plural3') def fn(**_): ... revise_resources(registry=registry, insights=insights, group=None, resources=[r1, r2]) assert not insights.watched_resources assert_logs([r"Unresolved resources cannot be served"])
def test_callback_is_not_called_with_mismatching_resource( match_fn, callback, handler, cause, ): cause = dataclasses.replace(cause, resource=Resource(group='x', version='y', plural='z')) result = match_fn(handler=handler, cause=cause) assert not result assert not callback.called
def test_irrelevant_resources_are_ignored(registry, resource, decorator): @decorator(*resource, registry=registry) def fn(**_): pass irrelevant_resource = Resource('grp', 'vers', 'plural') webhooks = build_webhooks(registry._webhooks.get_all_handlers(), resources=[irrelevant_resource], name_suffix='sfx', client_config={}) assert len(webhooks) == 1 assert len(webhooks[0]['rules']) == 0
async def watch_objs( *, settings: configuration.OperatorSettings, resource: references.Resource, namespace: references.Namespace, since: Optional[str] = None, operator_pause_waiter: aiotasks.Future, ) -> AsyncIterator[bodies.RawInput]: """ Watch objects of a specific resource type. The cluster-scoped call is used in two cases: * The resource itself is cluster-scoped, and namespacing makes not sense. * The operator serves all namespaces for the namespaced custom resource. Otherwise, the namespace-scoped call is used: * The resource is namespace-scoped AND operator is namespaced-restricted. """ params: Dict[str, str] = {} params['watch'] = 'true' if since is not None: params['resourceVersion'] = since if settings.watching.server_timeout is not None: params['timeoutSeconds'] = str(settings.watching.server_timeout) connect_timeout = (settings.watching.connect_timeout if settings.watching.connect_timeout is not None else settings.networking.connect_timeout if settings.networking.connect_timeout is not None else settings.networking.request_timeout) # Stream the parsed events from the response until it is closed server-side, # or until it is closed client-side by the pause-waiting future's callbacks. try: async for raw_input in api.stream( url=resource.get_url(namespace=namespace, params=params), logger=logger, settings=settings, stopper=operator_pause_waiter, timeout=aiohttp.ClientTimeout( total=settings.watching.client_timeout, sock_connect=connect_timeout, ), ): yield raw_input except (aiohttp.ClientConnectionError, aiohttp.ClientPayloadError, asyncio.TimeoutError): pass
def test_labels_specific_filter(registry, resource, decorator, label_value, exp_expr): @decorator(*resource, registry=registry, labels={'lbl': label_value}) def fn(**_): pass irrelevant_resource = Resource('grp', 'vers', 'plural') webhooks = build_webhooks(registry._webhooks.get_all_handlers(), resources=[irrelevant_resource], name_suffix='sfx', client_config={}) assert len(webhooks) == 1 assert webhooks[0]['objectSelector'] == {'matchExpressions': [exp_expr]}
async def test_waiting_for_delayed_resources_ends_once_delivered(timer): resource = Resource('', 'v1', 'namespaces') backbone = Backbone() async def delayed_injection(delay: float): await asyncio.sleep(delay) await backbone.fill(resources=[resource]) task = asyncio.create_task(delayed_injection(0.1)) with timer: found_resource = await backbone.wait_for(NAMESPACES) await task assert 0.1 < timer.seconds < 0.11 assert found_resource == resource
def test_labels_callable_filter(registry, resource, decorator): @decorator(*resource, registry=registry, labels={'lbl': lambda *_, **__: None}) def fn(**_): pass irrelevant_resource = Resource('grp', 'vers', 'plural') webhooks = build_webhooks( registry._webhooks.get_all_handlers(), resources=[irrelevant_resource], name_suffix='sfx', client_config={}) assert len(webhooks) == 1 assert webhooks[0]['objectSelector'] is None
def test_indexed_resources_are_duplicated_in_watched_resources( registry, decorator, insights): r1 = Resource(group='group1', version='version1', plural='plural1', verbs=VERBS) @decorator('group1', 'version1', 'plural1') def fn(**_): ... revise_resources(registry=registry, insights=insights, group=None, resources=[r1]) assert insights.watched_resources assert insights.indexed_resources assert not insights.webhook_resources
def test_nonpatchable_excluded(registry, decorator, caplog, assert_logs, insights): r1 = Resource(group='group1', version='version1', plural='plural1', verbs=['watch', 'list']) @decorator('group1', 'version1', 'plural1') # because it patches! def fn(**_): ... revise_resources(registry=registry, insights=insights, group=None, resources=[r1]) assert not insights.watched_resources assert_logs([ r"Non-patchable resources will not be served: {plural1.version1.group1}" ])
def test_nonwatchable_excluded(registry, decorator, caplog, assert_logs): r1 = Resource(group='group1', version='version1', plural='plural1', verbs=[]) @decorator('group1', 'version1', 'plural1') def fn(**_): ... insights = Insights() revise_resources(registry=registry, insights=insights, group=None, resources=[r1]) assert not insights.resources assert_logs([ r"Non-watchable resources will not be served: {plural1.version1.group1}" ])
async def test_followups_for_addition(registry, apis_mock, group1_mock, timer, etype): e1 = RawEvent(type=etype, object=RawBody(spec={'group': 'group1'})) r1 = Resource(group='group1', version='version1', plural='plural1') insights = Insights() async def delayed_injection(delay: float): await asyncio.sleep(delay) await process_discovered_resource_event(insights=insights, raw_event=e1, registry=registry) task = asyncio.create_task(delayed_injection(0.1)) async with timer, async_timeout.timeout(1.0): async with insights.revised: await insights.revised.wait() await task assert 0.1 < timer.seconds < 1.0 assert insights.resources == {r1} assert apis_mock.called assert group1_mock.called