async def test_other_peering_objects_are_ignored(mocker, k8s_mocked, settings, replenished, peering_resource, peering_namespace): status = mocker.Mock() status.items.side_effect = Exception("This should not be called.") event = bodies.RawEvent( type='ADDED', # irrelevant object={ 'metadata': { 'name': 'their-name' }, 'status': status, }) wait_for = mocker.patch('asyncio.wait_for') settings.peering.name = 'our-name' await process_peering_event( raw_event=event, freeze_toggle=primitives.Toggle(), replenished=replenished, autoclean=False, identity='id', settings=settings, namespace=peering_namespace, resource=peering_resource, ) assert not status.items.called assert not k8s_mocked.patch_obj.called assert wait_for.call_count == 0
async def test_ignored_for_higher_priority_peer_when_already_on(caplog, assert_logs): event = bodies.RawEvent( type='ADDED', # irrelevant object={ 'metadata': {'name': 'name', 'namespace': 'namespace'}, # for matching 'status': { 'higher-prio': { 'priority': 101, 'lifetime': 10, 'lastseen': '2020-12-31T23:59:59' }, }, }) replenished = asyncio.Event() freeze_mode = primitives.Toggle(True) ourselves = Peer(id='id', name='name', namespace='namespace', priority=100) caplog.set_level(0) assert freeze_mode.is_on() await process_peering_event( raw_event=event, freeze_mode=freeze_mode, ourselves=ourselves, replenished=replenished, autoclean=False, ) assert freeze_mode.is_on() assert_logs([], prohibited=[ "Possibly conflicting operators", "Freezing all operators, including self", "Freezing operations in favour of", "Resuming operations after the freeze", ])
async def test_other_peering_objects_are_ignored(mocker, settings, our_name, our_namespace, their_name, their_namespace): status = mocker.Mock() status.items.side_effect = Exception("This should not be called.") event = bodies.RawEvent( type='ADDED', # irrelevant object={ 'metadata': { 'name': their_name, 'namespace': their_namespace }, 'status': status, }) settings.peering.name = our_name await process_peering_event( raw_event=event, freeze_mode=primitives.Toggle(), replenished=asyncio.Event(), autoclean=False, identity='id', settings=settings, namespace=our_namespace, ) assert not status.items.called
async def test_ignored_for_same_priority_peer_when_already_on( mocker, k8s_mocked, replenished, caplog, assert_logs, settings, peering_resource, peering_namespace): event = bodies.RawEvent( type='ADDED', # irrelevant object={ 'metadata': { 'name': 'name', 'namespace': peering_namespace }, # for matching 'status': { 'higher-prio': { 'priority': 100, 'lifetime': 10, 'lastseen': '2020-12-31T23:59:59' }, }, }) settings.peering.name = 'name' settings.peering.priority = 100 freeze_toggle = primitives.Toggle(True) wait_for = mocker.patch('asyncio.wait_for') caplog.set_level(0) assert freeze_toggle.is_on() await process_peering_event( raw_event=event, freeze_toggle=freeze_toggle, replenished=replenished, autoclean=False, namespace=peering_namespace, resource=peering_resource, identity='id', settings=settings, ) assert freeze_toggle.is_on() assert wait_for.call_count == 1 assert 9 < wait_for.call_args[1]['timeout'] < 10 assert not k8s_mocked.patch_obj.called assert_logs([ "Possibly conflicting operators", ], prohibited=[ "Freezing all operators, including self", "Freezing operations in favour of", "Resuming operations after the freeze", ])
def __init__( self, __src: Optional[Mapping[str, object]] = None, ) -> None: super().__init__() self._current = {} self._invalid = collections.defaultdict(list) self._lock = asyncio.Lock() if __src is not None: self._update_converted(__src) # Mark a pre-populated vault to be usable instantly, # or trigger the initial authentication for an empty vault. self._ready = primitives.Toggle(bool(self))
async def test_ignored_for_same_priority_peer_when_already_on( k8s_mocked, caplog, assert_logs, settings, peering_resource, peering_namespace): event = bodies.RawEvent( type='ADDED', # irrelevant object={ 'metadata': { 'name': 'name', 'namespace': peering_namespace }, # for matching 'status': { 'higher-prio': { 'priority': 100, 'lifetime': 10, 'lastseen': '2020-12-31T23:59:59' }, }, }) settings.peering.name = 'name' settings.peering.priority = 100 conflicts_found = primitives.Toggle(True) k8s_mocked.sleep_or_wait.return_value = 1 # as if interrupted by stream pressure caplog.set_level(0) assert conflicts_found.is_on() await process_peering_event( raw_event=event, conflicts_found=conflicts_found, autoclean=False, namespace=peering_namespace, resource=peering_resource, identity='id', settings=settings, ) assert conflicts_found.is_on() assert k8s_mocked.sleep_or_wait.call_count == 1 assert 9 < k8s_mocked.sleep_or_wait.call_args[0][0][0] < 10 assert not k8s_mocked.patch_obj.called assert_logs([ "Possibly conflicting operators", ], prohibited=[ "Pausing all operators, including self", "Pausing operations in favour of", "Resuming operations after the pause", ])
async def test_toggled_on_for_same_priority_peer_when_initially_off( caplog, assert_logs, settings): event = bodies.RawEvent( type='ADDED', # irrelevant object={ 'metadata': { 'name': 'name', 'namespace': 'namespace' }, # for matching 'status': { 'higher-prio': { 'priority': 100, 'lifetime': 10, 'lastseen': '2020-12-31T23:59:59' }, }, }) settings.peering.name = 'name' settings.peering.priority = 100 replenished = asyncio.Event() freeze_mode = primitives.Toggle(False) caplog.set_level(0) assert freeze_mode.is_off() await process_peering_event( raw_event=event, freeze_mode=freeze_mode, replenished=replenished, autoclean=False, namespace='namespace', identity='id', settings=settings, ) assert freeze_mode.is_on() assert_logs([ "Possibly conflicting operators", "Freezing all operators, including self", ], prohibited=[ "Freezing operations in favour of", "Resuming operations after the freeze", ])
async def test_resumes_immediately_on_expiration_of_blocking_peers( mocker, k8s_mocked, replenished, caplog, assert_logs, settings, priority, peering_resource, peering_namespace): event = bodies.RawEvent( type='ADDED', # irrelevant object={ 'metadata': { 'name': 'name', 'namespace': peering_namespace }, # for matching 'status': { 'higher-prio': { 'priority': priority, 'lifetime': 10, 'lastseen': '2020-12-31T23:59:59' }, }, }) settings.peering.name = 'name' settings.peering.priority = 100 freeze_toggle = primitives.Toggle(True) wait_for = mocker.patch('asyncio.wait_for', side_effect=asyncio.TimeoutError) caplog.set_level(0) assert freeze_toggle.is_on() await process_peering_event( raw_event=event, freeze_toggle=freeze_toggle, replenished=replenished, autoclean=False, namespace=peering_namespace, resource=peering_resource, identity='id', settings=settings, ) assert freeze_toggle.is_on() assert wait_for.call_count == 1 assert 9 < wait_for.call_args[1]['timeout'] < 10 assert k8s_mocked.patch_obj.called
async def test_resumes_immediately_on_expiration_of_blocking_peers( k8s_mocked, caplog, assert_logs, settings, priority, peering_resource, peering_namespace): event = bodies.RawEvent( type='ADDED', # irrelevant object={ 'metadata': { 'name': 'name', 'namespace': peering_namespace }, # for matching 'status': { 'higher-prio': { 'priority': priority, 'lifetime': 10, 'lastseen': '2020-12-31T23:59:59' }, }, }) settings.peering.name = 'name' settings.peering.priority = 100 conflicts_found = primitives.Toggle(True) k8s_mocked.sleep_or_wait.return_value = None # as if finished sleeping uninterrupted caplog.set_level(0) assert conflicts_found.is_on() await process_peering_event( raw_event=event, conflicts_found=conflicts_found, autoclean=False, namespace=peering_namespace, resource=peering_resource, identity='id', settings=settings, ) assert conflicts_found.is_on() assert k8s_mocked.sleep_or_wait.call_count == 1 assert 9 < k8s_mocked.sleep_or_wait.call_args[0][0][0] < 10 assert k8s_mocked.patch_obj.called
async def spawn_tasks( *, lifecycle: Optional[lifecycles.LifeCycleFn] = None, registry: Optional[registries.OperatorRegistry] = None, memories: Optional[containers.ResourceMemories] = None, standalone: bool = False, priority: int = 0, peering_name: Optional[str] = None, liveness_endpoint: Optional[str] = None, namespace: Optional[str] = None, stop_flag: Optional[primitives.Flag] = None, ready_flag: Optional[primitives.Flag] = None, vault: Optional[credentials.Vault] = None, ) -> Tasks: """ Spawn all the tasks needed to run the operator. The tasks are properly inter-connected with the synchronisation primitives. """ loop = asyncio.get_running_loop() # The freezer and the registry are scoped to this whole task-set, to sync them all. lifecycle = lifecycle if lifecycle is not None else lifecycles.get_default_lifecycle() registry = registry if registry is not None else registries.get_default_registry() memories = memories if memories is not None else containers.ResourceMemories() vault = vault if vault is not None else global_vault vault = vault if vault is not None else credentials.Vault() event_queue: posting.K8sEventQueue = asyncio.Queue() freeze_mode: primitives.Toggle = primitives.Toggle() signal_flag: asyncio_Future = asyncio.Future() ready_flag = ready_flag if ready_flag is not None else asyncio.Event() tasks: MutableSequence[asyncio_Task] = [] # Global credentials store for this operator, also for CRD-reading & peering mode detection. auth.vault_var.set(vault) # Few common background forever-running infrastructural tasks (irregular root tasks). tasks.extend([ loop.create_task(_stop_flag_checker( signal_flag=signal_flag, stop_flag=stop_flag, )), loop.create_task(_startup_cleanup_activities( root_tasks=tasks, # used as a "live" view, populated later. ready_flag=ready_flag, registry=registry, vault=vault, # to purge & finalize the caches in the end. )), ]) # Keeping the credentials fresh and valid via the authentication handlers on demand. tasks.extend([ loop.create_task(_root_task_checker( name="credentials retriever", ready_flag=ready_flag, coro=activities.authenticator( registry=registry, vault=vault))), ]) # K8s-event posting. Events are queued in-memory and posted in the background. # NB: currently, it is a global task, but can be made per-resource or per-object. tasks.extend([ loop.create_task(_root_task_checker( name="poster of events", ready_flag=ready_flag, coro=posting.poster( event_queue=event_queue))), ]) # Liveness probing -- so that Kubernetes would know that the operator is alive. if liveness_endpoint: tasks.extend([ loop.create_task(_root_task_checker( name="health reporter", ready_flag=ready_flag, coro=probing.health_reporter( registry=registry, endpoint=liveness_endpoint))), ]) # Monitor the peers, unless explicitly disabled. ourselves: Optional[peering.Peer] = await peering.Peer.detect( id=peering.detect_own_id(), priority=priority, standalone=standalone, namespace=namespace, name=peering_name, ) if ourselves: tasks.extend([ loop.create_task(peering.peers_keepalive( ourselves=ourselves)), loop.create_task(_root_task_checker( name="watcher of peering", ready_flag=ready_flag, coro=queueing.watcher( namespace=namespace, resource=ourselves.resource, processor=functools.partial(peering.process_peering_event, ourselves=ourselves, freeze_mode=freeze_mode)))), ]) # Resource event handling, only once for every known resource (de-duplicated). for resource in registry.resources: tasks.extend([ loop.create_task(_root_task_checker( name=f"watcher of {resource.name}", ready_flag=ready_flag, coro=queueing.watcher( namespace=namespace, resource=resource, freeze_mode=freeze_mode, processor=functools.partial(processing.process_resource_event, lifecycle=lifecycle, registry=registry, memories=memories, resource=resource, event_queue=event_queue)))), ]) # On Ctrl+C or pod termination, cancel all tasks gracefully. if threading.current_thread() is threading.main_thread(): # Handle NotImplementedError when ran on Windows since asyncio only supports Unix signals try: loop.add_signal_handler(signal.SIGINT, signal_flag.set_result, signal.SIGINT) loop.add_signal_handler(signal.SIGTERM, signal_flag.set_result, signal.SIGTERM) except NotImplementedError: logger.warning("OS signals are ignored: can't add signal handler in Windows.") else: logger.warning("OS signals are ignored: running not in the main thread.") return tasks
async def spawn_tasks( *, lifecycle: Optional[lifecycles.LifeCycleFn] = None, registry: Optional[registries.OperatorRegistry] = None, settings: Optional[configuration.OperatorSettings] = None, memories: Optional[containers.ResourceMemories] = None, standalone: Optional[bool] = None, priority: Optional[int] = None, peering_name: Optional[str] = None, liveness_endpoint: Optional[str] = None, namespace: Optional[str] = None, stop_flag: Optional[primitives.Flag] = None, ready_flag: Optional[primitives.Flag] = None, vault: Optional[credentials.Vault] = None, ) -> Collection[aiotasks.Task]: """ Spawn all the tasks needed to run the operator. The tasks are properly inter-connected with the synchronisation primitives. """ loop = asyncio.get_running_loop() # The freezer and the registry are scoped to this whole task-set, to sync them all. lifecycle = lifecycle if lifecycle is not None else lifecycles.get_default_lifecycle( ) registry = registry if registry is not None else registries.get_default_registry( ) settings = settings if settings is not None else configuration.OperatorSettings( ) memories = memories if memories is not None else containers.ResourceMemories( ) vault = vault if vault is not None else global_vault vault = vault if vault is not None else credentials.Vault() event_queue: posting.K8sEventQueue = asyncio.Queue() freeze_mode: primitives.Toggle = primitives.Toggle() signal_flag: aiotasks.Future = asyncio.Future() started_flag: asyncio.Event = asyncio.Event() tasks: MutableSequence[aiotasks.Task] = [] # Map kwargs into the settings object. if peering_name is not None: settings.peering.mandatory = True settings.peering.name = peering_name if standalone is not None: settings.peering.standalone = standalone if priority is not None: settings.peering.priority = priority # Global credentials store for this operator, also for CRD-reading & peering mode detection. auth.vault_var.set(vault) # Special case: pass the settings container through the user-side handlers (no explicit args). # Toolkits have to keep the original operator context somehow, and the only way is contextvars. posting.settings_var.set(settings) # Few common background forever-running infrastructural tasks (irregular root tasks). tasks.append( aiotasks.create_task(name="stop-flag checker", coro=_stop_flag_checker(signal_flag=signal_flag, stop_flag=stop_flag))) tasks.append( aiotasks.create_task(name="ultimate termination", coro=_ultimate_termination(settings=settings, stop_flag=stop_flag))) tasks.append( aiotasks.create_task( name="startup/cleanup activities", coro=_startup_cleanup_activities( root_tasks=tasks, # used as a "live" view, populated later. ready_flag=ready_flag, started_flag=started_flag, registry=registry, settings=settings, vault=vault))) # to purge & finalize the caches in the end. # Kill all the daemons gracefully when the operator exits (so that they are not "hung"). tasks.append( aiotasks.create_guarded_task( name="daemon killer", flag=started_flag, logger=logger, coro=daemons.daemon_killer(settings=settings, memories=memories))) # Keeping the credentials fresh and valid via the authentication handlers on demand. tasks.append( aiotasks.create_guarded_task(name="credentials retriever", flag=started_flag, logger=logger, coro=activities.authenticator( registry=registry, settings=settings, vault=vault))) # K8s-event posting. Events are queued in-memory and posted in the background. # NB: currently, it is a global task, but can be made per-resource or per-object. tasks.append( aiotasks.create_guarded_task( name="poster of events", flag=started_flag, logger=logger, coro=posting.poster(event_queue=event_queue))) # Liveness probing -- so that Kubernetes would know that the operator is alive. if liveness_endpoint: tasks.append( aiotasks.create_guarded_task(name="health reporter", flag=started_flag, logger=logger, coro=probing.health_reporter( registry=registry, settings=settings, endpoint=liveness_endpoint))) # Monitor the peers, unless explicitly disabled. if await peering.detect_presence(namespace=namespace, settings=settings): identity = peering.detect_own_id(manual=False) tasks.append( aiotasks.create_guarded_task(name="peering keepalive", flag=started_flag, logger=logger, coro=peering.keepalive( namespace=namespace, settings=settings, identity=identity))) tasks.append( aiotasks.create_guarded_task( name="watcher of peering", flag=started_flag, logger=logger, coro=queueing.watcher( namespace=namespace, settings=settings, resource=peering.guess_resource(namespace=namespace), processor=functools.partial(peering.process_peering_event, namespace=namespace, settings=settings, identity=identity, freeze_mode=freeze_mode)))) # Resource event handling, only once for every known resource (de-duplicated). for resource in registry.resources: tasks.append( aiotasks.create_guarded_task( name=f"watcher of {resource.name}", flag=started_flag, logger=logger, coro=queueing.watcher(namespace=namespace, settings=settings, resource=resource, freeze_mode=freeze_mode, processor=functools.partial( processing.process_resource_event, lifecycle=lifecycle, registry=registry, settings=settings, memories=memories, resource=resource, event_queue=event_queue)))) # On Ctrl+C or pod termination, cancel all tasks gracefully if possible. if threading.current_thread() is threading.main_thread(): # Handle NotImplementedError when ran on Windows since asyncio only supports Unix signals try: loop.add_signal_handler(signal.SIGINT, signal_flag.set_result, signal.SIGINT) loop.add_signal_handler(signal.SIGTERM, signal_flag.set_result, signal.SIGTERM) except NotImplementedError: logger.warning( "OS signals are ignored: can't add signal handler in Windows.") else: logger.warning( "OS signals are ignored: running not in the main thread.") return tasks