def watcher_in_background(settings, resource, event_loop, worker_spy, stream): # Prevent remembering the streaming objects in the mocks. async def do_nothing(*args, **kwargs): pass # Prevent any real streaming for the very beginning, before it even starts. stream.feed([]) # Spawn a watcher in the background. coro = watcher( namespace=None, resource=resource, settings=settings, processor=do_nothing, ) task = event_loop.create_task(coro) try: # Go for a test. yield task finally: # Terminate the watcher to cleanup the loop. task.cancel() try: event_loop.run_until_complete(task) except asyncio.CancelledError: pass
async def spawn_missing_watchers( *, processor: queueing.WatchStreamProcessor, settings: configuration.OperatorSettings, resources: Collection[references.Resource], indexable: Collection[references.Resource], namespaces: Collection[references.Namespace], ensemble: Ensemble, ) -> None: # Block the operator globally until specialised per-resource-kind blockers are created. # NB: Must be created before the point of parallelisation! operator_blocked = await ensemble.operator_indexed.make_toggle( name="orchestration blocker") # Spawn watchers and create the specialised per-resource-kind blockers. for resource, namespace in itertools.product(resources, namespaces): namespace = namespace if resource.namespaced else None dkey = EnsembleKey(resource=resource, namespace=namespace) if dkey not in ensemble.watcher_tasks: what = f"{resource}@{namespace}" resource_indexed: Optional[primitives.Toggle] = None if resource in indexable: resource_indexed = await ensemble.operator_indexed.make_toggle( name=what) ensemble.watcher_tasks[dkey] = aiotasks.create_guarded_task( name=f"watcher for {what}", logger=logger, cancellable=True, coro=queueing.watcher( operator_paused=ensemble.operator_paused, operator_indexed=ensemble.operator_indexed, resource_indexed=resource_indexed, settings=settings, resource=resource, namespace=namespace, processor=functools.partial(processor, resource=resource))) # Unblock globally, let the specialised per-resource-kind blockers hold the readiness. await ensemble.operator_indexed.drop_toggle(operator_blocked) # Ensure that all guarded tasks got control for a moment to enter the guard. await asyncio.sleep(0)
async def spawn_missing_peerings( *, settings: configuration.OperatorSettings, identity: peering.Identity, resources: Collection[references.Resource], namespaces: Collection[references.Namespace], ensemble: Ensemble, ) -> None: for resource, namespace in itertools.product(resources, namespaces): dkey = EnsembleKey(resource=resource, namespace=namespace) if dkey not in ensemble.peering_tasks: what = f"{settings.peering.name}@{namespace}" is_preactivated = settings.peering.mandatory conflicts_found = await ensemble.operator_paused.make_toggle( is_preactivated, name=what) ensemble.conflicts_found[dkey] = conflicts_found ensemble.pinging_tasks[dkey] = aiotasks.create_guarded_task( name=f"peering keep-alive for {what}", logger=logger, cancellable=True, coro=peering.keepalive(namespace=namespace, resource=resource, settings=settings, identity=identity)) ensemble.peering_tasks[dkey] = aiotasks.create_guarded_task( name=f"peering observer for {what}", logger=logger, cancellable=True, coro=queueing.watcher(settings=settings, resource=resource, namespace=namespace, processor=functools.partial( peering.process_peering_event, conflicts_found=conflicts_found, namespace=namespace, resource=resource, settings=settings, identity=identity))) # Ensure that all guarded tasks got control for a moment to enter the guard. await asyncio.sleep(0)
def watcher_in_background(resource, handler, event_loop, worker_spy, stream): # Prevent any real streaming for the very beginning, before it even starts. stream.feed([]) # Spawn a watcher in the background. coro = watcher( namespace=None, resource=resource, handler=handler, ) task = event_loop.create_task(coro) try: # Go for a test. yield task finally: # Terminate the watcher to cleanup the loop. task.cancel() try: event_loop.run_until_complete(task) except asyncio.CancelledError: pass
async def spawn_tasks( *, lifecycle: Optional[lifecycles.LifeCycleFn] = None, registry: Optional[registries.OperatorRegistry] = None, memories: Optional[containers.ResourceMemories] = None, standalone: bool = False, priority: int = 0, peering_name: Optional[str] = None, liveness_endpoint: Optional[str] = None, namespace: Optional[str] = None, stop_flag: Optional[primitives.Flag] = None, ready_flag: Optional[primitives.Flag] = None, vault: Optional[credentials.Vault] = None, ) -> Tasks: """ Spawn all the tasks needed to run the operator. The tasks are properly inter-connected with the synchronisation primitives. """ loop = asyncio.get_running_loop() # The freezer and the registry are scoped to this whole task-set, to sync them all. lifecycle = lifecycle if lifecycle is not None else lifecycles.get_default_lifecycle() registry = registry if registry is not None else registries.get_default_registry() memories = memories if memories is not None else containers.ResourceMemories() vault = vault if vault is not None else global_vault vault = vault if vault is not None else credentials.Vault() event_queue: posting.K8sEventQueue = asyncio.Queue() freeze_mode: primitives.Toggle = primitives.Toggle() signal_flag: asyncio_Future = asyncio.Future() ready_flag = ready_flag if ready_flag is not None else asyncio.Event() tasks: MutableSequence[asyncio_Task] = [] # Global credentials store for this operator, also for CRD-reading & peering mode detection. auth.vault_var.set(vault) # Few common background forever-running infrastructural tasks (irregular root tasks). tasks.extend([ loop.create_task(_stop_flag_checker( signal_flag=signal_flag, stop_flag=stop_flag, )), loop.create_task(_startup_cleanup_activities( root_tasks=tasks, # used as a "live" view, populated later. ready_flag=ready_flag, registry=registry, vault=vault, # to purge & finalize the caches in the end. )), ]) # Keeping the credentials fresh and valid via the authentication handlers on demand. tasks.extend([ loop.create_task(_root_task_checker( name="credentials retriever", ready_flag=ready_flag, coro=activities.authenticator( registry=registry, vault=vault))), ]) # K8s-event posting. Events are queued in-memory and posted in the background. # NB: currently, it is a global task, but can be made per-resource or per-object. tasks.extend([ loop.create_task(_root_task_checker( name="poster of events", ready_flag=ready_flag, coro=posting.poster( event_queue=event_queue))), ]) # Liveness probing -- so that Kubernetes would know that the operator is alive. if liveness_endpoint: tasks.extend([ loop.create_task(_root_task_checker( name="health reporter", ready_flag=ready_flag, coro=probing.health_reporter( registry=registry, endpoint=liveness_endpoint))), ]) # Monitor the peers, unless explicitly disabled. ourselves: Optional[peering.Peer] = await peering.Peer.detect( id=peering.detect_own_id(), priority=priority, standalone=standalone, namespace=namespace, name=peering_name, ) if ourselves: tasks.extend([ loop.create_task(peering.peers_keepalive( ourselves=ourselves)), loop.create_task(_root_task_checker( name="watcher of peering", ready_flag=ready_flag, coro=queueing.watcher( namespace=namespace, resource=ourselves.resource, processor=functools.partial(peering.process_peering_event, ourselves=ourselves, freeze_mode=freeze_mode)))), ]) # Resource event handling, only once for every known resource (de-duplicated). for resource in registry.resources: tasks.extend([ loop.create_task(_root_task_checker( name=f"watcher of {resource.name}", ready_flag=ready_flag, coro=queueing.watcher( namespace=namespace, resource=resource, freeze_mode=freeze_mode, processor=functools.partial(processing.process_resource_event, lifecycle=lifecycle, registry=registry, memories=memories, resource=resource, event_queue=event_queue)))), ]) # On Ctrl+C or pod termination, cancel all tasks gracefully. if threading.current_thread() is threading.main_thread(): # Handle NotImplementedError when ran on Windows since asyncio only supports Unix signals try: loop.add_signal_handler(signal.SIGINT, signal_flag.set_result, signal.SIGINT) loop.add_signal_handler(signal.SIGTERM, signal_flag.set_result, signal.SIGTERM) except NotImplementedError: logger.warning("OS signals are ignored: can't add signal handler in Windows.") else: logger.warning("OS signals are ignored: running not in the main thread.") return tasks
async def spawn_tasks( lifecycle: Optional[Callable] = None, registry: Optional[registries.GlobalRegistry] = None, standalone: bool = False, priority: int = 0, peering_name: Optional[str] = None, namespace: Optional[str] = None, ) -> Collection[asyncio.Task]: """ Spawn all the tasks needed to run the operator. The tasks are properly inter-connected with the synchronisation primitives. """ loop = asyncio.get_running_loop() # The freezer and the registry are scoped to this whole task-set, to sync them all. lifecycle = lifecycle if lifecycle is not None else lifecycles.get_default_lifecycle( ) registry = registry if registry is not None else registries.get_default_registry( ) event_queue = asyncio.Queue(loop=loop) freeze_flag = asyncio.Event(loop=loop) should_stop = asyncio.Event(loop=loop) tasks = [] # A top-level task for external stopping by setting a stop-flag. Once set, # this task will exit, and thus all other top-level tasks will be cancelled. tasks.extend([ loop.create_task(_stop_flag_checker(should_stop)), ]) # K8s-event posting. Events are queued in-memory and posted in the background. # NB: currently, it is a global task, but can be made per-resource or per-object. tasks.extend([ loop.create_task(posting.poster(event_queue=event_queue)), ]) # Monitor the peers, unless explicitly disabled. ourselves: Optional[peering.Peer] = peering.Peer.detect( id=peering.detect_own_id(), priority=priority, standalone=standalone, namespace=namespace, name=peering_name, ) if ourselves: tasks.extend([ loop.create_task(peering.peers_keepalive(ourselves=ourselves)), loop.create_task( queueing.watcher( namespace=namespace, resource=ourselves.resource, handler=functools.partial( peering.peers_handler, ourselves=ourselves, freeze=freeze_flag))), # freeze is set/cleared ]) # Resource event handling, only once for every known resource (de-duplicated). for resource in registry.resources: tasks.extend([ loop.create_task( queueing.watcher( namespace=namespace, resource=resource, handler=functools.partial( handling.custom_object_handler, lifecycle=lifecycle, registry=registry, resource=resource, event_queue=event_queue, freeze=freeze_flag))), # freeze is only checked ]) # On Ctrl+C or pod termination, cancel all tasks gracefully. if threading.current_thread() is threading.main_thread(): loop.add_signal_handler(signal.SIGINT, should_stop.set) loop.add_signal_handler(signal.SIGTERM, should_stop.set) else: logger.warning( "OS signals are ignored: running not in the main thread.") return tasks
async def spawn_tasks( *, lifecycle: Optional[lifecycles.LifeCycleFn] = None, registry: Optional[registries.OperatorRegistry] = None, settings: Optional[configuration.OperatorSettings] = None, memories: Optional[containers.ResourceMemories] = None, standalone: Optional[bool] = None, priority: Optional[int] = None, peering_name: Optional[str] = None, liveness_endpoint: Optional[str] = None, namespace: Optional[str] = None, stop_flag: Optional[primitives.Flag] = None, ready_flag: Optional[primitives.Flag] = None, vault: Optional[credentials.Vault] = None, ) -> Collection[aiotasks.Task]: """ Spawn all the tasks needed to run the operator. The tasks are properly inter-connected with the synchronisation primitives. """ loop = asyncio.get_running_loop() # The freezer and the registry are scoped to this whole task-set, to sync them all. lifecycle = lifecycle if lifecycle is not None else lifecycles.get_default_lifecycle( ) registry = registry if registry is not None else registries.get_default_registry( ) settings = settings if settings is not None else configuration.OperatorSettings( ) memories = memories if memories is not None else containers.ResourceMemories( ) vault = vault if vault is not None else global_vault vault = vault if vault is not None else credentials.Vault() event_queue: posting.K8sEventQueue = asyncio.Queue() freeze_name = f"{peering_name!r}@{namespace}" if namespace else f"cluster-wide {peering_name!r}" freeze_checker = primitives.ToggleSet() freeze_toggle = await freeze_checker.make_toggle(name=freeze_name) signal_flag: aiotasks.Future = asyncio.Future() started_flag: asyncio.Event = asyncio.Event() tasks: MutableSequence[aiotasks.Task] = [] # Map kwargs into the settings object. if peering_name is not None: settings.peering.mandatory = True settings.peering.name = peering_name if standalone is not None: settings.peering.standalone = standalone if priority is not None: settings.peering.priority = priority # Global credentials store for this operator, also for CRD-reading & peering mode detection. auth.vault_var.set(vault) # Special case: pass the settings container through the user-side handlers (no explicit args). # Toolkits have to keep the original operator context somehow, and the only way is contextvars. posting.settings_var.set(settings) # Few common background forever-running infrastructural tasks (irregular root tasks). tasks.append( aiotasks.create_task(name="stop-flag checker", coro=_stop_flag_checker(signal_flag=signal_flag, stop_flag=stop_flag))) tasks.append( aiotasks.create_task(name="ultimate termination", coro=_ultimate_termination(settings=settings, stop_flag=stop_flag))) tasks.append( aiotasks.create_task( name="startup/cleanup activities", coro=_startup_cleanup_activities( root_tasks=tasks, # used as a "live" view, populated later. ready_flag=ready_flag, started_flag=started_flag, registry=registry, settings=settings, vault=vault))) # to purge & finalize the caches in the end. # Kill all the daemons gracefully when the operator exits (so that they are not "hung"). tasks.append( aiotasks.create_guarded_task( name="daemon killer", flag=started_flag, logger=logger, coro=daemons.daemon_killer(settings=settings, memories=memories))) # Keeping the credentials fresh and valid via the authentication handlers on demand. tasks.append( aiotasks.create_guarded_task(name="credentials retriever", flag=started_flag, logger=logger, coro=activities.authenticator( registry=registry, settings=settings, vault=vault))) # K8s-event posting. Events are queued in-memory and posted in the background. # NB: currently, it is a global task, but can be made per-resource or per-object. tasks.append( aiotasks.create_guarded_task( name="poster of events", flag=started_flag, logger=logger, coro=posting.poster(event_queue=event_queue))) # Liveness probing -- so that Kubernetes would know that the operator is alive. if liveness_endpoint: tasks.append( aiotasks.create_guarded_task(name="health reporter", flag=started_flag, logger=logger, coro=probing.health_reporter( registry=registry, settings=settings, endpoint=liveness_endpoint))) # Monitor the peers, unless explicitly disabled. if await peering.detect_presence(namespace=namespace, settings=settings): identity = peering.detect_own_id(manual=False) tasks.append( aiotasks.create_guarded_task(name="peering keepalive", flag=started_flag, logger=logger, coro=peering.keepalive( namespace=namespace, settings=settings, identity=identity))) tasks.append( aiotasks.create_guarded_task( name="watcher of peering", flag=started_flag, logger=logger, coro=queueing.watcher( namespace=namespace, settings=settings, resource=peering.guess_resource(namespace=namespace), processor=functools.partial(peering.process_peering_event, namespace=namespace, settings=settings, identity=identity, freeze_toggle=freeze_toggle)))) # Resource event handling, only once for every known resource (de-duplicated). for resource in registry.resources: tasks.append( aiotasks.create_guarded_task( name=f"watcher of {resource.name}", flag=started_flag, logger=logger, coro=queueing.watcher(namespace=namespace, settings=settings, resource=resource, freeze_checker=freeze_checker, processor=functools.partial( processing.process_resource_event, lifecycle=lifecycle, registry=registry, settings=settings, memories=memories, resource=resource, event_queue=event_queue)))) # On Ctrl+C or pod termination, cancel all tasks gracefully. if threading.current_thread() is threading.main_thread(): # Handle NotImplementedError when ran on Windows since asyncio only supports Unix signals try: loop.add_signal_handler(signal.SIGINT, signal_flag.set_result, signal.SIGINT) loop.add_signal_handler(signal.SIGTERM, signal_flag.set_result, signal.SIGTERM) except NotImplementedError: logger.warning( "OS signals are ignored: can't add signal handler in Windows.") else: logger.warning( "OS signals are ignored: running not in the main thread.") return tasks