async def test_continuation_when_overdue(clock, sleep): wakeup = asyncio.Event() logger = logging.getLogger() throttler = Throttler() clock.return_value = 1000 # simulated "now" sleep.return_value = 55 # simulated sleep time left async with throttled(throttler=throttler, logger=logger, delays=[123, 234], wakeup=wakeup): raise Exception() sleep.reset_mock() clock.return_value = 2000 # simulated "now" sleep.return_value = None # simulated sleep time left async with throttled(throttler=throttler, logger=logger, delays=[...], wakeup=wakeup): raise Exception() assert throttler.last_used_delay == 234 assert throttler.source_of_delays is not None assert throttler.active_until is None # means: no sleep time is left assert sleep.mock_calls == [ call(123 - 1000, wakeup=wakeup), call(234, wakeup=wakeup) ]
async def test_recommends_running_immediately_after_continued(sleep): logger = logging.getLogger() throttler = Throttler() sleep.return_value = 33 # simulated sleep time left async with throttled(throttler=throttler, logger=logger, delays=[123]): raise Exception() sleep.return_value = None # simulated sleep time left async with throttled(throttler=throttler, logger=logger, delays=[...]) as should_run: remembered_should_run = should_run assert remembered_should_run is True
async def test_recommends_skipping_immediately_after_interrupted_error(sleep): logger = logging.getLogger() throttler = Throttler() sleep.return_value = 33 # simulated sleep time left async with throttled(throttler=throttler, logger=logger, delays=[123]): raise Exception() sleep.return_value = 33 # simulated sleep time left async with throttled(throttler=throttler, logger=logger, delays=[...]) as should_run: remembered_should_run = should_run assert remembered_should_run is False
async def test_resets_on_success(sleep): logger = logging.getLogger() throttler = Throttler() async with throttled(throttler=throttler, logger=logger, delays=[123]): raise Exception() sleep.reset_mock() async with throttled(throttler=throttler, logger=logger, delays=[...]): pass assert throttler.last_used_delay is None assert throttler.source_of_delays is None assert throttler.active_until is None assert sleep.mock_calls == []
async def test_recommends_running_initially(): logger = logging.getLogger() throttler = Throttler() async with throttled(throttler=throttler, logger=logger, delays=[123]) as should_run: remembered_should_run = should_run assert remembered_should_run is True
async def test_remains_inactive_on_success(): logger = logging.getLogger() throttler = Throttler() async with throttled(throttler=throttler, logger=logger, delays=[123]): pass assert throttler.source_of_delays is None assert throttler.last_used_delay is None
async def test_logging_when_deactivates_on_reentry(sleep, caplog): caplog.set_level(0) logger = logging.getLogger() throttler = Throttler() sleep.return_value = 55 # simulated sleep time left async with throttled(throttler=throttler, logger=logger, delays=[123]): raise Exception("boo!") sleep.return_value = None # simulated sleep time left async with throttled(throttler=throttler, logger=logger, delays=[...]): pass assert caplog.messages == [ "Throttling for 123 seconds due to an unexpected error: Exception('boo!')", "Throttling is over. Switching back to normal operations.", ]
async def test_sleeps_for_the_next_delay_when_active(sleep): logger = logging.getLogger() throttler = Throttler() async with throttled(throttler=throttler, logger=logger, delays=[123, 234]): raise Exception() sleep.reset_mock() async with throttled(throttler=throttler, logger=logger, delays=[...]): raise Exception() assert throttler.last_used_delay == 234 assert throttler.source_of_delays is not None assert next(throttler.source_of_delays, 999) == 999 assert throttler.active_until is None # means: no sleep time left assert sleep.mock_calls == [call(234, wakeup=None)]
async def test_escalates_unexpected_errors(exc_cls, kwargs): logger = logging.getLogger() throttler = Throttler() with pytest.raises(exc_cls): async with throttled(throttler=throttler, logger=logger, delays=[123], **kwargs): raise exc_cls()
async def test_renews_on_repeated_failure(sleep): logger = logging.getLogger() throttler = Throttler() async with throttled(throttler=throttler, logger=logger, delays=[123]): raise Exception() async with throttled(throttler=throttler, logger=logger, delays=[...]): pass sleep.reset_mock() async with throttled(throttler=throttler, logger=logger, delays=[234]): raise Exception() assert throttler.last_used_delay is 234 assert throttler.source_of_delays is not None assert throttler.active_until is None assert sleep.mock_calls == [call(234, wakeup=None)]
async def test_activates_on_expected_errors(exc_cls, kwargs): logger = logging.getLogger() throttler = Throttler() async with throttled(throttler=throttler, logger=logger, delays=[123], **kwargs): raise exc_cls() assert throttler.source_of_delays is not None assert throttler.last_used_delay is not None
async def test_logging_when_deactivates_immediately(caplog): caplog.set_level(0) logger = logging.getLogger() throttler = Throttler() async with throttled(throttler=throttler, logger=logger, delays=[123]): raise Exception("boo!") assert caplog.messages == [ "Throttling for 123 seconds due to an unexpected error: Exception('boo!')", "Throttling is over. Switching back to normal operations.", ]
async def test_skips_on_no_delays(sleep): logger = logging.getLogger() throttler = Throttler() async with throttled(throttler=throttler, logger=logger, delays=[]): raise Exception() assert throttler.last_used_delay is None assert throttler.source_of_delays is not None assert next(throttler.source_of_delays, 999) == 999 assert throttler.active_until is None # means: no sleep time left assert sleep.mock_calls == []
async def test_interruption(clock, sleep): wakeup = asyncio.Event() logger = logging.getLogger() throttler = Throttler() clock.return_value = 1000 # simulated "now" sleep.return_value = 55 # simulated sleep time left async with throttled(throttler=throttler, logger=logger, delays=[123, 234], wakeup=wakeup): raise Exception() assert throttler.last_used_delay == 123 assert throttler.source_of_delays is not None assert throttler.active_until == 1123 # means: some sleep time is left assert sleep.mock_calls == [call(123, wakeup=wakeup)]
async def process_resource_event( lifecycle: execution.LifeCycleFn, indexers: indexing.OperatorIndexers, registry: registries.OperatorRegistry, settings: configuration.OperatorSettings, memories: inventory.ResourceMemories, memobase: ephemera.AnyMemo, resource: references.Resource, raw_event: bodies.RawEvent, event_queue: posting.K8sEventQueue, stream_pressure: Optional[asyncio.Event] = None, # None for tests resource_indexed: Optional[aiotoggles.Toggle] = None, # None for tests & observation operator_indexed: Optional[aiotoggles.ToggleSet] = None, # None for tests & observation ) -> None: """ Handle a single custom object low-level watch-event. Convert the low-level events, as provided by the watching/queueing tasks, to the high-level causes, and then call the cause-handling logic. """ # Recall what is stored about that object. Share it in little portions with the consumers. # And immediately forget it if the object is deleted from the cluster (but keep in memory). raw_type, raw_body = raw_event['type'], raw_event['object'] memory = await memories.recall(raw_body, noticed_by_listing=raw_type is None, memobase=memobase) if memory.daemons_memory.live_fresh_body is not None: memory.daemons_memory.live_fresh_body._replace_with(raw_body) if raw_type == 'DELETED': await memories.forget(raw_body) # Convert to a heavy mapping-view wrapper only now, when heavy processing begins. # Raw-event streaming, queueing, and batching use regular lightweight dicts. # Why here? 1. Before it splits into multiple causes & handlers for the same object's body; # 2. After it is batched (queueing); 3. While the "raw" parsed JSON is still known; # 4. Same as where a patch object of a similar wrapping semantics is created. live_fresh_body = memory.daemons_memory.live_fresh_body body = live_fresh_body if live_fresh_body is not None else bodies.Body(raw_body) patch = patches.Patch() # Different loggers for different cases with different verbosity and exposure. local_logger = loggers.LocalObjectLogger(body=body, settings=settings) terse_logger = loggers.TerseObjectLogger(body=body, settings=settings) event_logger = loggers.ObjectLogger(body=body, settings=settings) # Throttle the non-handler-related errors. The regular event watching/batching continues # to prevent queue overfilling, but the processing is skipped (events are ignored). # Choice of place: late enough to have a per-resource memory for a throttler; also, a logger. # But early enough to catch environment errors from K8s API, and from most of the complex code. async with throttlers.throttled( throttler=memory.error_throttler, logger=local_logger, delays=settings.batching.error_delays, wakeup=stream_pressure, ) as should_run: if should_run: # Each object has its own prefixed logger, to distinguish parallel handling. posting.event_queue_loop_var.set(asyncio.get_running_loop()) posting.event_queue_var.set(event_queue) # till the end of this object's task. # [Pre-]populate the indices. This must be lightweight. await indexing.index_resource( registry=registry, indexers=indexers, settings=settings, resource=resource, raw_event=raw_event, body=body, memo=memory.memo, memory=memory.indexing_memory, logger=terse_logger, ) # Wait for all other individual resources and all other resource kinds' lists to finish. # If this one has changed while waiting for the global readiness, let it be reprocessed. if operator_indexed is not None and resource_indexed is not None: await operator_indexed.drop_toggle(resource_indexed) if operator_indexed is not None: await operator_indexed.wait_for(True) # other resource kinds & objects. if stream_pressure is not None and stream_pressure.is_set(): return # Do the magic -- do the job. delays, matched = await process_resource_causes( lifecycle=lifecycle, indexers=indexers, registry=registry, settings=settings, resource=resource, raw_event=raw_event, body=body, patch=patch, memory=memory, local_logger=local_logger, event_logger=event_logger, ) # Whatever was done, apply the accumulated changes to the object, or sleep-n-touch for delays. # But only once, to reduce the number of API calls and the generated irrelevant events. # And only if the object is at least supposed to exist (not "GONE"), even if actually does not. if raw_event['type'] != 'DELETED': applied = await application.apply( settings=settings, resource=resource, body=body, patch=patch, logger=local_logger, delays=delays, stream_pressure=stream_pressure, ) if applied and matched: local_logger.debug("Handling cycle is finished, waiting for new changes.")