async def test_delayed_handlers_progress(registry, settings, handlers,
resource, cause_mock, cause_reason,
caplog, assert_logs, k8s_mocked, now,
delayed_iso, delay):
caplog.set_level(logging.DEBUG)
handlers.create_mock.side_effect = TemporaryError("oops", delay=delay)
handlers.update_mock.side_effect = TemporaryError("oops", delay=delay)
handlers.delete_mock.side_effect = TemporaryError("oops", delay=delay)
handlers.resume_mock.side_effect = TemporaryError("oops", delay=delay)
event_type = None if cause_reason == Reason.RESUME else 'irrelevant'
cause_mock.reason = cause_reason
with freezegun.freeze_time(now):
await process_resource_event(
lifecycle=kopf.lifecycles.all_at_once,
registry=registry,
settings=settings,
resource=resource,
indexers=OperatorIndexers(),
memories=ResourceMemories(),
memobase=Memo(),
raw_event={
'type': event_type,
'object': {}
},
event_queue=asyncio.Queue(),
)
assert handlers.create_mock.call_count == (1 if cause_reason
== Reason.CREATE else 0)
assert handlers.update_mock.call_count == (1 if cause_reason
== Reason.UPDATE else 0)
assert handlers.delete_mock.call_count == (1 if cause_reason
== Reason.DELETE else 0)
assert handlers.resume_mock.call_count == (1 if cause_reason
== Reason.RESUME else 0)
assert not k8s_mocked.sleep_or_wait.called
assert k8s_mocked.patch_obj.called
fname = f'{cause_reason}_fn'
patch = k8s_mocked.patch_obj.call_args_list[0][1]['patch']
assert patch['status']['kopf']['progress'][fname]['delayed'] == delayed_iso
assert_logs([
"Handler .+ is invoked",
"Handler .+ failed temporarily: oops",
])
async def test_retry_error_delays_handler(registry, settings, handlers,
extrahandlers, resource, cause_mock,
cause_type, caplog, assert_logs,
k8s_mocked):
caplog.set_level(logging.DEBUG)
name1 = f'{cause_type}_fn'
event_type = None if cause_type == Reason.RESUME else 'irrelevant'
cause_mock.reason = cause_type
handlers.create_mock.side_effect = TemporaryError("oops")
handlers.update_mock.side_effect = TemporaryError("oops")
handlers.delete_mock.side_effect = TemporaryError("oops")
handlers.resume_mock.side_effect = TemporaryError("oops")
await process_resource_event(
lifecycle=kopf.lifecycles.one_by_one,
registry=registry,
settings=settings,
resource=resource,
indexers=OperatorIndexers(),
memories=ResourceMemories(),
memobase=Memo(),
raw_event={
'type': event_type,
'object': {}
},
event_queue=asyncio.Queue(),
)
assert handlers.create_mock.call_count == (1 if cause_type == Reason.CREATE
else 0)
assert handlers.update_mock.call_count == (1 if cause_type == Reason.UPDATE
else 0)
assert handlers.delete_mock.call_count == (1 if cause_type == Reason.DELETE
else 0)
assert handlers.resume_mock.call_count == (1 if cause_type == Reason.RESUME
else 0)
assert not k8s_mocked.sleep_or_wait.called
assert k8s_mocked.patch_obj.called
patch = k8s_mocked.patch_obj.call_args_list[0][1]['patch']
assert patch['status']['kopf']['progress'] is not None
assert patch['status']['kopf']['progress'][name1]['failure'] is False
assert patch['status']['kopf']['progress'][name1]['success'] is False
assert patch['status']['kopf']['progress'][name1]['delayed']
assert_logs([
"Handler .+ failed temporarily: oops",
])
async def test_removed_and_remembered_on_temporary_errors(
resource, settings, registry, memories, indexers, index, caplog, event_type, handlers,
delay_kwargs, expected_delayed):
caplog.set_level(logging.DEBUG)
body = {'metadata': {'namespace': 'ns1', 'name': 'name1'}}
memory = await memories.recall(raw_body=body)
handlers.index_mock.side_effect = TemporaryError("boo!", **delay_kwargs)
await process_resource_event(
lifecycle=all_at_once,
registry=registry,
settings=settings,
resource=resource,
indexers=indexers,
memories=memories,
memobase=Memo(),
raw_event={'type': event_type, 'object': body},
event_queue=asyncio.Queue(),
resource_indexed=Toggle(), # used! only to enable indexing.
)
assert set(index) == set()
assert memory.indexing_state['index_fn'].finished == False
assert memory.indexing_state['index_fn'].failure == False
assert memory.indexing_state['index_fn'].success == False
assert memory.indexing_state['index_fn'].message == 'boo!'
assert memory.indexing_state['index_fn'].delayed == expected_delayed
async def fn(**kwargs):
dummy.mock()
dummy.kwargs = kwargs
dummy.steps['called'].set()
if dummy.mock.call_count >= 5:
kwargs['stopped']._stopper.set(reason=kopf.DaemonStoppingReason.NONE) # to exit the cycle
raise TemporaryError("boo!", delay=1.0)
async def test_delayed_handlers_progress(registry, handlers, resource,
cause_mock, cause_reason, caplog,
assert_logs, k8s_mocked, now, ts,
delay):
caplog.set_level(logging.DEBUG)
handlers.create_mock.side_effect = TemporaryError("oops", delay=delay)
handlers.update_mock.side_effect = TemporaryError("oops", delay=delay)
handlers.delete_mock.side_effect = TemporaryError("oops", delay=delay)
handlers.resume_mock.side_effect = TemporaryError("oops", delay=delay)
cause_mock.reason = cause_reason
with freezegun.freeze_time(now):
await resource_handler(
lifecycle=kopf.lifecycles.all_at_once,
registry=registry,
resource=resource,
event={
'type': 'irrelevant',
'object': cause_mock.body
},
freeze=asyncio.Event(),
replenished=asyncio.Event(),
event_queue=asyncio.Queue(),
)
assert handlers.create_mock.call_count == (1 if cause_reason
== Reason.CREATE else 0)
assert handlers.update_mock.call_count == (1 if cause_reason
== Reason.UPDATE else 0)
assert handlers.delete_mock.call_count == (1 if cause_reason
== Reason.DELETE else 0)
assert handlers.resume_mock.call_count == (1 if cause_reason
== Reason.RESUME else 0)
assert not k8s_mocked.sleep_or_wait.called
assert k8s_mocked.patch_obj.called
fname = f'{cause_reason}_fn'
patch = k8s_mocked.patch_obj.call_args_list[0][1]['patch']
assert patch['status']['kopf']['progress'][fname]['delayed'] == ts
assert_logs([
"Invoking handler .+",
"Handler .+ failed temporarily: oops",
])
async def test_retry_error_delays_handler(registry, handlers, extrahandlers,
resource, cause_mock, cause_type,
caplog, assert_logs, k8s_mocked):
caplog.set_level(logging.DEBUG)
name1 = f'{cause_type}_fn'
cause_mock.event = cause_type
handlers.create_mock.side_effect = TemporaryError("oops")
handlers.update_mock.side_effect = TemporaryError("oops")
handlers.delete_mock.side_effect = TemporaryError("oops")
handlers.resume_mock.side_effect = TemporaryError("oops")
await custom_object_handler(
lifecycle=kopf.lifecycles.one_by_one,
registry=registry,
resource=resource,
event={
'type': 'irrelevant',
'object': cause_mock.body
},
freeze=asyncio.Event(),
replenished=asyncio.Event(),
event_queue=asyncio.Queue(),
)
assert handlers.create_mock.call_count == (1
if cause_type == CREATE else 0)
assert handlers.update_mock.call_count == (1
if cause_type == UPDATE else 0)
assert handlers.delete_mock.call_count == (1
if cause_type == DELETE else 0)
assert handlers.resume_mock.call_count == (1
if cause_type == RESUME else 0)
assert not k8s_mocked.sleep_or_wait.called
assert k8s_mocked.patch_obj.called
patch = k8s_mocked.patch_obj.call_args_list[0][1]['patch']
assert patch['status']['kopf']['progress'] is not None
assert 'failure' not in patch['status']['kopf']['progress'][name1]
assert 'success' not in patch['status']['kopf']['progress'][name1]
assert 'delayed' in patch['status']['kopf']['progress'][name1]
assert_logs([
"Handler .+ failed temporarily: oops",
])
async def fn(retry, **kwargs):
dummy.mock()
dummy.kwargs = kwargs
dummy.steps['called'].set()
if not retry:
raise TemporaryError("boo!", delay=1.0)
else:
dummy.steps['finish'].set()
async def fn(retry, **kwargs):
dummy.mock()
dummy.kwargs = kwargs
dummy.steps['called'].set()
if not retry:
raise TemporaryError("boo!", delay=1.0)
else:
kwargs['stopped']._stopper.set(reason=kopf.DaemonStoppingReason.NONE) # to exit the cycle
dummy.steps['finish'].set()
def sample_fn(**_):
raise TemporaryError('to be retried', delay=123)
def sample_fn(**_):
mock()
raise TemporaryError('to be retried', delay=0)
async def fn(**kwargs):
dummy.kwargs = kwargs
dummy.steps['called'].set()
raise TemporaryError("boo!", delay=1.0)
assert 'warnings' not in response['response']
assert 'patchType' not in response['response']
assert 'patch' not in response['response']
assert response['response']['allowed'] is False
assert response['response']['status'] == {
'message': exp_msg,
'code': exp_code
}
@pytest.mark.parametrize('error1, error2, exp_msg', [
pytest.param(Exception("err1"),
Exception("err2"),
"err1",
id='builtin-first-samerank'),
pytest.param(TemporaryError("err1"),
TemporaryError("err2"),
"err1",
id='temp-first-samerank'),
pytest.param(PermanentError("err1"),
PermanentError("err2"),
"err1",
id='perm-first-samerank'),
pytest.param(AdmissionError("err1"),
AdmissionError("err2"),
"err1",
id='adms-first-samerank'),
pytest.param(Exception("err1"),
TemporaryError("err2"),
"err2",
id='temp-over-builtin'),