def kopf_runner(kube):
logger = logging.getLogger()
old_handlers = logger.handlers[:]
args = [
"--verbose",
"--standalone",
"--namespace",
kube._ns,
"-m",
"secret_sync.handlers",
]
# Set the kopf watcher stream timeout to something small so we don't have
# to wait too long at the end of the tests for all the background watcher
# threads to end.
settings = kopf.OperatorSettings()
settings.watching.server_timeout = 1
try:
with KopfRunner(["run", *args], settings=settings) as runner:
# Remove any extra log handlers that starting kopf may have added.
# The built-in pytest log capture does what we need already.
for handler in logger.handlers[:]:
if handler not in old_handlers:
logger.removeHandler(handler)
yield runner
finally:
# The runner captures all output, so print it for pytest to capture.
print(runner.stdout)
def test_resource_lifecycle():
# Run an operator and simulate some activity with the operated resource.
with KopfRunner(["run", "--verbose", "--standalone", handler_py],
timeout=60) as runner:
for file in os.listdir(input_dir):
input_yaml = os.path.join(input_dir, file)
subprocess.run(f"kubectl apply -f {input_yaml}",
shell=True,
check=True,
timeout=10,
capture_output=True)
time.sleep(40) # give it some time to react
subprocess.run(f"kubectl delete -f {input_yaml}",
shell=True,
check=True,
timeout=10,
capture_output=True)
time.sleep(40) # give it some time to react
# Ensure that the operator did not die on start, or during the operation.
assert runner.exception is None
assert runner.exit_code == 0
def test_exception_from_invoke_escalates(mocker, kwargs):
class SampleError(Exception):
pass
mocker.patch('click.testing.CliRunner.invoke', side_effect=SampleError)
with pytest.raises(SampleError):
with KopfRunner(['run', 'non-existent.py', '--standalone'], **kwargs):
pass
def test_all_examples_are_runnable(mocker, with_crd, with_peering, exampledir):
# If the example has its own opinion on the timing, try to respect it.
# See e.g. /examples/99-all-at-once/example.py.
example_py = exampledir / 'example.py'
m = re.search(r'^E2E_CREATE_TIME\s*=\s*(\d+)$', example_py.read_text(),
re.M)
e2e_create_time = eval(m.group(1)) if m else None
m = re.search(r'^E2E_DELETE_TIME\s*=\s*(\d+)$', example_py.read_text(),
re.M)
e2e_delete_time = eval(m.group(1)) if m else None
m = re.search(r'^E2E_TRACEBACKS\s*=\s*(\w+)$', example_py.read_text(),
re.M)
e2e_tracebacks = eval(m.group(1)) if m else None
# check whether there are mandatory deletion handlers or not
m = re.search(r'@kopf\.on\.delete\((\s|.*)?(optional=(\w+))?\)',
example_py.read_text(), re.M)
requires_finalizer = False
if m:
requires_finalizer = True
if m.group(2):
requires_finalizer = not eval(m.group(3))
# To prevent lengthy sleeps on the simulated retries.
mocker.patch('kopf.reactor.handling.DEFAULT_RETRY_DELAY', 1)
# To prevent lengthy threads in the loop executor when the process exits.
mocker.patch('kopf.clients.watching.DEFAULT_STREAM_TIMEOUT', 10)
# Run an operator and simulate some activity with the operated resource.
with KopfRunner(['run', '--verbose', str(example_py)]) as runner:
subprocess.run("kubectl apply -f examples/obj.yaml",
shell=True,
check=True)
time.sleep(
e2e_create_time
or 1) # give it some time to react and to sleep and to retry
subprocess.run("kubectl delete -f examples/obj.yaml",
shell=True,
check=True)
time.sleep(e2e_delete_time or 1) # give it some time to react
# Ensure that the operator did not die on start, or during the operation.
assert runner.exception is None
assert runner.exit_code == 0
# There are usually more than these messages, but we only check for the certain ones.
# This just shows us that the operator is doing something, it is alive.
if requires_finalizer:
assert '[default/kopf-example-1] Adding the finalizer' in runner.stdout
assert '[default/kopf-example-1] Creation event:' in runner.stdout
if requires_finalizer:
assert '[default/kopf-example-1] Deletion event:' in runner.stdout
assert '[default/kopf-example-1] Deleted, really deleted' in runner.stdout
if not e2e_tracebacks:
assert 'Traceback (most recent call last):' not in runner.stdout
def test_command_invocation_works():
with KopfRunner(['--help']) as runner:
pass
assert runner.exc_info
assert runner.exc_info[0] is SystemExit
assert runner.exc_info[1].code == 0
assert runner.exit_code == 0
assert runner.exception is None
assert runner.output.startswith("Usage:")
assert runner.stdout.startswith("Usage:")
assert runner.stdout_bytes.startswith(b"Usage:")
def kopf_runner(request, cratedb_crd):
# Make all handlers available to the runner
from crate.operator import main
# Make sure KUBECONFIG env variable is set because KOPF depends on it
env = {
"CRATEDB_OPERATOR_KUBECONFIG":
request.config.getoption(KUBECONFIG_OPTION),
}
with mock.patch.dict(os.environ, env):
with KopfRunner(["run", "--standalone", main.__file__]) as runner:
yield runner
def test_registry_and_settings_are_propagated(mocker):
operator_mock = mocker.patch('kopf.reactor.running.operator')
registry = OperatorRegistry()
settings = OperatorSettings()
with KopfRunner(['run', '--standalone'],
registry=registry,
settings=settings) as runner:
pass
assert runner.exit_code == 0
assert runner.exception is None
assert operator_mock.called
assert operator_mock.call_args[1]['registry'] is registry
assert operator_mock.call_args[1]['settings'] is settings
def test_create_delete_run():
with KopfRunner(['run', 'nauta_operator.py'], timeout=10) as runner:
subprocess.run(
['kubectl', 'delete', '-f', 'example_runs/test_run.yaml'],
check=False)
time.sleep(2)
subprocess.run(
['kubectl', 'apply', '-f', 'example_runs/test_run.yaml'],
check=True)
time.sleep(2)
subprocess.run(
['kubectl', 'delete', '-f', 'example_runs/test_run.yaml'],
check=True)
time.sleep(2)
assert runner.exit_code == 0
assert runner.exception is None
assert 'Run test-run created.' in runner.stdout
assert 'Run test-run deleted.' in runner.stdout
def test_operator():
'''
Tester for the operator.
'''
with KopfRunner(['run', '--verbose', 'examples/example.py']) as runner:
# do something while the operator is running.
subprocess.run("kubectl apply -f examples/obj.yaml",
shell=True,
check=True)
# time.sleep(1) # give it some time to react and to sleep and to retry
subprocess.run("kubectl delete -f examples/obj.yaml",
shell=True,
check=True)
# time.sleep(1) # give it some time to react
assert runner.exit_code == 0
assert runner.exception is None
assert 'And here we are!' in runner.stdout
assert 'Deleted, really deleted' in runner.stdout
def test_all_examples_are_runnable(mocker, with_crd, exampledir):
# If the example has its own opinion on the timing, try to respect it.
# See e.g. /examples/99-all-at-once/example.py.
example_py = exampledir / 'example.py'
m = re.search(r'^E2E_CREATE_TIME\s*=\s*(.+)$', example_py.read_text(), re.M)
e2e_create_time = eval(m.group(1)) if m else None
m = re.search(r'^E2E_DELETE_TIME\s*=\s*(.+)$', example_py.read_text(), re.M)
e2e_delete_time = eval(m.group(1)) if m else None
m = re.search(r'^E2E_TRACEBACKS\s*=\s*(.+)$', example_py.read_text(), re.M)
e2e_tracebacks = eval(m.group(1)) if m else None
m = re.search(r'^E2E_SUCCESS_COUNTS\s*=\s*(.+)$', example_py.read_text(), re.M)
e2e_success_counts = eval(m.group(1)) if m else None
m = re.search(r'^E2E_FAILURE_COUNTS\s*=\s*(.+)$', example_py.read_text(), re.M)
e2e_failure_counts = eval(m.group(1)) if m else None
m = re.search(r'@kopf.on.create\(', example_py.read_text(), re.M)
e2e_test_creation = bool(m)
m = re.search(r'@kopf.on.(create|update|delete)\(', example_py.read_text(), re.M)
e2e_test_highlevel = bool(m)
# check whether there are mandatory deletion handlers or not
m = re.search(r'@kopf\.on\.delete\((\s|.*)?(optional=(\w+))?\)', example_py.read_text(), re.M)
requires_finalizer = False
if m:
requires_finalizer = True
if m.group(2):
requires_finalizer = not eval(m.group(3))
# Skip the e2e test if the framework-optional but test-required library is missing.
m = re.search(r'import kubernetes', example_py.read_text(), re.M)
if m:
pytest.importorskip('kubernetes')
# To prevent lengthy sleeps on the simulated retries.
mocker.patch('kopf.reactor.handling.DEFAULT_RETRY_DELAY', 1)
# To prevent lengthy threads in the loop executor when the process exits.
mocker.patch('kopf.config.WatchersConfig.default_stream_timeout', 10)
# Run an operator and simulate some activity with the operated resource.
with KopfRunner(['run', '--standalone', '--verbose', str(example_py)], timeout=60) as runner:
subprocess.run("kubectl apply -f examples/obj.yaml", shell=True, check=True)
time.sleep(e2e_create_time or 2) # give it some time to react and to sleep and to retry
subprocess.run("kubectl delete -f examples/obj.yaml", shell=True, check=True)
time.sleep(e2e_delete_time or 1) # give it some time to react
# Verify that the operator did not die on start, or during the operation.
assert runner.exception is None
assert runner.exit_code == 0
# There are usually more than these messages, but we only check for the certain ones.
# This just shows us that the operator is doing something, it is alive.
if requires_finalizer:
assert '[default/kopf-example-1] Adding the finalizer' in runner.stdout
if e2e_test_creation:
assert '[default/kopf-example-1] Creation event:' in runner.stdout
if requires_finalizer:
assert '[default/kopf-example-1] Deletion event:' in runner.stdout
if e2e_test_highlevel:
assert '[default/kopf-example-1] Deleted, really deleted' in runner.stdout
if not e2e_tracebacks:
assert 'Traceback (most recent call last):' not in runner.stdout
# Verify that once a handler succeeds, it is never re-executed again.
handler_names = re.findall(r"Handler '(.+?)' succeeded", runner.stdout)
if e2e_success_counts is not None:
checked_names = [name for name in handler_names if name in e2e_success_counts]
name_counts = collections.Counter(checked_names)
assert name_counts == e2e_success_counts
else:
name_counts = collections.Counter(handler_names)
assert set(name_counts.values()) == {1}
# Verify that once a handler fails, it is never re-executed again.
handler_names = re.findall(r"Handler '(.+?)' failed permanently", runner.stdout)
if e2e_failure_counts is not None:
checked_names = [name for name in handler_names if name in e2e_failure_counts]
name_counts = collections.Counter(checked_names)
assert name_counts == e2e_failure_counts
else:
name_counts = collections.Counter(handler_names)
assert not name_counts
def test_absent_file_fails():
with pytest.raises(FileNotFoundError):
with KopfRunner(['run', 'non-existent.py', '--standalone']):
pass
def test_bad_syntax_file_fails(tmpdir):
path = tmpdir.join('handlers.py')
path.write("""This is a Python syntax error!""")
with pytest.raises((IndentationError, SyntaxError)):
with KopfRunner(['run', str(path), '--standalone']):
pass
def test_kopfrunner(self):
api = pykube.HTTPClient(pykube.KubeConfig.from_env())
doc = {
'apiVersion': 'v1',
'kind': 'Pod',
'metadata': {
'generateName': 'oaat-testing-',
'namespace': 'default',
'annotations': {
'kawaja.net/testannotation': 'annotationvalue'
},
'labels': {
'testlabel': 'labelvalue'
}
},
'spec': {
'containers': [{
'name': 'oaat-testing',
'image': 'busybox',
'command': ['sleep', '50']
}
],
'restartPolicy': 'Never'
},
}
pod = pykube.Pod(api, doc)
with KopfRunner([
'run', '--namespace=default', '--verbose',
'tests/operator_overseer.py']) as runner:
pod.create()
time.sleep(1)
pod.reload()
annotations1 = deepcopy(pod.annotations)
pod.annotations['readytodelete'] = 'true'
pod.update()
time.sleep(3)
try:
pod.reload()
except pykube.exceptions.HTTPError as exc:
self.assertRegex(str(exc), f'"{pod.name}" not found', exc)
self.maxDiff = None
self.assertEqual(runner.exit_code, 0)
self.assertIsNone(runner.exception)
self.assertRegex(runner.stdout, r'all overseer tests successful')
self.assertRegex(runner.stdout, r'\[1\] successful')
self.assertRegex(runner.stdout, r'\[8\] successful')
self.assertRegex(runner.stdout, r'\[9\] successful')
self.assertRegex(runner.stdout, r'\[10\] successful')
self.assertRegex(runner.stdout, r'ERROR.*error message')
self.assertRegex(runner.stdout, r'WARNING.*warning message')
self.assertRegex(runner.stdout, r'INFO.*info message')
self.assertRegex(runner.stdout, r'DEBUG.*debug message')
self.assertRegex(
runner.stdout, r'Patching with.*new_status.: None')
self.assertRegex(
runner.stdout, r'Patching with.*new_status2.: .new_state.')
self.assertRegex(
runner.stdout, r'removed annotation testannotation')
self.assertEqual(
annotations1.get(
'kawaja.net/testannotation', 'missing'),
'missing')
self.assertRegex(
runner.stdout,
r'added annotation new_annotation=annotation_value')
self.assertEqual(
annotations1['kawaja.net/new_annotation'],
'annotation_value')
self.assertRegex(runner.stdout, r'ERROR.*reterror')
self.assertRegex(runner.stdout, r'WARNING.*retwarning')
self.assertRegex(runner.stdout, r'INFO.*retinfo')
self.assertRegex(
runner.stdout, r'status.: {[^{]*.state.: .retstate.')
self.assertRegex(runner.stdout, r'\[12\] successful')
self.assertRegex(runner.stdout, r'\[13\] successful')
def test_wrong_command_fails():
with pytest.raises(SystemExit) as e:
with KopfRunner(['unexistent-command']):
pass
assert e.value.code == 2
def test_exception_from_runner_escalates_by_default():
with pytest.raises(FileNotFoundError):
with KopfRunner(['run', 'non-existent.py', '--standalone']):
pass
def test_exception_from_runner_escalates_with_reraise():
with pytest.raises(FileNotFoundError):
with KopfRunner(['run', 'non-existent.py', '--standalone'],
reraise=True):
pass
def test_exception_from_runner_suppressed_with_no_reraise():
with KopfRunner(['run', 'non-existent.py', '--standalone'],
reraise=False) as runner:
pass
assert runner.exception is not None
assert isinstance(runner.exception, FileNotFoundError)
def test_all_examples_are_runnable(mocker, settings, with_crd, exampledir,
caplog):
# If the example has its own opinion on the timing, try to respect it.
# See e.g. /examples/99-all-at-once/example.py.
example_py = exampledir / 'example.py'
e2e_startup_time_limit = _parse_e2e_value(str(example_py),
'E2E_STARTUP_TIME_LIMIT')
e2e_startup_stop_words = _parse_e2e_value(str(example_py),
'E2E_STARTUP_STOP_WORDS')
e2e_cleanup_time_limit = _parse_e2e_value(str(example_py),
'E2E_CLEANUP_TIME_LIMIT')
e2e_cleanup_stop_words = _parse_e2e_value(str(example_py),
'E2E_CLEANUP_STOP_WORDS')
e2e_creation_time_limit = _parse_e2e_value(str(example_py),
'E2E_CREATION_TIME_LIMIT')
e2e_creation_stop_words = _parse_e2e_value(str(example_py),
'E2E_CREATION_STOP_WORDS')
e2e_deletion_time_limit = _parse_e2e_value(str(example_py),
'E2E_DELETION_TIME_LIMIT')
e2e_deletion_stop_words = _parse_e2e_value(str(example_py),
'E2E_DELETION_STOP_WORDS')
e2e_tracebacks = _parse_e2e_value(str(example_py), 'E2E_TRACEBACKS')
e2e_success_counts = _parse_e2e_value(str(example_py),
'E2E_SUCCESS_COUNTS')
e2e_failure_counts = _parse_e2e_value(str(example_py),
'E2E_FAILURE_COUNTS')
e2e_test_creation = _parse_e2e_presence(str(example_py),
r'@kopf.on.create\(')
e2e_test_highlevel = _parse_e2e_presence(
str(example_py), r'@kopf.on.(create|update|delete)\(')
# check whether there are mandatory deletion handlers or not
m = re.search(r'@kopf\.on\.delete\((\s|.*)?(optional=(\w+))?\)',
example_py.read_text(), re.M)
requires_finalizer = False
if m:
requires_finalizer = True
if m.group(2):
requires_finalizer = not eval(m.group(3))
# Skip the e2e test if the framework-optional but test-required library is missing.
m = re.search(r'import kubernetes', example_py.read_text(), re.M)
if m:
pytest.importorskip('kubernetes')
# To prevent lengthy sleeps on the simulated retries.
mocker.patch('kopf.reactor.handling.DEFAULT_RETRY_DELAY', 1)
# To prevent lengthy threads in the loop executor when the process exits.
settings.watching.server_timeout = 10
# Run an operator and simulate some activity with the operated resource.
with KopfRunner(['run', '--standalone', '--verbose',
str(example_py)],
timeout=60) as runner:
# Give it some time to start.
_sleep_till_stopword(caplog=caplog,
delay=e2e_startup_time_limit,
patterns=e2e_startup_stop_words
or ['Client is configured'])
# Trigger the reaction. Give it some time to react and to sleep and to retry.
subprocess.run("kubectl apply -f examples/obj.yaml",
shell=True,
check=True,
timeout=10,
capture_output=True)
_sleep_till_stopword(caplog=caplog,
delay=e2e_creation_time_limit,
patterns=e2e_creation_stop_words)
# Trigger the reaction. Give it some time to react.
subprocess.run("kubectl delete -f examples/obj.yaml",
shell=True,
check=True,
timeout=10,
capture_output=True)
_sleep_till_stopword(caplog=caplog,
delay=e2e_deletion_time_limit,
patterns=e2e_deletion_stop_words)
# Give it some time to finish.
_sleep_till_stopword(caplog=caplog,
delay=e2e_cleanup_time_limit,
patterns=e2e_cleanup_stop_words
or ['Hung tasks', 'Root tasks'])
# Verify that the operator did not die on start, or during the operation.
assert runner.exception is None
assert runner.exit_code == 0
# There are usually more than these messages, but we only check for the certain ones.
# This just shows us that the operator is doing something, it is alive.
if requires_finalizer:
assert '[default/kopf-example-1] Adding the finalizer' in runner.stdout
if e2e_test_creation:
assert '[default/kopf-example-1] Creation event:' in runner.stdout
if requires_finalizer:
assert '[default/kopf-example-1] Deletion event:' in runner.stdout
if e2e_test_highlevel:
assert '[default/kopf-example-1] Deleted, really deleted' in runner.stdout
if not e2e_tracebacks:
assert 'Traceback (most recent call last):' not in runner.stdout
# Verify that once a handler succeeds, it is never re-executed again.
handler_names = re.findall(r"'(.+?)' succeeded", runner.stdout)
if e2e_success_counts is not None:
checked_names = [
name for name in handler_names if name in e2e_success_counts
]
name_counts = collections.Counter(checked_names)
assert name_counts == e2e_success_counts
else:
name_counts = collections.Counter(handler_names)
assert set(name_counts.values()) == {1}
# Verify that once a handler fails, it is never re-executed again.
handler_names = re.findall(
r"'(.+?)' failed (?:permanently|with an exception. Will stop.)",
runner.stdout)
if e2e_failure_counts is not None:
checked_names = [
name for name in handler_names if name in e2e_failure_counts
]
name_counts = collections.Counter(checked_names)
assert name_counts == e2e_failure_counts
else:
name_counts = collections.Counter(handler_names)
assert not name_counts
def test_all_examples_are_runnable(mocker, settings, with_crd, exampledir,
caplog):
# If the example has its own opinion on the timing, try to respect it.
# See e.g. /examples/99-all-at-once/example.py.
example_py = exampledir / 'example.py'
e2e = E2EParser(str(example_py))
# Skip the e2e test if the framework-optional but test-required library is missing.
if e2e.imports_kubernetes:
pytest.importorskip('kubernetes')
# To prevent lengthy sleeps on the simulated retries.
mocker.patch('kopf._core.actions.execution.DEFAULT_RETRY_DELAY', 1)
# To prevent lengthy threads in the loop executor when the process exits.
settings.watching.server_timeout = 10
# Run an operator and simulate some activity with the operated resource.
with KopfRunner(
[
'run', '--all-namespaces', '--standalone', '--verbose',
str(example_py)
],
timeout=60,
) as runner:
# Give it some time to start.
_sleep_till_stopword(caplog=caplog,
delay=e2e.startup_time_limit,
patterns=e2e.startup_stop_words
or ['Client is configured'])
# Trigger the reaction. Give it some time to react and to sleep and to retry.
subprocess.run("kubectl apply -f examples/obj.yaml",
shell=True,
check=True,
timeout=10,
capture_output=True)
_sleep_till_stopword(caplog=caplog,
delay=e2e.creation_time_limit,
patterns=e2e.creation_stop_words)
# Trigger the reaction. Give it some time to react.
subprocess.run("kubectl delete -f examples/obj.yaml",
shell=True,
check=True,
timeout=10,
capture_output=True)
_sleep_till_stopword(caplog=caplog,
delay=e2e.deletion_time_limit,
patterns=e2e.deletion_stop_words)
# Give it some time to finish.
_sleep_till_stopword(caplog=caplog,
delay=e2e.cleanup_time_limit,
patterns=e2e.cleanup_stop_words
or ['Hung tasks', 'Root tasks'])
# Verify that the operator did not die on start, or during the operation.
assert runner.exception is None
assert runner.exit_code == 0
# There are usually more than these messages, but we only check for the certain ones.
# This just shows us that the operator is doing something, it is alive.
if e2e.has_mandatory_on_delete:
assert '[default/kopf-example-1] Adding the finalizer' in runner.stdout
if e2e.has_on_create:
assert '[default/kopf-example-1] Creation is in progress:' in runner.stdout
if e2e.has_mandatory_on_delete:
assert '[default/kopf-example-1] Deletion is in progress:' in runner.stdout
if e2e.has_changing_handlers:
assert '[default/kopf-example-1] Deleted, really deleted' in runner.stdout
if not e2e.allow_tracebacks:
assert 'Traceback (most recent call last):' not in runner.stdout
# Verify that once a handler succeeds, it is never re-executed again.
handler_names = re.findall(r"'(.+?)' succeeded", runner.stdout)
if e2e.success_counts is not None:
checked_names = [
name for name in handler_names if name in e2e.success_counts
]
name_counts = collections.Counter(checked_names)
assert name_counts == e2e.success_counts
else:
name_counts = collections.Counter(handler_names)
assert set(name_counts.values()) == {1}
# Verify that once a handler fails, it is never re-executed again.
handler_names = re.findall(
r"'(.+?)' failed (?:permanently|with an exception. Will stop.)",
runner.stdout)
if e2e.failure_counts is not None:
checked_names = [
name for name in handler_names if name in e2e.failure_counts
]
name_counts = collections.Counter(checked_names)
assert name_counts == e2e.failure_counts
else:
name_counts = collections.Counter(handler_names)
assert not name_counts
