def check_with_ignore(self):
num_nodes = 200
@cluster(num_nodes=num_nodes)
@ignore
def function():
return "hi"
assert hasattr(function, "marks")
test_context_list = MarkedFunctionExpander(function=function).expand()
assert len(test_context_list) == 1
assert test_context_list[0].expected_num_nodes == num_nodes
# order shouldn't matter here
@ignore
@cluster(num_nodes=num_nodes)
def function():
return "hi"
assert hasattr(function, "marks")
test_context_list = MarkedFunctionExpander(function=function).expand()
assert len(test_context_list) == 1
assert test_context_list[0].expected_num_nodes == num_nodes
def _expand_function(self, t_ctx):
expander = MarkedFunctionExpander(
t_ctx.session_context,
t_ctx.module,
t_ctx.cls,
t_ctx.function,
t_ctx.file,
t_ctx.cluster)
return expander.expand(self.injected_args)
def _expand_function(self, t_ctx):
expander = MarkedFunctionExpander(
t_ctx.session_context,
t_ctx.module,
t_ctx.cls,
t_ctx.function,
t_ctx.file,
t_ctx.cluster)
return expander.expand(self.injected_args)
def check_for_inter_test_memory_leak(self):
"""Until v0.3.10, ducktape had a serious source of potential memory leaks.
Because test_context objects held a reference to all services for the duration of a test run, the memory
used by any individual service would not be garbage-collected until well after *all* tests had run.
This memory leak was discovered in Kafka system tests, where many long-running system tests were enough
to cumulatively use up the memory on the test machine, causing a cascade of test failures due to
inability to allocate any more memory.
This test provides a regression check against this type of memory leak; it fails without the fix, and passes
with it.
"""
# Get a list of test_context objects for the test runner
ctx_list = []
test_methods = [MemoryLeakTest.test_leak]
for f in test_methods:
ctx_list.extend(MarkedFunctionExpander(session_context=self.session_context, cls=MemoryLeakTest, function=f,
file=MEMORY_LEAK_TEST_FILE, cluster=self.cluster).expand())
assert len(ctx_list) == N_TEST_CASES # Sanity check
queue = Queue.Queue()
runner = InstrumentedTestRunner(self.cluster, self.session_context, Mock(), ctx_list, queue=queue)
runner.run_all_tests()
measurements = []
while not queue.empty():
measurements.append(queue.get())
self.validate_memory_measurements(measurements)
def check_defaults_method(self):
class C(object):
@defaults(z=[1, 2])
@matrix(x=[1], y=[1, 2])
@parametrize(x=3, y=4)
def function(self, x=1, y=2, z=-1):
return x, y, z
assert parametrized(C.function)
injected_args_list = [m.injected_args for m in C.function.marks]
assert len(injected_args_list) == 3
context_list = MarkedFunctionExpander(None,
function=C.function).expand()
assert len(context_list) == 6
expected_output = {(1, 1, 1), (1, 1, 2), (1, 2, 1), (1, 2, 2),
(3, 4, 1), (3, 4, 2)}
output = set()
c = C()
for ctx in context_list:
f = ctx.function
injected_args = ctx.injected_args
assert f(c) == (injected_args['x'], injected_args['y'],
injected_args['z'])
output.add(ctx.function(C()))
assert output == expected_output
def check_simple_run(self):
"""Check expected behavior when running a single test."""
mock_cluster = LocalhostCluster(num_nodes=1000)
session_context = tests.ducktape_mock.session_context()
test_methods = [
TestThingy.test_pi, TestThingy.test_ignore1,
TestThingy.test_ignore2
]
ctx_list = []
for f in test_methods:
ctx_list.extend(
MarkedFunctionExpander(session_context=session_context,
cls=TestThingy,
function=f,
file=TEST_THINGY_FILE,
cluster=mock_cluster).expand())
runner = TestRunner(mock_cluster, session_context, Mock(), ctx_list)
results = runner.run_all_tests()
assert len(results) == 3
assert results.num_failed == 0
assert results.num_passed == 1
assert results.num_ignored == 2
result_with_data = filter(lambda r: r.data is not None, results)[0]
assert result_with_data.data == {"data": 3.14159}
def check_with_nodes_default_parametrize_matrix(self):
default_num_nodes = 5
parametrize_num_nodes = 3
matrix_num_nodes = 7
@cluster(num_nodes=default_num_nodes)
@defaults(y=[5])
@parametrize(x=100, y=200)
@cluster(num_nodes=parametrize_num_nodes)
@parametrize(x=10, y=20)
@parametrize(x=30, y=40)
@cluster(num_nodes=matrix_num_nodes)
@matrix(x=[1, 2])
def f(x, y):
return x, y
test_context_list = MarkedFunctionExpander(function=f).expand()
assert len(test_context_list) == 5
# {'x': 1, 'y': 5} -> using matrix with matrix_num_nodes
assert test_context_list[0].expected_num_nodes == matrix_num_nodes
# {'x': 2, 'y': 5} -> using matrix with matrix_num_nodes
assert test_context_list[1].expected_num_nodes == matrix_num_nodes
# {'x': 30, 'y': 40} -> using parametrize with cluster parametrize_num_nodes
assert test_context_list[2].expected_num_nodes == parametrize_num_nodes
# {'x': 10, 'y': 20} -> using parametrize with cluster parametrize_num_nodes
assert test_context_list[3].expected_num_nodes == parametrize_num_nodes
# {'x': 100, 'y': 200} -> using parametrize with default_num_nodes
assert test_context_list[4].expected_num_nodes == default_num_nodes
def check_copy_constructor(self):
"""Regression test against a bug introduced in 0.3.7
The TestContext copy constructor was copying the ServiceRegistry object by reference.
As a result, services registering themselves with one test context would be registered with the copied
context as well, resulting in the length of the service registry to grow additively from test to test.
This problem cropped up in particular with parametrized tests.
"""
expander = MarkedFunctionExpander(session_context=session_context(), cls=DummyTest, function=DummyTest.test_me, cluster=MagicMock())
ctx_list = expander.expand()
for ctx in ctx_list:
# Constructing an instance of the test class causes a service to be registered with the test context
ctx.cls(ctx)
# Ensure that each context.services object is a unique reference
assert len(set(id(ctx.services) for ctx in ctx_list)) == len(ctx_list)
def expand_function(*, func, sess_ctx):
"""
Inject parameters into function and generate context list.
"""
assert parametrized(func)
assert next(filter(lambda x: isinstance(x, IgniteVersionParametrize), func.marks), None)
return MarkedFunctionExpander(session_context=sess_ctx, function=func).expand()
def expand_function(*, func, sess_ctx):
"""
Inject parameters into function and generate context list.
"""
assert hasattr(func, "marks")
assert next(filter(lambda x: isinstance(x, ParametrizableClusterMetadata), func.marks), None)
return MarkedFunctionExpander(session_context=sess_ctx, function=func).expand()
def check_simple(self):
@ignore
def function(x=1, y=2, z=3):
return x, y, z
assert ignored(function)
context_list = MarkedFunctionExpander(function=function).expand()
assert len(context_list) == 1
assert context_list[0].ignore
def check_is_ignored_if_env_not_correct(self):
class C(object):
@env(JAVA_HOME="blah")
def function(self):
return 1
context_list = MarkedFunctionExpander(function=C.function,
cls=C).expand()
assert context_list[0].ignore
def check_simple_method(self):
class C(object):
@ignore
def function(self, x=1, y=2, z=3):
return x, y, z
assert ignored(C.function)
context_list = MarkedFunctionExpander(function=C.function, cls=C).expand()
assert len(context_list) == 1
assert context_list[0].ignore
def check_is_not_ignore_if_correct_env(self):
os.environ['test_key'] = 'test'
class C(object):
@env(test_key='test')
def function(self):
return 1
context_list = MarkedFunctionExpander(function=C.function,
cls=C).expand()
assert not context_list[0].ignore
def check_with_parametrize(self):
num_nodes = 200
@cluster(num_nodes=num_nodes)
@parametrize(x=1)
def f(x, y=2, z=3):
return x, y, z
test_context_list = MarkedFunctionExpander(function=f).expand()
assert len(test_context_list) == 1
assert test_context_list[0].expected_num_nodes == num_nodes
def check_basic_usage_num_nodes(self):
num_nodes = 200
@cluster(num_nodes=num_nodes)
def function():
return "hi"
assert hasattr(function, "marks")
test_context_list = MarkedFunctionExpander(function=function).expand()
assert len(test_context_list) == 1
assert test_context_list[0].expected_num_nodes == num_nodes
def check_basic_usage_arbitrary_metadata(self):
cluster_use_metadata = {'a': 2, 'b': 'hi', 'num_nodes': 300}
@cluster(**cluster_use_metadata)
def function():
return "hi"
assert hasattr(function, "marks")
test_context_list = MarkedFunctionExpander(function=function).expand()
assert len(test_context_list) == 1
assert test_context_list[
0].cluster_use_metadata == cluster_use_metadata
def check_invalid_ignore_all(self):
"""If there are no test cases to which ignore applies, it should raise an error"""
class C(object):
@parametrize(x=100, y=200, z=300)
@parametrize(x=100, z=300)
@parametrize(y=200)
@parametrize()
@ignore
def function(self, x=1, y=2, z=3):
return x, y, z
assert ignored(C.function)
with pytest.raises(AssertionError):
MarkedFunctionExpander(function=C.function, cls=C).expand()
def check_ignore_all(self):
@ignore
@parametrize(x=100, y=200, z=300)
@parametrize(x=100, z=300)
@parametrize(y=200)
@parametrize()
def function(x=1, y=2, z=3):
return x, y, z
assert ignored(function)
context_list = MarkedFunctionExpander(function=function).expand()
assert len(context_list) == 4
for ctx in context_list:
assert ctx.ignore
def check_simple(self):
@parametrize(x=100, z=300)
def function(x=1, y=2, z=3):
return x, y, z
assert parametrized(function)
injected_args_list = [m.injected_args for m in function.marks]
assert len(injected_args_list) == 1
assert injected_args_list[0] == {"x": 100, "z": 300}
context_list = MarkedFunctionExpander(function=function).expand()
all_f = [cxt.function for cxt in context_list]
assert all_f[0]() == (100, 2, 300)
def _do_expand(self,
test_file,
test_class,
test_methods,
cluster=None,
session_context=None):
ctx_list = []
for f in test_methods:
ctx_list.extend(
MarkedFunctionExpander(session_context=session_context,
cls=test_class,
function=f,
file=test_file,
cluster=cluster).expand())
return ctx_list
def check_invalid_specific_ignore(self):
"""If there are no test cases to which ignore applies, it should raise an error
Keeping in mind annotations "point down": they only apply to test cases physically below.
"""
class C(object):
@parametrize(x=100, y=200, z=300)
@parametrize(x=100, z=300)
@parametrize(y=200)
@parametrize()
@ignore(x=100, y=200, z=300)
def function(self, x=1, y=2, z=3):
return x, y, z
assert ignored(C.function)
with pytest.raises(AssertionError):
MarkedFunctionExpander(function=C.function, cls=C).expand()
def check_simple(self):
@matrix(x=[1, 2], y=[-1, -2])
def function(x=1, y=2, z=3):
return x, y, z
assert parametrized(function)
injected_args_list = [m.injected_args for m in function.marks]
assert len(injected_args_list) == 1
context_list = MarkedFunctionExpander(None, function=function).expand()
assert len(context_list) == 4
for ctx in context_list:
f = ctx.function
injected_args = ctx.injected_args
assert f() == (injected_args['x'], injected_args['y'], 3)
def check_basic_usage_cluster_spec(self):
num_nodes = 200
@cluster(cluster_spec=ClusterSpec.simple_linux(num_nodes))
def function():
return "hi"
assert hasattr(function, "marks")
test_context_list = MarkedFunctionExpander(function=function).expand()
assert len(test_context_list) == 1
assert len(
test_context_list[0].expected_cluster_spec.nodes.os_to_nodes) == 1
assert len(
test_context_list[0].expected_cluster_spec.nodes.os_to_nodes.get(
'linux')) == num_nodes
def check_ignore_specific(self):
@ignore(x=100, y=200, z=300)
@parametrize(x=100, y=200, z=300)
@parametrize(x=100, z=300)
@parametrize(y=200)
@parametrize()
def function(x=1, y=2, z=3):
return x, y, z
assert ignored(function)
context_list = MarkedFunctionExpander(None, function=function).expand()
assert len(context_list) == 4
for ctx in context_list:
if ctx.injected_args == {"x": 100, "y": 200, "z": 300}:
assert ctx.ignore
else:
assert not ctx.ignore
def check_ignore_all_method(self):
"""Check @ignore() with no arguments used with various parametrizations on a method."""
class C(object):
@ignore
@parametrize(x=100, y=200, z=300)
@parametrize(x=100, z=300)
@parametrize(y=200)
@matrix(x=[1, 2, 3])
@parametrize()
def function(self, x=1, y=2, z=3):
return x, y, z
assert ignored(C.function)
context_list = MarkedFunctionExpander(function=C.function, cls=C).expand()
assert len(context_list) == 7
for ctx in context_list:
assert ctx.ignore
def check_no_override(self):
""" cluster use metadata should apply to all test cases physically below it, except for those which already
have cluster use metadata.
"""
num_nodes_1 = 200
num_nodes_2 = 42
# num_nodes_2 should *not* override num_nodes_1
@cluster(num_nodes=num_nodes_2)
@cluster(num_nodes=num_nodes_1)
def f(x, y=2, z=3):
return x, y, z
test_context_list = MarkedFunctionExpander(function=f).expand()
assert len(test_context_list) == 1
assert test_context_list[0].expected_num_nodes == num_nodes_1
def check_beneath_parametrize(self):
""" Annotations such as cluster, which add metadata, but do not create new tests, add the metadata to
all test cases physically below the annotation.
In the case of a parametrized test, when @cluster has no parametrize annotations below it,
there are not any test cases to which it will apply, so none of the resulting tests should have
associated cluster size metadata.
"""
num_nodes = 200
@parametrize(x=1)
@cluster(num_nodes=num_nodes)
def f(x, y=2, z=3):
return x, y, z
with pytest.raises(AssertionError):
MarkedFunctionExpander(function=f).expand()
def check_parametrized_with_multiple_cluster_annotations(self):
num_nodes_1 = 10
num_nodes_2 = 20
# num_nodes_1 should *not* override num_nodes_2
@cluster(num_nodes=num_nodes_1)
@parametrize(x=1)
@parametrize(x=1.5)
@cluster(num_nodes=num_nodes_2)
@parametrize(x=2)
def f(x, y=2, z=3):
return x, y, z
test_context_list = MarkedFunctionExpander(function=f).expand()
assert len(test_context_list) == 3
assert test_context_list[0].expected_num_nodes == num_nodes_1
assert test_context_list[1].expected_num_nodes == num_nodes_1
assert test_context_list[2].expected_num_nodes == num_nodes_2
def check_stacked(self):
@parametrize(x=100, y=200, z=300)
@parametrize(x=100, z=300)
@parametrize(y=200)
@parametrize()
def function(x=1, y=2, z=3):
return x, y, z
assert parametrized(function)
injected_args_list = [m.injected_args for m in function.marks]
assert len(injected_args_list) == 4
context_list = MarkedFunctionExpander(None, function=function).expand()
all_f = [cxt.function for cxt in context_list]
assert all_f[0]() == (100, 200, 300)
assert all_f[1]() == (100, 2, 300)
assert all_f[2]() == (1, 200, 3)
assert all_f[3]() == (1, 2, 3)
def check_only_defaults(self):
@defaults(x=[3], z=[1, 2])
def function(x=1, y=2, z=-1):
return x, y, z
assert parametrized(function)
injected_args_list = [m.injected_args for m in function.marks]
assert len(injected_args_list) == 1
context_list = MarkedFunctionExpander(None, function=function).expand()
assert len(context_list) == 2
expected_output = {(3, 2, 1), (3, 2, 2)}
output = set()
for ctx in context_list:
output.add(ctx.function())
assert output == expected_output