class CheckAllocateFree(object):
def setup_method(self, _):
self.cluster = LocalhostCluster()
self.session_context = session_context()
self.context = test_context(self.session_context, cluster=self.cluster)
def check_allocate_free(self):
"""Check that allocating and freeing nodes works.
This regression test catches the error with Service.free() introduced in v0.3.3 and fixed in v0.3.4
"""
# Node allocation takes place during service instantiation
initial_cluster_size = len(self.cluster)
self.service = DummyService(self.context, 10)
assert self.cluster.num_available_nodes() == initial_cluster_size - 10
self.service.free()
assert self.cluster.num_available_nodes() == initial_cluster_size
def check_order(self):
"""Check expected behavior with service._order method"""
self.dummy0 = DummyService(self.context, 4)
self.diffDummy0 = DifferentDummyService(self.context, 100)
self.dummy1 = DummyService(self.context, 1)
self.diffDummy1 = DifferentDummyService(self.context, 2)
self.diffDummy2 = DifferentDummyService(self.context, 5)
assert self.dummy0._order == 0
assert self.dummy1._order == 1
assert self.diffDummy0._order == 0
assert self.diffDummy1._order == 1
assert self.diffDummy2._order == 2
class CheckAllocateFree(object):
def setup_method(self, method):
self.cluster = LocalhostCluster()
self.session_context = session_context(cluster=self.cluster)
self.context = test_context(self.session_context)
def check_allocate_free(self):
"""Check that allocating and freeing nodes works.
This regression test catches the error with Service.free() introduced in v0.3.3 and fixed in v0.3.4
"""
# Node allocation takes place during service instantiation
initial_cluster_size = len(self.cluster)
self.service = DummyService(self.context, 10)
assert self.cluster.num_available_nodes() == initial_cluster_size - 10
self.service.free()
assert self.cluster.num_available_nodes() == initial_cluster_size
def check_order(self):
"""Check expected behavior with service._order method"""
self.dummy0 = DummyService(self.context, 4)
self.diffDummy0 = DifferentDummyService(self.context, 100)
self.dummy1 = DummyService(self.context, 1)
self.diffDummy1 = DifferentDummyService(self.context, 2)
self.diffDummy2 = DifferentDummyService(self.context, 5)
assert self.dummy0._order == 0
assert self.dummy1._order == 1
assert self.diffDummy0._order == 0
assert self.diffDummy1._order == 1
assert self.diffDummy2._order == 2
def check_simple_run(self):
"""Check expected behavior when running a single test."""
mock_cluster = LocalhostCluster(num_nodes=1000)
session_context = tests.ducktape_mock.session_context()
test_methods = [
TestThingy.test_pi, TestThingy.test_ignore1,
TestThingy.test_ignore2, TestThingy.test_failure
]
ctx_list = self._do_expand(test_file=TEST_THINGY_FILE,
test_class=TestThingy,
test_methods=test_methods,
cluster=mock_cluster,
session_context=session_context)
runner = TestRunner(mock_cluster, session_context, Mock(), ctx_list, 1)
results = runner.run_all_tests()
assert len(results) == 4
assert results.num_flaky == 0
assert results.num_failed == 1
assert results.num_passed == 1
assert results.num_ignored == 2
result_with_data = [r for r in results if r.data is not None][0]
assert result_with_data.data == {"data": 3.14159}
def check_simple_run(self):
"""Check expected behavior when running a single test."""
mock_cluster = LocalhostCluster(num_nodes=1000)
session_context = tests.ducktape_mock.session_context()
test_methods = [
TestThingy.test_pi, TestThingy.test_ignore1,
TestThingy.test_ignore2
]
ctx_list = []
for f in test_methods:
ctx_list.extend(
MarkedFunctionExpander(session_context=session_context,
cls=TestThingy,
function=f,
file=TEST_THINGY_FILE,
cluster=mock_cluster).expand())
runner = TestRunner(mock_cluster, session_context, Mock(), ctx_list)
results = runner.run_all_tests()
assert len(results) == 3
assert results.num_failed == 0
assert results.num_passed == 1
assert results.num_ignored == 2
result_with_data = filter(lambda r: r.data is not None, results)[0]
assert result_with_data.data == {"data": 3.14159}
def check_test_failure_with_too_many_nodes_requested(self):
mock_cluster = LocalhostCluster(num_nodes=1000)
session_context = tests.ducktape_mock.session_context(debug=True)
ctx_list = self._do_expand(
test_file=BAD_ACTOR_TEST_FILE,
test_class=BadActorTest,
test_methods=[BadActorTest.test_too_many_nodes],
cluster=mock_cluster,
session_context=session_context)
ctx_list.extend(
self._do_expand(test_file=VARIOUS_NUM_NODES_TEST_FILE,
test_class=VariousNumNodesTest,
test_methods=[VariousNumNodesTest.test_one_node_a],
cluster=mock_cluster,
session_context=session_context))
runner = TestRunner(mock_cluster, session_context, Mock(), ctx_list, 1)
results = runner.run_all_tests()
assert results.num_flaky == 0
assert results.num_failed == 1
assert results.num_passed == 1
assert results.num_ignored == 0
passed = [r for r in results if r.test_status == PASS]
failed = [r for r in results if r.test_status == FAIL]
assert passed[
0].test_id == 'tests.runner.resources.test_various_num_nodes.VariousNumNodesTest.test_one_node_a'
assert failed[
0].test_id == 'tests.runner.resources.test_bad_actor.BadActorTest.test_too_many_nodes'
def check_exits_if_failed_to_initialize(self):
"""Validate that runner exits correctly when tests failed to initialize.
"""
mock_cluster = LocalhostCluster(num_nodes=1000)
session_context = tests.ducktape_mock.session_context()
ctx_list = self._do_expand(test_file=FAILS_TO_INIT_TEST_FILE,
test_class=FailsToInitTest,
test_methods=[FailsToInitTest.test_nothing],
cluster=mock_cluster,
session_context=session_context)
ctx_list.extend(
self._do_expand(test_file=FAILS_TO_INIT_IN_SETUP_TEST_FILE,
test_class=FailsToInitInSetupTest,
test_methods=[FailsToInitInSetupTest.test_nothing],
cluster=mock_cluster,
session_context=session_context))
runner = TestRunner(mock_cluster, session_context, Mock(), ctx_list)
results = runner.run_all_tests()
# These tests fail to initialize, each class has two test methods, so should have 4 results, all failed
assert len(results) == 4
assert results.num_failed == 4
assert results.num_passed == 0
assert results.num_ignored == 0
class CheckAllocateFree(object):
def setup_method(self, method):
self.cluster = LocalhostCluster()
self.session_context = session_context(cluster=self.cluster)
self.context = test_context(self.session_context)
def check_allocate_free(self):
"""Check that allocating and freeing nodes works.
This regression test catches the error with Service.free() introduced in v0.3.3 and fixed in v0.3.4
"""
# Node allocation takes place during service instantiation
initial_cluster_size = len(self.cluster)
self.service = DummyService(self.context, 10)
assert self.cluster.num_available_nodes() == initial_cluster_size - 10
self.service.free()
assert self.cluster.num_available_nodes() == initial_cluster_size
class CheckAllocateFree(object):
def setup_method(self, method):
self.cluster = LocalhostCluster()
self.session_context = session_context(cluster=self.cluster)
self.context = test_context(self.session_context)
def check_allocate_free(self):
"""Check that allocating and freeing nodes works.
This regression test catches the error with Service.free() introduced in v0.3.3 and fixed in v0.3.4
"""
# Node allocation takes place during service instantiation
initial_cluster_size = len(self.cluster)
self.service = DummyService(self.context, 10)
assert self.cluster.num_available_nodes() == initial_cluster_size - 10
self.service.free()
assert self.cluster.num_available_nodes() == initial_cluster_size
class CheckLocalhostCluster(object):
def setup_method(self, method):
self.cluster = LocalhostCluster()
def check_size(self):
len(self.cluster) >= 2**31 - 1
def check_request_free(self):
available = self.cluster.num_available_nodes()
initial_size = len(self.cluster)
# Should be able to allocate arbitrarily many nodes
slots = self.cluster.request(100)
assert (len(slots) == 100)
for slot in slots:
assert (slot.account.hostname == 'localhost')
assert (slot.account.user is None)
assert (slot.account.ssh_args is None)
assert (self.cluster.num_available_nodes() == (available - 100))
assert len(self.cluster) == initial_size # This shouldn't change
self.cluster.free(slots)
assert (self.cluster.num_available_nodes() == available)
class CheckLocalhostCluster(object):
def setup_method(self, _):
self.cluster = LocalhostCluster(is_type_based=False)
def check_size(self):
len(self.cluster) >= 2**31 - 1
def check_pickleable(self):
cluster = LocalhostCluster(is_type_based=False)
pickle.dumps(cluster)
def check_request_free(self):
available = self.cluster.num_available_nodes()
initial_size = len(self.cluster)
# Should be able to allocate arbitrarily many nodes
nodes = self.cluster.alloc(Service.setup_cluster_spec(num_nodes=100))
assert (len(nodes) == 100)
for i, node in enumerate(nodes):
assert node.account.hostname == 'localhost%d' % i
assert node.account.ssh_hostname == 'localhost'
assert node.account.ssh_config.hostname == 'localhost'
assert node.account.ssh_config.port == 22
assert node.account.user is None
assert (self.cluster.num_available_nodes() == (available - 100))
assert len(self.cluster) == initial_size # This shouldn't change
self.cluster.free(nodes)
assert (self.cluster.num_available_nodes() == available)
class CheckLocalhostCluster(object):
def setup_method(self, method):
self.cluster = LocalhostCluster()
def check_size(self):
len(self.cluster) >= 2 ** 31 - 1
def check_request_free(self):
available = self.cluster.num_available_nodes()
initial_size = len(self.cluster)
# Should be able to allocate arbitrarily many nodes
slots = self.cluster.request(100)
assert(len(slots) == 100)
for slot in slots:
assert(slot.account.hostname == 'localhost')
assert(slot.account.user is None)
assert(slot.account.ssh_args is None)
assert(self.cluster.num_available_nodes() == (available - 100))
assert len(self.cluster) == initial_size # This shouldn't change
self.cluster.free(slots)
assert(self.cluster.num_available_nodes() == available)
def check_pickleable(self):
"""Check that session_context object is pickleable
This is necessary so that SessionContext objects can be shared between processes when using python
multiprocessing module.
"""
kwargs = {
"session_id": "hello-123",
"results_dir": self.tempdir,
"cluster": LocalhostCluster(is_type_based=False),
"globals": {}
}
session_context = SessionContext(**kwargs)
pickle.dumps(session_context)
def check_exit_first(self):
"""Confirm that exit_first in session context has desired effect of preventing any tests from running
after the first test failure.
"""
mock_cluster = LocalhostCluster(num_nodes=1000)
session_context = tests.ducktape_mock.session_context(
**{"exit_first": True})
test_methods = [FailingTest.test_fail]
ctx_list = self._do_expand(test_file=FAILING_TEST_FILE,
test_class=FailingTest,
test_methods=test_methods,
cluster=mock_cluster,
session_context=session_context)
runner = TestRunner(mock_cluster, session_context, Mock(), ctx_list)
results = runner.run_all_tests()
assert len(ctx_list) > 1
assert len(results) == 1
def check_sends_result_when_error_reporting_exception(self, exc_msg_mock):
"""Validates that an error when reporting an exception in the test doesn't prevent subsequent tests
from executing"""
mock_cluster = LocalhostCluster(num_nodes=1000)
session_context = tests.ducktape_mock.session_context()
test_methods = [TestThingy.test_failure, TestThingy.test_pi]
ctx_list = self._do_expand(test_file=TEST_THINGY_FILE,
test_class=TestThingy,
test_methods=test_methods,
cluster=mock_cluster,
session_context=session_context)
runner = TestRunner(mock_cluster, session_context, Mock(), ctx_list)
results = runner.run_all_tests()
assert len(results) == 2
assert results.num_failed == 1
assert results.num_passed == 1
assert results.num_ignored == 0
def check_runner_timeout(self):
"""Check process cleanup and error handling in a parallel runner client run."""
mock_cluster = LocalhostCluster(num_nodes=1000)
session_context = tests.ducktape_mock.session_context(
max_parallel=1000, test_runner_timeout=1)
test_methods = [TestThingy.test_delayed, TestThingy.test_failure]
ctx_list = self._do_expand(test_file=TEST_THINGY_FILE,
test_class=TestThingy,
test_methods=test_methods,
cluster=mock_cluster,
session_context=session_context)
runner = TestRunner(mock_cluster, session_context, Mock(), ctx_list, 1)
with pytest.raises(TimeoutError):
runner.run_all_tests()
assert not runner._client_procs
def check_deflake_run(self):
"""Check expected behavior when running a single test."""
mock_cluster = LocalhostCluster(num_nodes=1000)
session_context = tests.ducktape_mock.session_context()
test_methods = [TestThingy.test_flaky, TestThingy.test_failure]
ctx_list = self._do_expand(test_file=TEST_THINGY_FILE,
test_class=TestThingy,
test_methods=test_methods,
cluster=mock_cluster,
session_context=session_context)
runner = TestRunner(mock_cluster, session_context, Mock(), ctx_list, 2)
results = runner.run_all_tests()
assert len(results) == 2
assert results.num_flaky == 1
assert results.num_failed == 1
assert results.num_passed == 0
assert results.num_ignored == 0
def check_runner_report_junit(self):
"""Check we can serialize results into a xunit xml format. Also ensures that the XML report
adheres to the Junit spec using xpath queries"""
mock_cluster = LocalhostCluster(num_nodes=1000)
session_context = tests.ducktape_mock.session_context()
test_methods = [
TestThingy.test_pi, TestThingy.test_ignore1,
TestThingy.test_ignore2, TestThingy.test_failure
]
ctx_list = self._do_expand(test_file=TEST_THINGY_FILE,
test_class=TestThingy,
test_methods=test_methods,
cluster=mock_cluster,
session_context=session_context)
runner = TestRunner(mock_cluster, session_context, Mock(), ctx_list)
results = runner.run_all_tests()
JUnitReporter(results).report()
xml_report = os.path.join(session_context.results_dir, "report.xml")
assert os.path.exists(xml_report)
tree = ET.parse(xml_report)
assert len(tree.findall('./testsuite/testcase/failure')) == 1
assert len(tree.findall('./testsuite/testcase/skipped')) == 2
assert len(tree.findall('./testsuite/testcase')) == 4
passed = tree.findall("./testsuite/testcase/[@status='pass']")
assert len(passed) == 1
assert passed[0].get("classname") == "TestThingy"
assert passed[0].get("name") == "test_pi"
failures = tree.findall("./testsuite/testcase/[@status='fail']")
assert len(failures) == 1
assert failures[0].get("classname") == "TestThingy"
assert failures[0].get("name") == "test_failure"
ignores = tree.findall("./testsuite/testcase/[@status='ignore']")
assert len(ignores) == 2
assert ignores[0].get("classname") == "TestThingy"
assert ignores[1].get("classname") == "TestThingy"
assert ignores[0].get("name") == "test_ignore1"
assert ignores[1].get("name") == "test_ignore2.x=5"
def check_simple_messaging(self):
s_context = session_context()
cluster = LocalhostCluster(num_nodes=1000)
t_context = test_context(s_context, cluster)
client_id = "test-runner-{}-{}".format(os.getpid(), id(self))
receiver_response_factory = EventResponseFactory()
receiver = Receiver(5556, 5656)
receiver.start()
port = receiver.port
try:
p = mp.Process(target=self.ready_response, args=(client_id, port))
p.start()
event = receiver.recv(timeout=10000)
assert event["event_type"] == ClientEventFactory.READY
logging.info('replying to client')
receiver.send(receiver_response_factory.ready(event, s_context, t_context, cluster))
finally:
p.join()
def check_run_failure_with_bad_cluster_allocation(self):
"""Check test should be marked failed if it under-utilizes the cluster resources."""
mock_cluster = LocalhostCluster(num_nodes=1000)
session_context = tests.ducktape_mock.session_context(
fail_bad_cluster_utilization="fail_bad_cluster_utilization")
test_methods = [
ClusterTestThingy.test_bad_num_nodes,
ClusterTestThingy.test_good_num_nodes
]
ctx_list = self._do_expand(test_file=TEST_THINGY_FILE,
test_class=ClusterTestThingy,
test_methods=test_methods,
cluster=mock_cluster,
session_context=session_context)
runner = TestRunner(mock_cluster, session_context, Mock(), ctx_list, 1)
results = runner.run_all_tests()
assert len(results) == 2
assert results.num_flaky == 0
assert results.num_failed == 1
assert results.num_passed == 1
assert results.num_ignored == 0
class CheckLocalhostCluster(object):
def setup_method(self, method):
self.cluster = LocalhostCluster()
def check_request_free(self):
available = self.cluster.num_available_nodes()
# Should be able to allocate arbitrarily many nodes
slots = self.cluster.request(100)
assert (len(slots) == 100)
for slot in slots:
assert (slot.account.hostname == 'localhost')
assert (slot.account.user == None)
assert (slot.account.ssh_args == None)
assert (self.cluster.num_available_nodes() == (available - 100))
self.cluster.free(slots)
assert (self.cluster.num_available_nodes() == available)
class CheckLocalhostCluster(object):
def setup_method(self, method):
self.cluster = LocalhostCluster()
def check_request_free(self):
available = self.cluster.num_available_nodes()
# Should be able to allocate arbitrarily many nodes
slots = self.cluster.request(100)
assert(len(slots) == 100)
for slot in slots:
assert(slot.account.hostname == 'localhost')
assert(slot.account.user == None)
assert(slot.account.ssh_args == None)
assert(self.cluster.num_available_nodes() == (available - 100))
self.cluster.free(slots)
assert(self.cluster.num_available_nodes() == available)
def check_pickleable(self):
cluster = LocalhostCluster()
pickle.dumps(cluster)
def setup_method(self, _):
mock_cluster = LocalhostCluster()
self.session_context = tests.ducktape_mock.session_context(mock_cluster)
def setup_method(self, method):
self.cluster = LocalhostCluster()
def check_pickleable(self):
cluster = LocalhostCluster(is_type_based=False)
pickle.dumps(cluster)
def setup_method(self, _):
self.cluster = LocalhostCluster(is_type_based=False)
def setup_method(self, _):
self.cluster = LocalhostCluster()
self.session_context = session_context()
self.context = test_context(self.session_context, cluster=self.cluster)
def setup_method(self, method):
self.cluster = LocalhostCluster()
self.session_context = session_context(cluster=self.cluster)
self.context = test_context(self.session_context)
def setup_method(self, method):
self.cluster = LocalhostCluster()
def setup_method(self, _):
self.cluster = LocalhostCluster(num_nodes=100)
self.session_context = tests.ducktape_mock.session_context()
def setup_method(self, _):
self.cluster = LocalhostCluster(is_type_based=False)
self.session_context = session_context()
self.context = test_context(self.session_context, cluster=self.cluster)