Example #1
0
class CheckAllocateFree(object):
    def setup_method(self, _):
        self.cluster = LocalhostCluster()
        self.session_context = session_context()
        self.context = test_context(self.session_context, cluster=self.cluster)

    def check_allocate_free(self):
        """Check that allocating and freeing nodes works.

        This regression test catches the error with Service.free() introduced in v0.3.3 and fixed in v0.3.4
        """

        # Node allocation takes place during service instantiation
        initial_cluster_size = len(self.cluster)
        self.service = DummyService(self.context, 10)
        assert self.cluster.num_available_nodes() == initial_cluster_size - 10

        self.service.free()
        assert self.cluster.num_available_nodes() == initial_cluster_size

    def check_order(self):
        """Check expected behavior with service._order method"""
        self.dummy0 = DummyService(self.context, 4)
        self.diffDummy0 = DifferentDummyService(self.context, 100)
        self.dummy1 = DummyService(self.context, 1)
        self.diffDummy1 = DifferentDummyService(self.context, 2)
        self.diffDummy2 = DifferentDummyService(self.context, 5)

        assert self.dummy0._order == 0
        assert self.dummy1._order == 1
        assert self.diffDummy0._order == 0
        assert self.diffDummy1._order == 1
        assert self.diffDummy2._order == 2
Example #2
0
class CheckAllocateFree(object):

    def setup_method(self, method):
        self.cluster = LocalhostCluster()
        self.session_context = session_context(cluster=self.cluster)
        self.context = test_context(self.session_context)

    def check_allocate_free(self):
        """Check that allocating and freeing nodes works.

        This regression test catches the error with Service.free() introduced in v0.3.3 and fixed in v0.3.4
        """

        # Node allocation takes place during service instantiation
        initial_cluster_size = len(self.cluster)
        self.service = DummyService(self.context, 10)
        assert self.cluster.num_available_nodes() == initial_cluster_size - 10

        self.service.free()
        assert self.cluster.num_available_nodes() == initial_cluster_size

    def check_order(self):
        """Check expected behavior with service._order method"""
        self.dummy0 = DummyService(self.context, 4)
        self.diffDummy0 = DifferentDummyService(self.context, 100)
        self.dummy1 = DummyService(self.context, 1)
        self.diffDummy1 = DifferentDummyService(self.context, 2)
        self.diffDummy2 = DifferentDummyService(self.context, 5)

        assert self.dummy0._order == 0
        assert self.dummy1._order == 1
        assert self.diffDummy0._order == 0
        assert self.diffDummy1._order == 1
        assert self.diffDummy2._order == 2
Example #3
0
    def check_simple_run(self):
        """Check expected behavior when running a single test."""
        mock_cluster = LocalhostCluster(num_nodes=1000)
        session_context = tests.ducktape_mock.session_context()

        test_methods = [
            TestThingy.test_pi, TestThingy.test_ignore1,
            TestThingy.test_ignore2, TestThingy.test_failure
        ]
        ctx_list = self._do_expand(test_file=TEST_THINGY_FILE,
                                   test_class=TestThingy,
                                   test_methods=test_methods,
                                   cluster=mock_cluster,
                                   session_context=session_context)
        runner = TestRunner(mock_cluster, session_context, Mock(), ctx_list, 1)

        results = runner.run_all_tests()
        assert len(results) == 4
        assert results.num_flaky == 0
        assert results.num_failed == 1
        assert results.num_passed == 1
        assert results.num_ignored == 2

        result_with_data = [r for r in results if r.data is not None][0]
        assert result_with_data.data == {"data": 3.14159}
Example #4
0
    def check_simple_run(self):
        """Check expected behavior when running a single test."""
        mock_cluster = LocalhostCluster(num_nodes=1000)
        session_context = tests.ducktape_mock.session_context()

        test_methods = [
            TestThingy.test_pi, TestThingy.test_ignore1,
            TestThingy.test_ignore2
        ]
        ctx_list = []
        for f in test_methods:
            ctx_list.extend(
                MarkedFunctionExpander(session_context=session_context,
                                       cls=TestThingy,
                                       function=f,
                                       file=TEST_THINGY_FILE,
                                       cluster=mock_cluster).expand())

        runner = TestRunner(mock_cluster, session_context, Mock(), ctx_list)

        results = runner.run_all_tests()
        assert len(results) == 3
        assert results.num_failed == 0
        assert results.num_passed == 1
        assert results.num_ignored == 2

        result_with_data = filter(lambda r: r.data is not None, results)[0]
        assert result_with_data.data == {"data": 3.14159}
Example #5
0
    def check_test_failure_with_too_many_nodes_requested(self):
        mock_cluster = LocalhostCluster(num_nodes=1000)
        session_context = tests.ducktape_mock.session_context(debug=True)

        ctx_list = self._do_expand(
            test_file=BAD_ACTOR_TEST_FILE,
            test_class=BadActorTest,
            test_methods=[BadActorTest.test_too_many_nodes],
            cluster=mock_cluster,
            session_context=session_context)
        ctx_list.extend(
            self._do_expand(test_file=VARIOUS_NUM_NODES_TEST_FILE,
                            test_class=VariousNumNodesTest,
                            test_methods=[VariousNumNodesTest.test_one_node_a],
                            cluster=mock_cluster,
                            session_context=session_context))
        runner = TestRunner(mock_cluster, session_context, Mock(), ctx_list, 1)
        results = runner.run_all_tests()
        assert results.num_flaky == 0
        assert results.num_failed == 1
        assert results.num_passed == 1
        assert results.num_ignored == 0
        passed = [r for r in results if r.test_status == PASS]
        failed = [r for r in results if r.test_status == FAIL]
        assert passed[
            0].test_id == 'tests.runner.resources.test_various_num_nodes.VariousNumNodesTest.test_one_node_a'
        assert failed[
            0].test_id == 'tests.runner.resources.test_bad_actor.BadActorTest.test_too_many_nodes'
Example #6
0
    def check_exits_if_failed_to_initialize(self):
        """Validate that runner exits correctly when tests failed to initialize.
        """
        mock_cluster = LocalhostCluster(num_nodes=1000)
        session_context = tests.ducktape_mock.session_context()

        ctx_list = self._do_expand(test_file=FAILS_TO_INIT_TEST_FILE,
                                   test_class=FailsToInitTest,
                                   test_methods=[FailsToInitTest.test_nothing],
                                   cluster=mock_cluster,
                                   session_context=session_context)
        ctx_list.extend(
            self._do_expand(test_file=FAILS_TO_INIT_IN_SETUP_TEST_FILE,
                            test_class=FailsToInitInSetupTest,
                            test_methods=[FailsToInitInSetupTest.test_nothing],
                            cluster=mock_cluster,
                            session_context=session_context))

        runner = TestRunner(mock_cluster, session_context, Mock(), ctx_list)
        results = runner.run_all_tests()
        # These tests fail to initialize, each class has two test methods, so should have 4 results, all failed
        assert len(results) == 4
        assert results.num_failed == 4
        assert results.num_passed == 0
        assert results.num_ignored == 0
Example #7
0
class CheckAllocateFree(object):
    def setup_method(self, method):
        self.cluster = LocalhostCluster()
        self.session_context = session_context(cluster=self.cluster)
        self.context = test_context(self.session_context)

    def check_allocate_free(self):
        """Check that allocating and freeing nodes works.

        This regression test catches the error with Service.free() introduced in v0.3.3 and fixed in v0.3.4
        """

        # Node allocation takes place during service instantiation
        initial_cluster_size = len(self.cluster)
        self.service = DummyService(self.context, 10)
        assert self.cluster.num_available_nodes() == initial_cluster_size - 10

        self.service.free()
        assert self.cluster.num_available_nodes() == initial_cluster_size
class CheckAllocateFree(object):

    def setup_method(self, method):
        self.cluster = LocalhostCluster()
        self.session_context = session_context(cluster=self.cluster)
        self.context = test_context(self.session_context)

    def check_allocate_free(self):
        """Check that allocating and freeing nodes works.

        This regression test catches the error with Service.free() introduced in v0.3.3 and fixed in v0.3.4
        """

        # Node allocation takes place during service instantiation
        initial_cluster_size = len(self.cluster)
        self.service = DummyService(self.context, 10)
        assert self.cluster.num_available_nodes() == initial_cluster_size - 10

        self.service.free()
        assert self.cluster.num_available_nodes() == initial_cluster_size
Example #9
0
class CheckLocalhostCluster(object):
    def setup_method(self, method):
        self.cluster = LocalhostCluster()

    def check_size(self):
        len(self.cluster) >= 2**31 - 1

    def check_request_free(self):
        available = self.cluster.num_available_nodes()
        initial_size = len(self.cluster)

        # Should be able to allocate arbitrarily many nodes
        slots = self.cluster.request(100)
        assert (len(slots) == 100)
        for slot in slots:
            assert (slot.account.hostname == 'localhost')
            assert (slot.account.user is None)
            assert (slot.account.ssh_args is None)

        assert (self.cluster.num_available_nodes() == (available - 100))
        assert len(self.cluster) == initial_size  # This shouldn't change

        self.cluster.free(slots)

        assert (self.cluster.num_available_nodes() == available)
Example #10
0
class CheckLocalhostCluster(object):
    def setup_method(self, _):
        self.cluster = LocalhostCluster(is_type_based=False)

    def check_size(self):
        len(self.cluster) >= 2**31 - 1

    def check_pickleable(self):
        cluster = LocalhostCluster(is_type_based=False)
        pickle.dumps(cluster)

    def check_request_free(self):
        available = self.cluster.num_available_nodes()
        initial_size = len(self.cluster)

        # Should be able to allocate arbitrarily many nodes
        nodes = self.cluster.alloc(Service.setup_cluster_spec(num_nodes=100))
        assert (len(nodes) == 100)
        for i, node in enumerate(nodes):
            assert node.account.hostname == 'localhost%d' % i
            assert node.account.ssh_hostname == 'localhost'
            assert node.account.ssh_config.hostname == 'localhost'
            assert node.account.ssh_config.port == 22
            assert node.account.user is None

        assert (self.cluster.num_available_nodes() == (available - 100))
        assert len(self.cluster) == initial_size  # This shouldn't change

        self.cluster.free(nodes)

        assert (self.cluster.num_available_nodes() == available)
class CheckLocalhostCluster(object):
    def setup_method(self, method):
        self.cluster = LocalhostCluster()

    def check_size(self):
        len(self.cluster) >= 2 ** 31 - 1

    def check_request_free(self):
        available = self.cluster.num_available_nodes()
        initial_size = len(self.cluster)

        # Should be able to allocate arbitrarily many nodes
        slots = self.cluster.request(100)
        assert(len(slots) == 100)
        for slot in slots:
            assert(slot.account.hostname == 'localhost')
            assert(slot.account.user is None)
            assert(slot.account.ssh_args is None)

        assert(self.cluster.num_available_nodes() == (available - 100))
        assert len(self.cluster) == initial_size  # This shouldn't change

        self.cluster.free(slots)

        assert(self.cluster.num_available_nodes() == available)
Example #12
0
    def check_pickleable(self):
        """Check that session_context object is pickleable
        This is necessary so that SessionContext objects can be shared between processes when using python
        multiprocessing module.
        """

        kwargs = {
            "session_id": "hello-123",
            "results_dir": self.tempdir,
            "cluster": LocalhostCluster(is_type_based=False),
            "globals": {}
        }
        session_context = SessionContext(**kwargs)
        pickle.dumps(session_context)
Example #13
0
    def check_exit_first(self):
        """Confirm that exit_first in session context has desired effect of preventing any tests from running
        after the first test failure.
        """
        mock_cluster = LocalhostCluster(num_nodes=1000)
        session_context = tests.ducktape_mock.session_context(
            **{"exit_first": True})

        test_methods = [FailingTest.test_fail]
        ctx_list = self._do_expand(test_file=FAILING_TEST_FILE,
                                   test_class=FailingTest,
                                   test_methods=test_methods,
                                   cluster=mock_cluster,
                                   session_context=session_context)
        runner = TestRunner(mock_cluster, session_context, Mock(), ctx_list)
        results = runner.run_all_tests()
        assert len(ctx_list) > 1
        assert len(results) == 1
Example #14
0
    def check_sends_result_when_error_reporting_exception(self, exc_msg_mock):
        """Validates that an error when reporting an exception in the test doesn't prevent subsequent tests
        from executing"""
        mock_cluster = LocalhostCluster(num_nodes=1000)
        session_context = tests.ducktape_mock.session_context()
        test_methods = [TestThingy.test_failure, TestThingy.test_pi]
        ctx_list = self._do_expand(test_file=TEST_THINGY_FILE,
                                   test_class=TestThingy,
                                   test_methods=test_methods,
                                   cluster=mock_cluster,
                                   session_context=session_context)
        runner = TestRunner(mock_cluster, session_context, Mock(), ctx_list)

        results = runner.run_all_tests()
        assert len(results) == 2
        assert results.num_failed == 1
        assert results.num_passed == 1
        assert results.num_ignored == 0
Example #15
0
    def check_runner_timeout(self):
        """Check process cleanup and error handling in a parallel runner client run."""
        mock_cluster = LocalhostCluster(num_nodes=1000)
        session_context = tests.ducktape_mock.session_context(
            max_parallel=1000, test_runner_timeout=1)

        test_methods = [TestThingy.test_delayed, TestThingy.test_failure]
        ctx_list = self._do_expand(test_file=TEST_THINGY_FILE,
                                   test_class=TestThingy,
                                   test_methods=test_methods,
                                   cluster=mock_cluster,
                                   session_context=session_context)
        runner = TestRunner(mock_cluster, session_context, Mock(), ctx_list, 1)

        with pytest.raises(TimeoutError):
            runner.run_all_tests()

        assert not runner._client_procs
Example #16
0
    def check_deflake_run(self):
        """Check expected behavior when running a single test."""
        mock_cluster = LocalhostCluster(num_nodes=1000)
        session_context = tests.ducktape_mock.session_context()

        test_methods = [TestThingy.test_flaky, TestThingy.test_failure]
        ctx_list = self._do_expand(test_file=TEST_THINGY_FILE,
                                   test_class=TestThingy,
                                   test_methods=test_methods,
                                   cluster=mock_cluster,
                                   session_context=session_context)
        runner = TestRunner(mock_cluster, session_context, Mock(), ctx_list, 2)

        results = runner.run_all_tests()
        assert len(results) == 2
        assert results.num_flaky == 1
        assert results.num_failed == 1
        assert results.num_passed == 0
        assert results.num_ignored == 0
Example #17
0
    def check_runner_report_junit(self):
        """Check we can serialize results into a xunit xml format. Also ensures that the XML report
        adheres to the Junit spec using xpath queries"""
        mock_cluster = LocalhostCluster(num_nodes=1000)
        session_context = tests.ducktape_mock.session_context()
        test_methods = [
            TestThingy.test_pi, TestThingy.test_ignore1,
            TestThingy.test_ignore2, TestThingy.test_failure
        ]
        ctx_list = self._do_expand(test_file=TEST_THINGY_FILE,
                                   test_class=TestThingy,
                                   test_methods=test_methods,
                                   cluster=mock_cluster,
                                   session_context=session_context)
        runner = TestRunner(mock_cluster, session_context, Mock(), ctx_list)

        results = runner.run_all_tests()
        JUnitReporter(results).report()
        xml_report = os.path.join(session_context.results_dir, "report.xml")
        assert os.path.exists(xml_report)
        tree = ET.parse(xml_report)
        assert len(tree.findall('./testsuite/testcase/failure')) == 1
        assert len(tree.findall('./testsuite/testcase/skipped')) == 2
        assert len(tree.findall('./testsuite/testcase')) == 4

        passed = tree.findall("./testsuite/testcase/[@status='pass']")
        assert len(passed) == 1
        assert passed[0].get("classname") == "TestThingy"
        assert passed[0].get("name") == "test_pi"

        failures = tree.findall("./testsuite/testcase/[@status='fail']")
        assert len(failures) == 1
        assert failures[0].get("classname") == "TestThingy"
        assert failures[0].get("name") == "test_failure"

        ignores = tree.findall("./testsuite/testcase/[@status='ignore']")
        assert len(ignores) == 2
        assert ignores[0].get("classname") == "TestThingy"
        assert ignores[1].get("classname") == "TestThingy"

        assert ignores[0].get("name") == "test_ignore1"
        assert ignores[1].get("name") == "test_ignore2.x=5"
Example #18
0
    def check_simple_messaging(self):
        s_context = session_context()
        cluster = LocalhostCluster(num_nodes=1000)
        t_context = test_context(s_context, cluster)

        client_id = "test-runner-{}-{}".format(os.getpid(), id(self))
        receiver_response_factory = EventResponseFactory()

        receiver = Receiver(5556, 5656)
        receiver.start()
        port = receiver.port

        try:
            p = mp.Process(target=self.ready_response, args=(client_id, port))
            p.start()

            event = receiver.recv(timeout=10000)
            assert event["event_type"] == ClientEventFactory.READY
            logging.info('replying to client')
            receiver.send(receiver_response_factory.ready(event, s_context, t_context, cluster))
        finally:
            p.join()
Example #19
0
    def check_run_failure_with_bad_cluster_allocation(self):
        """Check test should be marked failed if it under-utilizes the cluster resources."""
        mock_cluster = LocalhostCluster(num_nodes=1000)
        session_context = tests.ducktape_mock.session_context(
            fail_bad_cluster_utilization="fail_bad_cluster_utilization")

        test_methods = [
            ClusterTestThingy.test_bad_num_nodes,
            ClusterTestThingy.test_good_num_nodes
        ]
        ctx_list = self._do_expand(test_file=TEST_THINGY_FILE,
                                   test_class=ClusterTestThingy,
                                   test_methods=test_methods,
                                   cluster=mock_cluster,
                                   session_context=session_context)
        runner = TestRunner(mock_cluster, session_context, Mock(), ctx_list, 1)

        results = runner.run_all_tests()

        assert len(results) == 2
        assert results.num_flaky == 0
        assert results.num_failed == 1
        assert results.num_passed == 1
        assert results.num_ignored == 0
Example #20
0
class CheckLocalhostCluster(object):
    def setup_method(self, method):
        self.cluster = LocalhostCluster()

    def check_request_free(self):
        available = self.cluster.num_available_nodes()

        # Should be able to allocate arbitrarily many nodes
        slots = self.cluster.request(100)
        assert (len(slots) == 100)
        for slot in slots:
            assert (slot.account.hostname == 'localhost')
            assert (slot.account.user == None)
            assert (slot.account.ssh_args == None)

        assert (self.cluster.num_available_nodes() == (available - 100))

        self.cluster.free(slots)

        assert (self.cluster.num_available_nodes() == available)
Example #21
0
class CheckLocalhostCluster(object):
    def setup_method(self, method):
        self.cluster = LocalhostCluster()

    def check_request_free(self):
        available = self.cluster.num_available_nodes()

        # Should be able to allocate arbitrarily many nodes
        slots = self.cluster.request(100)
        assert(len(slots) == 100)
        for slot in slots:
            assert(slot.account.hostname == 'localhost')
            assert(slot.account.user == None)
            assert(slot.account.ssh_args == None)

        assert(self.cluster.num_available_nodes() == (available - 100))

        self.cluster.free(slots)

        assert(self.cluster.num_available_nodes() == available)
Example #22
0
 def check_pickleable(self):
     cluster = LocalhostCluster()
     pickle.dumps(cluster)
Example #23
0
 def setup_method(self, _):
     mock_cluster = LocalhostCluster()
     self.session_context = tests.ducktape_mock.session_context(mock_cluster)
 def setup_method(self, method):
     self.cluster = LocalhostCluster()
Example #25
0
 def check_pickleable(self):
     cluster = LocalhostCluster(is_type_based=False)
     pickle.dumps(cluster)
Example #26
0
 def setup_method(self, _):
     self.cluster = LocalhostCluster(is_type_based=False)
Example #27
0
 def setup_method(self, _):
     self.cluster = LocalhostCluster()
     self.session_context = session_context()
     self.context = test_context(self.session_context, cluster=self.cluster)
Example #28
0
 def setup_method(self, method):
     self.cluster = LocalhostCluster()
     self.session_context = session_context(cluster=self.cluster)
     self.context = test_context(self.session_context)
Example #29
0
 def setup_method(self, method):
     self.cluster = LocalhostCluster()
Example #30
0
 def setup_method(self, _):
     self.cluster = LocalhostCluster(num_nodes=100)
     self.session_context = tests.ducktape_mock.session_context()
Example #31
0
 def setup_method(self, _):
     self.cluster = LocalhostCluster(is_type_based=False)
     self.session_context = session_context()
     self.context = test_context(self.session_context, cluster=self.cluster)