Example #1
0
 def check_test_loader_include_class_exclude_method(self):
     loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())
     included = [os.path.join(discover_dir(), "test_b.py::TestBB")]
     excluded = [os.path.join(discover_dir(), "test_b.py::TestBB.test_bb_one")]
     tests = loader.load(included, excluded)
     # TestBB contains 2 test methods, but 1 is excluded
     assert len(tests) == 1
Example #2
0
 def check_test_loader_include_dir_exclude_file(self):
     loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())
     excluded_file_a = os.path.join(discover_dir(), "test_a.py")
     excluded_file_b = os.path.join(discover_dir(), "test_b.py")
     num_excluded = num_tests_in_file(excluded_file_a) + num_tests_in_file(excluded_file_b)
     tests = loader.load([discover_dir()], [excluded_file_a, excluded_file_b])
     assert len(tests) == num_tests_in_dir(discover_dir()) - num_excluded
    def check_test_loader_multiple_files(self):
        loader = TestLoader(self.SESSION_CONTEXT)
        file_a = os.path.join(discover_dir(), "test_a.py")
        file_b = os.path.join(discover_dir(), "test_b.py")

        tests = loader.discover([file_a, file_b])
        assert len(tests) == num_tests_in_file(file_a) + num_tests_in_file(file_b)
Example #4
0
    def check_test_loader_with_absolute_path(self):
        """
        Test loading suites using absolute paths to other imported suites as well as absolute paths
        to tests in the suite
        """
        with tempfile.TemporaryDirectory() as td:
            temp_suite1 = os.path.join(td, 'temp_suite1.yml')
            temp_suite2 = os.path.join(td, 'temp_suite2.yml')
            with open(temp_suite1, 'w') as f:
                test_yaml1 = yaml.dump({
                    'import':
                    str(os.path.join(td, 'temp_suite2.yml')),
                    'suite': [
                        os.path.abspath(
                            os.path.join(discover_dir(), "test_a.py"))
                    ]
                })
                f.write(test_yaml1)

            with open(temp_suite2, 'w') as f:
                test_yaml2 = yaml.dump({
                    'suite': [
                        os.path.abspath(
                            os.path.join(discover_dir(), "test_b.py"))
                    ]
                })
                f.write(test_yaml2)

            loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())
            tests = loader.load([temp_suite1])
            assert len(tests) == 4
Example #5
0
 def check_test_loader_include_file_without_tests(self):
     with pytest.raises(LoaderException):
         loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())
         loader.load([
             os.path.join(discover_dir(), "sub_dir_no_tests",
                          "just_some_file.py")
         ])
Example #6
0
 def check_test_loader_multiple_files(self):
     loader = TestLoader(self.SESSION_CONTEXT)
     tests = loader.discover([
         os.path.join(discover_dir(), "test_a.py"),
         os.path.join(discover_dir(), "test_b.py")
     ])
     assert len(tests) == 4
Example #7
0
 def check_test_loader_allow_exclude_file_without_tests(self):
     loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())
     tests = loader.load([discover_dir()], [
         os.path.join(discover_dir(), "sub_dir_no_tests",
                      "just_some_file.py")
     ])
     assert len(tests) == num_tests_in_dir(discover_dir())
Example #8
0
 def check_test_loader_multiple_files(self):
     loader = TestLoader(self.SESSION_CONTEXT)
     tests = loader.discover([
         os.path.join(discover_dir(), "test_a.py"),
         os.path.join(discover_dir(), "test_b.py")
     ])
     assert len(tests) == 4
Example #9
0
    def check_test_loader_with_file(self, dir_, file_name):
        """Check discovery on a file. """
        loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())
        module_path = os.path.join(dir_, file_name)

        tests = loader.load([module_path])
        assert len(tests) == num_tests_in_file(module_path)
Example #10
0
    def check_test_loader_with_file(self):
        """Check discovery on a file. """
        loader = TestLoader(self.SESSION_CONTEXT)
        module_path = os.path.join(discover_dir(), "test_a.py")

        tests = loader.discover([module_path])
        assert len(tests) == num_tests_in_file(module_path)
Example #11
0
    def check_test_loader_multiple_files(self):
        loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())
        file_a = os.path.join(discover_dir(), "test_a.py")
        file_b = os.path.join(discover_dir(), "test_b.py")

        tests = loader.load([file_a, file_b])
        assert len(tests) == num_tests_in_file(file_a) + num_tests_in_file(file_b)
Example #12
0
 def check_test_loader_exclude_subdir(self):
     loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())
     included_dir = discover_dir()
     excluded_dir = sub_dir_a()
     tests = loader.load([included_dir], [excluded_dir])
     assert len(tests) == num_tests_in_dir(included_dir) - num_tests_in_dir(
         excluded_dir)
Example #13
0
    def _collect_test_context(self, directory, file_name, cls_name, method_name, injected_args):
        loader = TestLoader(self.session_context, self.logger, injected_args=injected_args, cluster=self.cluster)
        loaded_context_list = loader.discover(directory, file_name, cls_name, method_name)

        assert len(loaded_context_list) == 1
        test_context = loaded_context_list[0]
        test_context.cluster = self.cluster
        return test_context
Example #14
0
 def check_loader_with_non_yml_file(self):
     """
     test loading a test file as an import
     """
     file = os.path.join(discover_dir(), 'test_suite_import_py.yml')
     loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())
     with pytest.raises(LoaderException, match=r'Failed to load test suite from file: \S+test_a\.py'):
         loader.load([file])
Example #15
0
 def check_test_loader_include_dir_exclude_class(self):
     """Check test discovery with discover class syntax."""
     loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())
     tests = loader.load(
         [discover_dir()],
         [os.path.join(discover_dir(), "test_b.py::TestBB")])
     # TestBB contains 2 test methods
     assert len(tests) == num_tests_in_dir(discover_dir()) - 2
Example #16
0
 def check_test_loader_include_dir_exclude_method(self):
     loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())
     excluded = [
         os.path.join(discover_dir(), "test_b.py::TestBB.test_bb_one")
     ]
     tests = loader.load([discover_dir()], excluded)
     # excluded 1 method only
     assert len(tests) == num_tests_in_dir(discover_dir()) - 1
Example #17
0
 def check_test_loader_with_parametrize_with_objects(self):
     loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())
     parameters = '{"d": {"a": "A"}, "lst": ["whatever"]}'
     included = [os.path.join(
         discover_dir(), 'test_decorated.py::TestObjectParameters.test_thing@{}'.format(parameters))]
     tests = loader.load(included)
     assert len(tests) == 1
     assert tests[0].injected_args == {'d': {'a': 'A'}, 'lst': ['whatever']}
Example #18
0
 def check_test_loader_with_matrix_params(self):
     loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())
     included = [os.path.join(discover_dir(), 'test_decorated.py::TestMatrix.test_thing@{"x": 1,"y": "test "}')]
     tests = loader.load(included)
     # TestMatrix contains a single parametrized method. Since we only provide a single set of params, we should
     # end up with a single context
     assert len(tests) == 1
     assert tests[0].injected_args == {'x': 1, 'y': 'test '}
Example #19
0
    def _collect_test_context(self, directory, file_name, cls_name, method_name, injected_args):
        loader = TestLoader(self.session_context, self.logger, injected_args=injected_args, cluster=self.cluster)
        loaded_context_list = loader.discover(directory, file_name, cls_name, method_name)

        assert len(loaded_context_list) == 1
        test_context = loaded_context_list[0]
        test_context.cluster = self.cluster
        return test_context
Example #20
0
    def check_test_loader_with_class(self):
        """Check test discovery with discover class syntax."""
        loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())
        tests = loader.load([os.path.join(discover_dir(), "test_b.py::TestBB")])
        assert len(tests) == 2

        # Sanity check, test that it discovers two test class & 3 tests if it searches the whole module
        tests = loader.load([os.path.join(discover_dir(), "test_b.py")])
        assert len(tests) == 3
Example #21
0
 def check_test_loader_raises_with_both_injected_args_and_parameters(self):
     """One should not use both --parameters command-line flag and parameterized test symbols at the same time.
     Loader will explicitly raise in such cases to avoid confusing behavior.
     """
     injected_args = {"x": 1, "y": "test "}
     loader = TestLoader(self.SESSION_CONTEXT, logger=Mock(), injected_args=injected_args)
     included = [os.path.join(discover_dir(), 'test_decorated.py::TestMatrix.test_thing@{"x": 1,"y": "test "}')]
     with pytest.raises(LoaderException, match='Cannot use both'):
         loader.load(included)
Example #22
0
    def check_test_loader_with_class(self):
        """Check test discovery with discover class syntax."""
        loader = TestLoader(self.SESSION_CONTEXT)
        tests = loader.discover([os.path.join(discover_dir(), "test_b.py::TestBB")])
        assert len(tests) == 2

        # Sanity check, test that it discovers two test class & 3 tests if it searches the whole module
        tests = loader.discover([os.path.join(discover_dir(), "test_b.py")])
        assert len(tests) == 3
Example #23
0
def main():
    """Ducktape entry point. This contains top level logic for ducktape command-line program which does the following:

        Discover tests
        Initialize cluster for distributed services
        Run tests
        Report a summary of all results
    """
    args = parse_args()

    # Make .ducktape directory where metadata such as the last used session_id is stored
    if not os.path.isdir(ConsoleConfig.METADATA_DIR):
        os.makedirs(ConsoleConfig.METADATA_DIR)

    # Generate a shared 'global' identifier for this test run and create the directory
    # in which all test results will be stored
    session_id = generate_session_id(ConsoleConfig.SESSION_ID_FILE)
    results_dir = generate_results_dir(session_id)

    setup_results_directory(results_dir, session_id)
    session_context = SessionContext(session_id, results_dir, cluster=None, args=args)

    # Discover and load tests to be run
    extend_import_paths(args.test_path)
    loader = TestLoader(session_context)
    try:
        test_classes = loader.discover(args.test_path)
    except LoaderException as e:
        print "Failed while trying to discover tests: {}".format(e)
        sys.exit(1)

    if args.collect_only:
        print test_classes
        sys.exit(0)

    # Initializing the cluster is slow, so do so only if
    # tests are sure to be run
    session_context.cluster = VagrantCluster()

    # Run the tests
    runner = SerialTestRunner(session_context, test_classes)
    test_results = runner.run_all_tests()

    # Report results
    # TODO command-line hook for type of reporter
    reporter = SimpleStdoutReporter(test_results)
    reporter.report()
    reporter = SimpleFileReporter(test_results)
    reporter.report()

    # Generate HTML reporter
    reporter = HTMLReporter(test_results)
    reporter.report()

    if not test_results.get_aggregate_success():
        sys.exit(1)
Example #24
0
 def check_test_loader_with_params_special_chars(self):
     loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())
     included = [os.path.join(discover_dir(
     ), r'test_decorated.py::TestParametrizdeSpecial.test_special_characters_params'
        r'@{"version": "6.1.0", "chars": "!@#$%^&*()_+::.,/? \"{}\\"}')]
     tests = loader.load(included)
     # TestMatrix contains a single parametrized method. Since we only provide a single set of params, we should
     # end up with a single context
     assert len(tests) == 1
     assert tests[0].injected_args == {'version': '6.1.0', 'chars': '!@#$%^&*()_+::.,/? \"{}\\'}
Example #25
0
 def check_test_loader_with_test_suites_and_files(self, expected_count, input_symbols, excluded_symbols):
     """
     When both files and test suites are loaded, files (both included and excluded) are
     loaded after and separately from the test suites, so even if a test suite excludes file A,
     it will be included if it's passed directly. And if file A is excluded directly, even if any of
     the test suites includes it, it will still be excluded.
     """
     loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())
     tests = loader.load(input_symbols, excluded_test_symbols=excluded_symbols)
     assert len(tests) == expected_count
Example #26
0
 def check_test_loader_with_parametrize(self):
     loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())
     included = [
         os.path.join(
             discover_dir(),
             'test_decorated.py::TestParametrized.test_thing@{"x":1,"y":2}')
     ]
     tests = loader.load(included)
     assert len(tests) == 1
     assert tests[0].injected_args == {'x': 1, 'y': 2}
Example #27
0
    def check_parse_discovery_symbol(self):
        """Check that "test discovery symbol" parsing logic works correctly"""
        parsed_symbols = [
            {
                'dir': 'path/to/dir',
                'file': '',
                'cls': '',
                'method': ''
            },
            {
                'dir': 'path/to/dir',
                'file': 'test_file.py',
                'cls': '',
                'method': ''
            },
            {
                'dir': 'path/to/dir',
                'file': 'test_file.py',
                'cls': 'ClassName',
                'method': ''
            },
            {
                'dir': 'path/to/dir',
                'file': 'test_file.py',
                'cls': 'ClassName',
                'method': 'method'
            },
            {
                'dir': 'path/to/dir',
                'file': '',
                'cls': 'ClassName',
                'method': ''
            },
        ]

        loader = TestLoader(tests.ducktape_mock.session_context(), logger=Mock())
        for parsed in parsed_symbols:
            symbol = join_parsed_symbol_components(parsed)

            expected_parsed = (
                normalize_ending_slash(parsed['dir']),
                parsed['file'],
                parsed['cls'],
                parsed['method']
            )

            actually_parsed = loader._parse_discovery_symbol(symbol)
            actually_parsed = (
                normalize_ending_slash(actually_parsed[0]),
                actually_parsed[1],
                actually_parsed[2],
                actually_parsed[3]
            )

            assert actually_parsed == expected_parsed, "%s did not parse as expected" % symbol
Example #28
0
    def check_parse_discovery_symbol(self):
        """Check that "test discovery symbol" parsing logic works correctly"""
        parsed_symbols = [
            {
                'dir': 'path/to/dir',
                'file': '',
                'cls': '',
                'method': ''
            },
            {
                'dir': 'path/to/dir',
                'file': 'test_file.py',
                'cls': '',
                'method': ''
            },
            {
                'dir': 'path/to/dir',
                'file': 'test_file.py',
                'cls': 'ClassName',
                'method': ''
            },
            {
                'dir': 'path/to/dir',
                'file': 'test_file.py',
                'cls': 'ClassName',
                'method': 'method'
            },
            {
                'dir': 'path/to/dir',
                'file': '',
                'cls': 'ClassName',
                'method': ''
            },
        ]

        loader = TestLoader(tests.ducktape_mock.session_context(), logger=Mock())
        for parsed in parsed_symbols:
            symbol = join_parsed_symbol_components(parsed)

            expected_parsed = (
                normalize_ending_slash(parsed['dir']),
                parsed['file'],
                parsed['cls'],
                parsed['method']
            )

            actually_parsed = loader._parse_discovery_symbol(symbol)
            actually_parsed = (
                normalize_ending_slash(actually_parsed[0]),
                actually_parsed[1],
                actually_parsed[2],
                actually_parsed[3]
            )

            assert actually_parsed == expected_parsed, "%s did not parse as expected" % symbol
Example #29
0
 def check_test_loader_with_multiple_matrix_params(self):
     loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())
     params = '[{"x": 1,"y": "test "}, {"x": 2,"y": "I\'m"}]'
     included = [os.path.join(discover_dir(), 'test_decorated.py::TestMatrix.test_thing@{}'.format(params))]
     tests = loader.load(included)
     # TestMatrix contains a single parametrized method.
     # We provide two sets of params, so we should end up with two contexts
     assert len(tests) == 2
     injected_args = map(lambda t: t.injected_args, tests)
     assert {'x': 1, 'y': 'test '} in injected_args
     assert {'x': 2, 'y': "I'm"} in injected_args
Example #30
0
    def _collect_test_context(self, directory, file_name, cls_name, method_name, injected_args):
        loader = TestLoader(self.session_context, self.logger, injected_args=injected_args, cluster=self.cluster)
        # TODO: deal with this in a more graceful fashion.
        #       In an unlikely even that discover either raises the exception or fails to find exactly one test
        #       we should probably continue trying other tests rather than killing this process
        loaded_context_list = loader.discover(directory, file_name, cls_name, method_name)

        assert len(loaded_context_list) == 1
        test_context = loaded_context_list[0]
        test_context.cluster = self.cluster
        return test_context
Example #31
0
 def check_test_loader_raises_on_params_not_found(self):
     loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())
     # parameter syntax is valid, but there is no such parameter defined in the test annotation in the code
     included = [
         os.path.join(
             discover_dir(),
             'test_decorated.py::TestMatrix.test_thing@{"x": 1,"y": "missing"}'
         )
     ]
     with pytest.raises(LoaderException, match='No tests to run'):
         loader.load(included)
Example #32
0
 def check_test_loader_with_invalid_subsets(self):
     """Check that the TestLoader throws an exception if the requests subset is larger than the number of subsets"""
     with pytest.raises(ValueError):
         TestLoader(self.SESSION_CONTEXT,
                    logger=Mock(),
                    subset=4,
                    subsets=4)
     with pytest.raises(ValueError):
         TestLoader(self.SESSION_CONTEXT,
                    logger=Mock(),
                    subset=5,
                    subsets=4)
Example #33
0
 def check_test_loader_exclude_with_params(self):
     """
     Checks behavior of exclude flag with parametrized annotations.
     Should exclude only a single permutation of the method
     """
     loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())
     # included 8 tests
     included = [os.path.join(discover_dir(), "test_decorated.py::TestMatrix")]
     # exclude 1 test
     excluded = [os.path.join(discover_dir(), 'test_decorated.py::TestMatrix.test_thing@{"x": 1,"y": "test "}')]
     tests = loader.load(included, excluded)
     assert len(tests) == 7
Example #34
0
 def check_test_loader_exclude_with_params_multiple(self):
     """
     Checks behavior of exclude flag with parametrized annotations.
     Should exclude two permutations of the method
     """
     loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())
     # include 8 tests
     included = [os.path.join(discover_dir(), "test_decorated.py::TestMatrix")]
     # exclude 2 tests
     params = '[{"x": 1,"y": "test "}, {"x": 2,"y": "I\'m"}]'
     excluded = [os.path.join(discover_dir(), 'test_decorated.py::TestMatrix.test_thing@{}'.format(params))]
     tests = loader.load(included, excluded)
     assert len(tests) == 6
Example #35
0
    def check_test_loader_exclude_with_injected_args(self):
        injected_args = {"x": 1, "y": -1}
        loader = TestLoader(self.SESSION_CONTEXT, logger=Mock(), injected_args=injected_args)

        included = [os.path.join(discover_dir(), "test_decorated.py")]
        excluded = [os.path.join(discover_dir(), "test_decorated.py::TestStackedMatrix")]
        tests = loader.load(included, excluded)
        # test_decorated.py contains 5 test methods total
        # we exclude 1 class with 1 method so should be 4
        # exclusion shouldn't care about injected args
        assert len(tests) == 5

        for t in tests:
            assert t.injected_args == injected_args
Example #36
0
    def check_test_loader_with_injected_args(self):
        """When the --parameters command-line option is used, the loader behaves a little bit differently:

        each test method annotated with @parametrize or @matrix should only expand to a single discovered test,
        and the injected args should be those passed in from command-line.
        """
        parameters = {"x": 1, "y": -1}
        loader = TestLoader(self.SESSION_CONTEXT, test_parameters=parameters)

        file = os.path.join(discover_dir(), "test_decorated.py")
        tests = loader.discover([file])
        assert len(tests) == 4

        for t in tests:
            assert t.injected_args == parameters
Example #37
0
    def check_test_loader_with_time_based_subsets(self):
        """Check that computation of subsets using a report with timing information correctly generates subsets that
        are optimized based on timing rather than number of tests.
        """

        file = os.path.join(discover_dir(), "test_b.py")
        report_url = "file://" + os.path.join(resources_dir(), "report.json")

        # The expected behavior of the current implementation is to add tests to each subset from largest to smallest,
        # using the least full subset each time. The test data with times of (10, 5, 1) should result in the first
        # subset containing 1 test and the second containing 2 (the opposite of the simple count-based strategy)

        loader = TestLoader(self.SESSION_CONTEXT, logger=Mock(), subset=0, subsets=2, historical_report=report_url)
        tests = loader.load([file])
        assert len(tests) == 1

        loader = TestLoader(self.SESSION_CONTEXT, logger=Mock(), subset=1, subsets=2, historical_report=report_url)
        tests = loader.load([file])
        assert len(tests) == 2
Example #38
0
    def check_test_loader_with_subsets(self):
        """Check that computation of subsets work properly. This validates both that the division of tests is correct
        (i.e. as even a distribution as we can get but uneven in the expected way when necessary) and that the division
        happens after the expansion of tests marked for possible expansion (e.g. matrix, parametrize)."""

        file = os.path.join(discover_dir(), "test_decorated.py")

        # The test file contains 15 tests. With 4 subsets, first three subsets should have an "extra"
        loader = TestLoader(self.SESSION_CONTEXT, logger=Mock(), subset=0, subsets=4)
        tests = loader.load([file])
        assert len(tests) == 4

        loader = TestLoader(self.SESSION_CONTEXT, logger=Mock(), subset=1, subsets=4)
        tests = loader.load([file])
        assert len(tests) == 4

        loader = TestLoader(self.SESSION_CONTEXT, logger=Mock(), subset=2, subsets=4)
        tests = loader.load([file])
        assert len(tests) == 4

        loader = TestLoader(self.SESSION_CONTEXT, logger=Mock(), subset=3, subsets=4)
        tests = loader.load([file])
        assert len(tests) == 3
Example #39
0
def main():
    """Ducktape entry point. This contains top level logic for ducktape command-line program which does the following:

        Discover tests
        Initialize cluster for distributed services
        Run tests
        Report a summary of all results
    """
    args_dict = parse_args(sys.argv[1:])

    parameters = None
    if args_dict["parameters"]:
        try:
            parameters = json.loads(args_dict["parameters"])
        except ValueError as e:
            print "parameters are not valid json: " + str(e.message)
            sys.exit(1)

    args_dict["globals"] = get_user_defined_globals(args_dict.get("globals"))

    # Make .ducktape directory where metadata such as the last used session_id is stored
    if not os.path.isdir(ConsoleDefaults.METADATA_DIR):
        os.makedirs(ConsoleDefaults.METADATA_DIR)

    # Generate a shared 'global' identifier for this test run and create the directory
    # in which all test results will be stored
    session_id = generate_session_id(ConsoleDefaults.SESSION_ID_FILE)
    results_dir = generate_results_dir(args_dict["results_root"], session_id)
    setup_results_directory(results_dir)

    session_context = SessionContext(session_id=session_id, results_dir=results_dir, **args_dict)
    for k, v in args_dict.iteritems():
        session_context.logger.debug("Configuration: %s=%s", k, v)

    # Discover and load tests to be run
    extend_import_paths(args_dict["test_path"])
    loader = TestLoader(session_context, parameters)
    try:
        tests = loader.discover(args_dict["test_path"])
    except LoaderException as e:
        print "Failed while trying to discover tests: {}".format(e)
        sys.exit(1)

    if args_dict["collect_only"]:
        print "Collected %d tests:" % len(tests)
        for test in tests:
            print "    " + str(test)
        sys.exit(0)

    # Initializing the cluster is slow, so do so only if
    # tests are sure to be run
    try:
        (cluster_mod_name, cluster_class_name) = args_dict["cluster"].rsplit('.', 1)
        cluster_mod = importlib.import_module(cluster_mod_name)
        cluster_class = getattr(cluster_mod, cluster_class_name)
        session_context.cluster = cluster_class(cluster_file=args_dict["cluster_file"])
    except:
        print "Failed to load cluster: ", str(sys.exc_info()[0])
        print traceback.format_exc(limit=16)
        sys.exit(1)

    # Run the tests
    runner = SerialTestRunner(session_context, tests)
    test_results = runner.run_all_tests()

    # Report results
    reporter = SimpleStdoutSummaryReporter(test_results)
    reporter.report()
    reporter = SimpleFileSummaryReporter(test_results)
    reporter.report()

    # Generate HTML reporter
    reporter = HTMLSummaryReporter(test_results)
    reporter.report()

    update_latest_symlink(args_dict["results_root"], results_dir)
    if not test_results.get_aggregate_success():
        sys.exit(1)
Example #40
0
def main():
    """Ducktape entry point. This contains top level logic for ducktape command-line program which does the following:

        Discover tests
        Initialize cluster for distributed services
        Run tests
        Report a summary of all results
    """
    args = parse_args()
    if args.version:
        print ducktape_version()
        sys.exit(0)

    # Make .ducktape directory where metadata such as the last used session_id is stored
    if not os.path.isdir(ConsoleConfig.METADATA_DIR):
        os.makedirs(ConsoleConfig.METADATA_DIR)

    # Generate a shared 'global' identifier for this test run and create the directory
    # in which all test results will be stored
    session_id = generate_session_id(ConsoleConfig.SESSION_ID_FILE)
    results_dir = generate_results_dir(args.results_root, session_id)

    setup_results_directory(args.results_root, results_dir)
    session_context = SessionContext(session_id, results_dir, cluster=None, args=args)
    for k, v in vars(args).iteritems():
        session_context.logger.debug("Configuration: %s=%s", k, v)

    # Discover and load tests to be run
    extend_import_paths(args.test_path)
    loader = TestLoader(session_context)
    try:
        tests = loader.discover(args.test_path)
    except LoaderException as e:
        print "Failed while trying to discover tests: {}".format(e)
        sys.exit(1)

    if args.collect_only:
        print "Collected %d tests:" % len(tests)
        for test in tests:
            print "    " + str(test)
        sys.exit(0)

    # Initializing the cluster is slow, so do so only if
    # tests are sure to be run
    try:
        (cluster_mod_name, cluster_class_name) = args.cluster.rsplit('.', 1)
        cluster_mod = importlib.import_module(cluster_mod_name)
        cluster_class = getattr(cluster_mod, cluster_class_name)
        session_context.cluster = cluster_class()
    except:
        print "Failed to load cluster: ", str(sys.exc_info()[0])
        print traceback.format_exc(limit=16)
        sys.exit(1)

    # Run the tests
    runner = SerialTestRunner(session_context, tests)
    test_results = runner.run_all_tests()

    # Report results
    # TODO command-line hook for type of reporter
    reporter = SimpleStdoutSummaryReporter(test_results)
    reporter.report()
    reporter = SimpleFileSummaryReporter(test_results)
    reporter.report()

    # Generate HTML reporter
    reporter = HTMLSummaryReporter(test_results)
    reporter.report()

    if not test_results.get_aggregate_success():
        sys.exit(1)
Example #41
0
 def check_test_loader_with_nonexistent_file(self):
     """Check discovery on a non-existent path should throw LoaderException"""
     with pytest.raises(LoaderException):
         loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())
         loader.load([os.path.join(discover_dir(), "file_that_does_not_exist.py")])
Example #42
0
 def check_test_loader_with_directory(self):
     """Check discovery on a directory."""
     loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())
     tests = loader.load([discover_dir()])
     assert len(tests) == num_tests_in_dir(discover_dir())
Example #43
0
 def check_test_loader_with_nonexistent_file(self):
     """Check discovery on a starting path that doesn't exist throws an"""
     with pytest.raises(LoaderException):
         loader = TestLoader(self.SESSION_CONTEXT)
         tests = loader.discover([os.path.join(discover_dir(), "file_that_does_not_exist.py")])
Example #44
0
 def check_test_loader_with_directory(self):
     """Check discovery on a directory."""
     loader = TestLoader(self.SESSION_CONTEXT)
     tests = loader.discover([discover_dir()])
     assert len(tests) == 5
Example #45
0
 def check_test_loader_with_file(self):
     """Check discovery on a file. """
     loader = TestLoader(self.SESSION_CONTEXT)
     tests = loader.discover([os.path.join(discover_dir(), "test_a.py")])
     assert len(tests) == 1