Exemple #1
0
def load_wpt_tests(xul_tester, requested_paths, excluded_paths, update_manifest=True):
    """Return a list of `RefTestCase` objects for the jsshell testharness.js
    tests filtered by the given paths and debug-ness."""
    repo_root = abspath(os.path.join(here, "..", "..", ".."))
    wp = os.path.join(repo_root, "testing", "web-platform")
    wpt = os.path.join(wp, "tests")

    sys_paths = [
        "python/mozterm",
        "python/mozboot",
        "testing/mozbase/mozcrash",
        "testing/mozbase/mozdevice",
        "testing/mozbase/mozfile",
        "testing/mozbase/mozinfo",
        "testing/mozbase/mozleak",
        "testing/mozbase/mozlog",
        "testing/mozbase/mozprocess",
        "testing/mozbase/mozprofile",
        "testing/mozbase/mozrunner",
        "testing/mozbase/mozversion",
        "testing/web-platform/",
        "testing/web-platform/tests/tools",
        "testing/web-platform/tests/tools/third_party/html5lib",
        "testing/web-platform/tests/tools/third_party/webencodings",
        "testing/web-platform/tests/tools/wptrunner",
        "testing/web-platform/tests/tools/wptserve",
        "third_party/python/requests",
    ]
    abs_sys_paths = [os.path.join(repo_root, path) for path in sys_paths]

    failed = False
    for path in abs_sys_paths:
        if not os.path.isdir(path):
            failed = True
            print("Could not add '%s' to the path")
    if failed:
        return []

    sys.path[0:0] = abs_sys_paths

    import manifestupdate
    from wptrunner import products, testloader, wptcommandline, wpttest, wptlogging

    manifest_root = tempfile.gettempdir()
    path_split = os.path.dirname(xul_tester.js_bin).split(os.path.sep)
    if path_split[-2:] == ["dist", "bin"]:
        maybe_root = os.path.join(*path_split[:-2])
        if os.path.exists(os.path.join(maybe_root, "_tests")):
            # Assume this is a gecko objdir.
            manifest_root = maybe_root

    logger = wptlogging.setup({}, {})

    test_manifests = manifestupdate.run(repo_root, manifest_root, logger,
                                        update=update_manifest)

    kwargs = vars(wptcommandline.create_parser().parse_args([]))
    kwargs.update({
        "config": os.path.join(manifest_root, "_tests", "web-platform", "wptrunner.local.ini"),
        "gecko_e10s": False,
        "verify": False,
        "wasm": xul_tester.test("wasmIsSupported()"),
    })
    wptcommandline.set_from_config(kwargs)

    def filter_jsshell_tests(it):
        for item_type, path, tests in it:
            tests = set(item for item in tests if item.jsshell)
            if tests:
                yield item_type, path, tests

    run_info_extras = products.load_product(kwargs["config"], "firefox")[-1](**kwargs)
    run_info = wpttest.get_run_info(kwargs["test_paths"]["/"]["metadata_path"],
                                    "firefox",
                                    debug=xul_tester.test("isDebugBuild"),
                                    extras=run_info_extras)
    release_or_beta = xul_tester.test("getBuildConfiguration().release_or_beta")
    run_info["release_or_beta"] = release_or_beta
    run_info["nightly_build"] = not release_or_beta

    path_filter = testloader.TestFilter(test_manifests,
                                        include=requested_paths,
                                        exclude=excluded_paths)
    loader = testloader.TestLoader(test_manifests,
                                   ["testharness"],
                                   run_info,
                                   manifest_filters=[path_filter, filter_jsshell_tests])

    extra_helper_paths = [
        os.path.join(here, "web-platform-test-shims.js"),
        os.path.join(wpt, "resources", "testharness.js"),
        os.path.join(here, "testharnessreport.js"),
    ]

    def resolve(test_path, script):
        if script.startswith("/"):
            return os.path.join(wpt, script[1:])

        return os.path.join(wpt, os.path.dirname(test_path), script)

    tests = []
    for test in loader.tests["testharness"]:
        test_path = os.path.relpath(test.path, wpt)
        scripts = [resolve(test_path, s) for s in test.scripts]
        extra_helper_paths_for_test = extra_helper_paths + scripts

        # We must create at least one test with the default options, along with
        # one test for each option given in a test-also annotation.
        options = [None]
        for m in test.itermeta():
            if m.has_key("test-also"):  # NOQA: W601
                options += m.get("test-also").split()
        for option in options:
            test_case = RefTestCase(
                wpt,
                test_path,
                extra_helper_paths=extra_helper_paths_for_test[:],
                wpt=test
            )
            if option:
                test_case.options.append(option)
            tests.append(test_case)
    return tests
Exemple #2
0
def load_wpt_tests(xul_tester, requested_paths, excluded_paths):
    """Return a list of `RefTestCase` objects for the jsshell testharness.js
    tests filtered by the given paths and debug-ness."""
    repo_root = abspath(os.path.join(here, "..", "..", ".."))
    wp = os.path.join(repo_root, "testing", "web-platform")
    wpt = os.path.join(wp, "tests")

    sys_paths = [
        "python/mozterm",
        "python/mozboot",
        "testing/mozbase/mozcrash",
        "testing/mozbase/mozdevice",
        "testing/mozbase/mozfile",
        "testing/mozbase/mozinfo",
        "testing/mozbase/mozleak",
        "testing/mozbase/mozlog",
        "testing/mozbase/mozprocess",
        "testing/mozbase/mozprofile",
        "testing/mozbase/mozrunner",
        "testing/web-platform/",
        "testing/web-platform/tests/tools",
        "testing/web-platform/tests/tools/third_party/html5lib",
        "testing/web-platform/tests/tools/third_party/webencodings",
        "testing/web-platform/tests/tools/wptrunner",
        "testing/web-platform/tests/tools/wptserve",
        "third_party/python/requests",
    ]
    abs_sys_paths = [os.path.join(repo_root, path) for path in sys_paths]

    failed = False
    for path in abs_sys_paths:
        if not os.path.isdir(path):
            failed = True
            print("Could not add '%s' to the path")
    if failed:
        return []

    sys.path[0:0] = abs_sys_paths

    import manifestupdate
    from wptrunner import products, testloader, wptcommandline, wpttest, wptlogging

    manifest_root = tempfile.gettempdir()
    path_split = os.path.dirname(xul_tester.js_bin).split(os.path.sep)
    if path_split[-2:] == ["dist", "bin"]:
        maybe_root = os.path.join(*path_split[:-2])
        if os.path.exists(os.path.join(maybe_root, "_tests")):
            # Assume this is a gecko objdir.
            manifest_root = maybe_root

    logger = wptlogging.setup({}, {})

    manifestupdate.run(repo_root, manifest_root, logger)

    kwargs = vars(wptcommandline.create_parser().parse_args([]))
    kwargs.update({
        "config": os.path.join(manifest_root, "_tests", "web-platform", "wptrunner.local.ini"),
        "gecko_e10s": False,
        "verify": False,
        "wasm": xul_tester.test("wasmIsSupported()"),
    })
    wptcommandline.set_from_config(kwargs)
    test_paths = kwargs["test_paths"]

    def filter_jsshell_tests(it):
        for test in it:
            if test[1].get("jsshell"):
                yield test

    test_manifests = testloader.ManifestLoader(test_paths, types=["testharness"],
                                               meta_filters=[filter_jsshell_tests]).load()

    run_info_extras = products.load_product(kwargs["config"], "firefox")[-1](**kwargs)
    run_info = wpttest.get_run_info(kwargs["test_paths"]["/"]["metadata_path"],
                                    "firefox",
                                    debug=xul_tester.test("isDebugBuild"),
                                    extras=run_info_extras)

    path_filter = testloader.TestFilter(test_manifests,
                                        include=requested_paths,
                                        exclude=excluded_paths)
    loader = testloader.TestLoader(test_manifests,
                                   ["testharness"],
                                   run_info,
                                   manifest_filters=[path_filter])

    extra_helper_paths = [
        os.path.join(here, "web-platform-test-shims.js"),
        os.path.join(wpt, "resources", "testharness.js"),
        os.path.join(here, "testharnessreport.js"),
    ]

    def resolve(test_path, script):
        if script.startswith("/"):
            return os.path.join(wpt, script[1:])

        return os.path.join(wpt, os.path.dirname(test_path), script)

    return [
        RefTestCase(
            wpt,
            test_path,
            extra_helper_paths=extra_helper_paths + [resolve(test_path, s) for s in test.scripts],
            wpt=test
        )
        for test_path, test in (
            (os.path.relpath(test.path, wpt), test) for test in loader.tests["testharness"]
        )
    ]
Exemple #3
0
def run(logger, src_root, obj_root, **kwargs):
    commandline.setup_logging(logger, {
        key: value
        for key, value in kwargs.items() if key.startswith("log_")
    })

    import manifestupdate

    sys.path.insert(
        0,
        os.path.abspath(
            os.path.join(os.path.dirname(__file__), "tests", "tools")),
    )
    from wptrunner import testloader, wpttest

    logger.info("Loading test manifest")
    test_manifests = manifestupdate.run(src_root, obj_root, logger)

    test_results = {}

    platforms = kwargs["platforms"]
    if platforms is None:
        platforms = run_infos.keys()

    for platform in platforms:
        platform_run_info = run_infos[platform]
        run_info_baseline = platform_run_info.copy()
        run_info_baseline["fission"] = False

        tests = {}

        for kind in ("baseline", "fission"):
            logger.info("Loading tests %s %s" % (platform, kind))
            run_info = platform_run_info.copy()
            run_info["fission"] = kind == "fission"

            test_loader = testloader.TestLoader(test_manifests,
                                                wpttest.enabled_tests,
                                                run_info,
                                                manifest_filters=[])
            tests[kind] = {
                test.id: test
                for _, _, test in test_loader.iter_tests()
                if test._test_metadata is not None
            }

        for test_id, baseline_test in tests["baseline"].items():
            fission_test = tests["fission"][test_id]

            if test_id not in test_results:
                test_results[test_id] = TestResult()

            test_result = test_results[test_id]

            baseline_bug = get_meta_prop(baseline_test, None, "bug")
            fission_bug = get_meta_prop(fission_test, None, "bug")
            if fission_bug and fission_bug != baseline_bug:
                test_result.bugs.add(fission_bug)

            if fission_test.disabled() and not baseline_test.disabled():
                test_result.disabled.add(platform)
                reason = get_meta_prop(fission_test, None, "disabled")
                if reason and maybe_bug_re.match(reason):
                    test_result.bugs.add(reason)

            baseline_results = allowed_results(baseline_test)
            fission_results = allowed_results(fission_test)
            result_is_regression = is_regression(baseline_results,
                                                 fission_results)

            if baseline_results != fission_results:
                logger.debug("  %s %s %s %s" %
                             (test_id, baseline_results, fission_results,
                              result_is_regression))

            if result_is_regression:
                test_result.add_regression(platform, baseline_results,
                                           fission_results)

            for (
                    name,
                    baseline_subtest_meta,
            ) in baseline_test._test_metadata.subtests.items():
                fission_subtest_meta = baseline_test._test_metadata.subtests[
                    name]
                if name not in test_result.subtest_results:
                    test_result.add_subtest(name)

                subtest_result = test_result.subtest_results[name]

                baseline_bug = get_meta_prop(baseline_test, name, "bug")
                fission_bug = get_meta_prop(fission_test, name, "bug")
                if fission_bug and fission_bug != baseline_bug:
                    subtest_result.bugs.add(fission_bug)

                if bool(fission_subtest_meta.disabled) and not bool(
                        baseline_subtest_meta.disabled):
                    subtest_result.disabled.add(platform)
                    if maybe_bug_re.match(fission_subtest_meta.disabled):
                        subtest_result.bugs.add(fission_subtest_meta.disabled)

                baseline_results = allowed_results(baseline_test, name)
                fission_results = allowed_results(fission_test, name)

                result_is_regression = is_regression(baseline_results,
                                                     fission_results)

                if baseline_results != fission_results:
                    logger.debug("    %s %s %s %s %s" % (
                        test_id,
                        name,
                        baseline_results,
                        fission_results,
                        result_is_regression,
                    ))

                if result_is_regression:
                    subtest_result.add_regression(platform, baseline_results,
                                                  fission_results)

    test_results = {
        test_id: result
        for test_id, result in test_results.items() if include_result(result)
    }

    if kwargs["all_json"] is not None:
        write_all(test_results, kwargs["all_json"])

    if kwargs["untriaged"] is not None:
        write_untriaged(test_results, kwargs["untriaged"])
Exemple #4
0
def load_wpt_tests(requested_paths, excluded_paths, debug, wasm):
    """Return a list of `RefTestCase` objects for the jsshell testharness.js
    tests filtered by the given paths and debug-ness."""
    repo_root = abspath(os.path.join(here, "..", "..", ".."))
    wp = os.path.join(repo_root, "testing", "web-platform")
    wpt = os.path.join(wp, "tests")

    sys_paths = [
        "python/mozterm",
        "testing/mozbase/mozcrash",
        "testing/mozbase/mozdevice",
        "testing/mozbase/mozfile",
        "testing/mozbase/mozinfo",
        "testing/mozbase/mozleak",
        "testing/mozbase/mozlog",
        "testing/mozbase/mozprocess",
        "testing/mozbase/mozprofile",
        "testing/mozbase/mozrunner",
        "testing/web-platform/tests/tools",
        "testing/web-platform/tests/tools/third_party/html5lib",
        "testing/web-platform/tests/tools/third_party/webencodings",
        "testing/web-platform/tests/tools/wptrunner",
        "testing/web-platform/tests/tools/wptserve",
    ]
    abs_sys_paths = [os.path.join(repo_root, path) for path in sys_paths]

    failed = False
    for path in abs_sys_paths:
        if not os.path.isdir(path):
            failed = True
            print("Could not add '%s' to the path")
    if failed:
        return []

    sys.path[0:0] = abs_sys_paths

    from wptrunner import products, testloader, wptcommandline, wpttest, wptlogging

    wptlogging.setup({}, {})
    kwargs = {
        "config": None,
        "tests_root": wpt,
        "metadata_root": os.path.join(wp, "meta"),
        "gecko_e10s": False,
        "verify": False,
        "wasm": wasm,
    }
    wptcommandline.set_from_config(kwargs)
    test_paths = kwargs["test_paths"]

    def filter_jsshell_tests(it):
        for test in it:
            if test[1].get("jsshell"):
                yield test

    test_manifests = testloader.ManifestLoader(test_paths,
                                               types=["testharness"],
                                               meta_filters=[
                                                   filter_jsshell_tests
                                               ]).load()

    run_info_extras = products.load_product(kwargs["config"],
                                            "firefox")[-1](**kwargs)
    run_info = wpttest.get_run_info(kwargs["metadata_root"],
                                    "firefox",
                                    debug=debug,
                                    extras=run_info_extras)

    path_filter = testloader.TestFilter(test_manifests,
                                        include=requested_paths,
                                        exclude=excluded_paths)
    loader = testloader.TestLoader(test_manifests, ["testharness"],
                                   run_info,
                                   manifest_filters=[path_filter])

    extra_helper_paths = [
        os.path.join(wpt, "resources", "testharness.js"),
        os.path.join(here, "testharnessreport.js"),
    ]

    def resolve(test_path, script):
        if script.startswith("/"):
            return os.path.join(wpt, script[1:])

        return os.path.join(wpt, os.path.dirname(test_path), script)

    return [
        RefTestCase(wpt,
                    test_path,
                    extra_helper_paths=extra_helper_paths +
                    [resolve(test_path, s) for s in test.scripts],
                    wpt=test)
        for test_path, test in ((os.path.relpath(test.path, wpt), test)
                                for test in loader.tests["testharness"])
    ]