def run(src_root, obj_root, logger_=None, **kwargs):
logger_obj = logger_ if logger_ is not None else logger
manifests = manifestupdate.run(src_root, obj_root, logger_obj, **kwargs)
rv = {}
dirs_seen = set()
for meta_root, test_path, test_metadata in iter_tests(manifests):
for dir_path in get_dir_paths(meta_root, test_path):
if dir_path not in dirs_seen:
dirs_seen.add(dir_path)
dir_manifest = get_dir_manifest(dir_path)
rel_path = os.path.relpath(dir_path, meta_root)
if dir_manifest:
add_manifest(rv, rel_path, dir_manifest)
else:
break
add_manifest(rv, test_path, test_metadata)
if kwargs["out_dir"]:
if not os.path.exists(kwargs["out_dir"]):
os.makedirs(kwargs["out_dir"])
out_path = os.path.join(kwargs["out_dir"], "summary.json")
with open(out_path, "w") as f:
json.dump(rv, f)
else:
print(json.dumps(rv, indent=2))
if kwargs["meta_dir"]:
update_wpt_meta(logger_obj, kwargs["meta_dir"], rv)
def main():
print("Downloading wpt manifest")
sys.path.insert(0, buildconfig.topsrcdir)
import manifestupdate
return 0 if manifestupdate.run(buildconfig.topsrcdir,
buildconfig.topobjdir) else 1
def update_manifest(self, logger, **kwargs):
import manifestupdate
return manifestupdate.run(logger=logger,
src_root=self.topsrcdir,
obj_root=self.topobjdir,
**kwargs)
def add_wpt_manifest_data(self):
if self._wpt_loaded:
return
wpt_path = os.path.join(self._srcdir, "testing", "web-platform")
sys.path = [wpt_path] + sys.path
import manifestupdate
# Set up a logger that will drop all the output
import logging
logger = logging.getLogger("manifestupdate")
logger.propogate = False
manifests = manifestupdate.run(self._srcdir,
self._objdir,
rebuild=False,
download=True,
config_path=None,
rewrite_config=True,
update=True,
logger=logger)
if not manifests:
print("Loading wpt manifest failed")
return
for manifest, data in manifests.iteritems():
tests_root = data["tests_path"]
for test_type, path, tests in manifest:
full_path = os.path.join(tests_root, path)
src_path = os.path.relpath(full_path, self._srcdir)
if test_type not in ["testharness", "reftest", "wdspec"]:
continue
for test in tests:
self._tests_by_path[src_path].append({
"path":
full_path,
"flavor":
"web-platform-tests",
"here":
os.path.dirname(path),
"manifest":
data["manifest_path"],
"name":
test.id,
"file_relpath":
path,
"head":
"",
"support-files":
"",
"subsuite":
test_type,
"dir_relpath":
os.path.dirname(src_path),
"srcdir_relpath":
src_path,
})
self._wpt_loaded = True
def load_wpt_tests(xul_tester, requested_paths, excluded_paths, update_manifest=True):
"""Return a list of `RefTestCase` objects for the jsshell testharness.js
tests filtered by the given paths and debug-ness."""
repo_root = abspath(os.path.join(here, "..", "..", ".."))
wp = os.path.join(repo_root, "testing", "web-platform")
wpt = os.path.join(wp, "tests")
sys_paths = [
"python/mozterm",
"python/mozboot",
"testing/mozbase/mozcrash",
"testing/mozbase/mozdevice",
"testing/mozbase/mozfile",
"testing/mozbase/mozinfo",
"testing/mozbase/mozleak",
"testing/mozbase/mozlog",
"testing/mozbase/mozprocess",
"testing/mozbase/mozprofile",
"testing/mozbase/mozrunner",
"testing/mozbase/mozversion",
"testing/web-platform/",
"testing/web-platform/tests/tools",
"testing/web-platform/tests/tools/third_party/html5lib",
"testing/web-platform/tests/tools/third_party/webencodings",
"testing/web-platform/tests/tools/wptrunner",
"testing/web-platform/tests/tools/wptserve",
"third_party/python/requests",
]
abs_sys_paths = [os.path.join(repo_root, path) for path in sys_paths]
failed = False
for path in abs_sys_paths:
if not os.path.isdir(path):
failed = True
print("Could not add '%s' to the path")
if failed:
return []
sys.path[0:0] = abs_sys_paths
import manifestupdate
from wptrunner import products, testloader, wptcommandline, wpttest, wptlogging
manifest_root = tempfile.gettempdir()
path_split = os.path.dirname(xul_tester.js_bin).split(os.path.sep)
if path_split[-2:] == ["dist", "bin"]:
maybe_root = os.path.join(*path_split[:-2])
if os.path.exists(os.path.join(maybe_root, "_tests")):
# Assume this is a gecko objdir.
manifest_root = maybe_root
logger = wptlogging.setup({}, {})
test_manifests = manifestupdate.run(repo_root, manifest_root, logger,
update=update_manifest)
kwargs = vars(wptcommandline.create_parser().parse_args([]))
kwargs.update({
"config": os.path.join(manifest_root, "_tests", "web-platform", "wptrunner.local.ini"),
"gecko_e10s": False,
"verify": False,
"wasm": xul_tester.test("wasmIsSupported()"),
})
wptcommandline.set_from_config(kwargs)
def filter_jsshell_tests(it):
for item_type, path, tests in it:
tests = set(item for item in tests if item.jsshell)
if tests:
yield item_type, path, tests
run_info_extras = products.load_product(kwargs["config"], "firefox")[-1](**kwargs)
run_info = wpttest.get_run_info(kwargs["test_paths"]["/"]["metadata_path"],
"firefox",
debug=xul_tester.test("isDebugBuild"),
extras=run_info_extras)
release_or_beta = xul_tester.test("getBuildConfiguration().release_or_beta")
run_info["release_or_beta"] = release_or_beta
run_info["nightly_build"] = not release_or_beta
path_filter = testloader.TestFilter(test_manifests,
include=requested_paths,
exclude=excluded_paths)
loader = testloader.TestLoader(test_manifests,
["testharness"],
run_info,
manifest_filters=[path_filter, filter_jsshell_tests])
extra_helper_paths = [
os.path.join(here, "web-platform-test-shims.js"),
os.path.join(wpt, "resources", "testharness.js"),
os.path.join(here, "testharnessreport.js"),
]
def resolve(test_path, script):
if script.startswith("/"):
return os.path.join(wpt, script[1:])
return os.path.join(wpt, os.path.dirname(test_path), script)
tests = []
for test in loader.tests["testharness"]:
test_path = os.path.relpath(test.path, wpt)
scripts = [resolve(test_path, s) for s in test.scripts]
extra_helper_paths_for_test = extra_helper_paths + scripts
# We must create at least one test with the default options, along with
# one test for each option given in a test-also annotation.
options = [None]
for m in test.itermeta():
if m.has_key("test-also"): # NOQA: W601
options += m.get("test-also").split()
for option in options:
test_case = RefTestCase(
wpt,
test_path,
extra_helper_paths=extra_helper_paths_for_test[:],
wpt=test
)
if option:
test_case.options.append(option)
tests.append(test_case)
return tests
def load_wpt_tests(xul_tester, requested_paths, excluded_paths):
"""Return a list of `RefTestCase` objects for the jsshell testharness.js
tests filtered by the given paths and debug-ness."""
repo_root = abspath(os.path.join(here, "..", "..", ".."))
wp = os.path.join(repo_root, "testing", "web-platform")
wpt = os.path.join(wp, "tests")
sys_paths = [
"python/mozterm",
"python/mozboot",
"testing/mozbase/mozcrash",
"testing/mozbase/mozdevice",
"testing/mozbase/mozfile",
"testing/mozbase/mozinfo",
"testing/mozbase/mozleak",
"testing/mozbase/mozlog",
"testing/mozbase/mozprocess",
"testing/mozbase/mozprofile",
"testing/mozbase/mozrunner",
"testing/web-platform/",
"testing/web-platform/tests/tools",
"testing/web-platform/tests/tools/third_party/html5lib",
"testing/web-platform/tests/tools/third_party/webencodings",
"testing/web-platform/tests/tools/wptrunner",
"testing/web-platform/tests/tools/wptserve",
"third_party/python/requests",
]
abs_sys_paths = [os.path.join(repo_root, path) for path in sys_paths]
failed = False
for path in abs_sys_paths:
if not os.path.isdir(path):
failed = True
print("Could not add '%s' to the path")
if failed:
return []
sys.path[0:0] = abs_sys_paths
import manifestupdate
from wptrunner import products, testloader, wptcommandline, wpttest, wptlogging
manifest_root = tempfile.gettempdir()
path_split = os.path.dirname(xul_tester.js_bin).split(os.path.sep)
if path_split[-2:] == ["dist", "bin"]:
maybe_root = os.path.join(*path_split[:-2])
if os.path.exists(os.path.join(maybe_root, "_tests")):
# Assume this is a gecko objdir.
manifest_root = maybe_root
logger = wptlogging.setup({}, {})
manifestupdate.run(repo_root, manifest_root, logger)
kwargs = vars(wptcommandline.create_parser().parse_args([]))
kwargs.update({
"config": os.path.join(manifest_root, "_tests", "web-platform", "wptrunner.local.ini"),
"gecko_e10s": False,
"verify": False,
"wasm": xul_tester.test("wasmIsSupported()"),
})
wptcommandline.set_from_config(kwargs)
test_paths = kwargs["test_paths"]
def filter_jsshell_tests(it):
for test in it:
if test[1].get("jsshell"):
yield test
test_manifests = testloader.ManifestLoader(test_paths, types=["testharness"],
meta_filters=[filter_jsshell_tests]).load()
run_info_extras = products.load_product(kwargs["config"], "firefox")[-1](**kwargs)
run_info = wpttest.get_run_info(kwargs["test_paths"]["/"]["metadata_path"],
"firefox",
debug=xul_tester.test("isDebugBuild"),
extras=run_info_extras)
path_filter = testloader.TestFilter(test_manifests,
include=requested_paths,
exclude=excluded_paths)
loader = testloader.TestLoader(test_manifests,
["testharness"],
run_info,
manifest_filters=[path_filter])
extra_helper_paths = [
os.path.join(here, "web-platform-test-shims.js"),
os.path.join(wpt, "resources", "testharness.js"),
os.path.join(here, "testharnessreport.js"),
]
def resolve(test_path, script):
if script.startswith("/"):
return os.path.join(wpt, script[1:])
return os.path.join(wpt, os.path.dirname(test_path), script)
return [
RefTestCase(
wpt,
test_path,
extra_helper_paths=extra_helper_paths + [resolve(test_path, s) for s in test.scripts],
wpt=test
)
for test_path, test in (
(os.path.relpath(test.path, wpt), test) for test in loader.tests["testharness"]
)
]
def add_wpt_manifest_data(self):
"""Adds manifest data for web-platform-tests into the list of available tests.
Upon invocation, this method will download from firefox-ci the most recent
version of the web-platform-tests manifests.
Once manifest is downloaded, this method will add details about each test
into the list of available tests.
"""
if self._wpt_loaded:
return
self._reset_state()
wpt_path = os.path.join(self.topsrcdir, "testing", "web-platform")
sys.path = [wpt_path] + sys.path
import manifestupdate
import logging
logger = logging.getLogger("manifestupdate")
logger.disabled = True
manifests = manifestupdate.run(
self.topsrcdir,
self.topobjdir,
rebuild=False,
download=True,
config_path=None,
rewrite_config=True,
update=True,
logger=logger,
)
if not manifests:
print("Loading wpt manifest failed")
return
for manifest, data in six.iteritems(manifests):
tests_root = data[
"tests_path"
] # full path on disk until web-platform tests directory
for test_type, path, tests in manifest:
full_path = mozpath.join(tests_root, path)
src_path = mozpath.relpath(full_path, self.topsrcdir)
if test_type not in ["testharness", "reftest", "wdspec", "crashtest"]:
continue
full_path = mozpath.join(tests_root, path) # absolute path on disk
src_path = mozpath.relpath(full_path, self.topsrcdir)
for test in tests:
testobj = {
"head": "",
"support-files": "",
"path": full_path,
"flavor": "web-platform-tests",
"subsuite": test_type,
"here": mozpath.dirname(path),
"name": test.id,
"file_relpath": src_path,
"srcdir_relpath": src_path,
"dir_relpath": mozpath.dirname(src_path),
}
group = self.get_wpt_group(testobj)
testobj["manifest"] = group
test_root = "tests"
if group.startswith("/_mozilla"):
test_root = os.path.join("mozilla", "tests")
group = group[len("/_mozilla") :]
group = group.lstrip("/")
testobj["manifest_relpath"] = os.path.join(
wpt_path, test_root, group
)
self._tests.append(testobj)
self._wpt_loaded = True
def run(logger, src_root, obj_root, **kwargs):
commandline.setup_logging(logger, {
key: value
for key, value in kwargs.items() if key.startswith("log_")
})
import manifestupdate
sys.path.insert(
0,
os.path.abspath(
os.path.join(os.path.dirname(__file__), "tests", "tools")),
)
from wptrunner import testloader, wpttest
logger.info("Loading test manifest")
test_manifests = manifestupdate.run(src_root, obj_root, logger)
test_results = {}
platforms = kwargs["platforms"]
if platforms is None:
platforms = run_infos.keys()
for platform in platforms:
platform_run_info = run_infos[platform]
run_info_baseline = platform_run_info.copy()
run_info_baseline["fission"] = False
tests = {}
for kind in ("baseline", "fission"):
logger.info("Loading tests %s %s" % (platform, kind))
run_info = platform_run_info.copy()
run_info["fission"] = kind == "fission"
test_loader = testloader.TestLoader(test_manifests,
wpttest.enabled_tests,
run_info,
manifest_filters=[])
tests[kind] = {
test.id: test
for _, _, test in test_loader.iter_tests()
if test._test_metadata is not None
}
for test_id, baseline_test in tests["baseline"].items():
fission_test = tests["fission"][test_id]
if test_id not in test_results:
test_results[test_id] = TestResult()
test_result = test_results[test_id]
baseline_bug = get_meta_prop(baseline_test, None, "bug")
fission_bug = get_meta_prop(fission_test, None, "bug")
if fission_bug and fission_bug != baseline_bug:
test_result.bugs.add(fission_bug)
if fission_test.disabled() and not baseline_test.disabled():
test_result.disabled.add(platform)
reason = get_meta_prop(fission_test, None, "disabled")
if reason and maybe_bug_re.match(reason):
test_result.bugs.add(reason)
baseline_results = allowed_results(baseline_test)
fission_results = allowed_results(fission_test)
result_is_regression = is_regression(baseline_results,
fission_results)
if baseline_results != fission_results:
logger.debug(" %s %s %s %s" %
(test_id, baseline_results, fission_results,
result_is_regression))
if result_is_regression:
test_result.add_regression(platform, baseline_results,
fission_results)
for (
name,
baseline_subtest_meta,
) in baseline_test._test_metadata.subtests.items():
fission_subtest_meta = baseline_test._test_metadata.subtests[
name]
if name not in test_result.subtest_results:
test_result.add_subtest(name)
subtest_result = test_result.subtest_results[name]
baseline_bug = get_meta_prop(baseline_test, name, "bug")
fission_bug = get_meta_prop(fission_test, name, "bug")
if fission_bug and fission_bug != baseline_bug:
subtest_result.bugs.add(fission_bug)
if bool(fission_subtest_meta.disabled) and not bool(
baseline_subtest_meta.disabled):
subtest_result.disabled.add(platform)
if maybe_bug_re.match(fission_subtest_meta.disabled):
subtest_result.bugs.add(fission_subtest_meta.disabled)
baseline_results = allowed_results(baseline_test, name)
fission_results = allowed_results(fission_test, name)
result_is_regression = is_regression(baseline_results,
fission_results)
if baseline_results != fission_results:
logger.debug(" %s %s %s %s %s" % (
test_id,
name,
baseline_results,
fission_results,
result_is_regression,
))
if result_is_regression:
subtest_result.add_regression(platform, baseline_results,
fission_results)
test_results = {
test_id: result
for test_id, result in test_results.items() if include_result(result)
}
if kwargs["all_json"] is not None:
write_all(test_results, kwargs["all_json"])
if kwargs["untriaged"] is not None:
write_untriaged(test_results, kwargs["untriaged"])
def load_wpt_tests(xul_tester, requested_paths, excluded_paths):
"""Return a list of `RefTestCase` objects for the jsshell testharness.js
tests filtered by the given paths and debug-ness."""
repo_root = abspath(os.path.join(here, "..", "..", ".."))
wp = os.path.join(repo_root, "testing", "web-platform")
wpt = os.path.join(wp, "tests")
sys_paths = [
"python/mozterm",
"python/mozboot",
"testing/mozbase/mozcrash",
"testing/mozbase/mozdevice",
"testing/mozbase/mozfile",
"testing/mozbase/mozinfo",
"testing/mozbase/mozleak",
"testing/mozbase/mozlog",
"testing/mozbase/mozprocess",
"testing/mozbase/mozprofile",
"testing/mozbase/mozrunner",
"testing/web-platform/",
"testing/web-platform/tests/tools",
"testing/web-platform/tests/tools/third_party/html5lib",
"testing/web-platform/tests/tools/third_party/webencodings",
"testing/web-platform/tests/tools/wptrunner",
"testing/web-platform/tests/tools/wptserve",
"third_party/python/requests",
]
abs_sys_paths = [os.path.join(repo_root, path) for path in sys_paths]
failed = False
for path in abs_sys_paths:
if not os.path.isdir(path):
failed = True
print("Could not add '%s' to the path")
if failed:
return []
sys.path[0:0] = abs_sys_paths
import manifestupdate
from wptrunner import products, testloader, wptcommandline, wpttest, wptlogging
manifest_root = tempfile.gettempdir()
path_split = os.path.dirname(xul_tester.js_bin).split(os.path.sep)
if path_split[-2:] == ["dist", "bin"]:
maybe_root = os.path.join(*path_split[:-2])
if os.path.exists(os.path.join(maybe_root, "_tests")):
# Assume this is a gecko objdir.
manifest_root = maybe_root
logger = wptlogging.setup({}, {})
manifestupdate.run(repo_root, manifest_root, logger)
kwargs = vars(wptcommandline.create_parser().parse_args([]))
kwargs.update({
"config": os.path.join(manifest_root, "_tests", "web-platform", "wptrunner.local.ini"),
"gecko_e10s": False,
"verify": False,
"wasm": xul_tester.test("wasmIsSupported()"),
})
wptcommandline.set_from_config(kwargs)
test_paths = kwargs["test_paths"]
def filter_jsshell_tests(it):
for test in it:
if test[1].get("jsshell"):
yield test
test_manifests = testloader.ManifestLoader(test_paths, types=["testharness"],
meta_filters=[filter_jsshell_tests]).load()
run_info_extras = products.load_product(kwargs["config"], "firefox")[-1](**kwargs)
run_info = wpttest.get_run_info(kwargs["test_paths"]["/"]["metadata_path"],
"firefox",
debug=xul_tester.test("isDebugBuild"),
extras=run_info_extras)
path_filter = testloader.TestFilter(test_manifests,
include=requested_paths,
exclude=excluded_paths)
loader = testloader.TestLoader(test_manifests,
["testharness"],
run_info,
manifest_filters=[path_filter])
extra_helper_paths = [
os.path.join(here, "web-platform-test-shims.js"),
os.path.join(wpt, "resources", "testharness.js"),
os.path.join(here, "testharnessreport.js"),
]
def resolve(test_path, script):
if script.startswith("/"):
return os.path.join(wpt, script[1:])
return os.path.join(wpt, os.path.dirname(test_path), script)
return [
RefTestCase(
wpt,
test_path,
extra_helper_paths=extra_helper_paths + [resolve(test_path, s) for s in test.scripts],
wpt=test
)
for test_path, test in (
(os.path.relpath(test.path, wpt), test) for test in loader.tests["testharness"]
)
]