def test_tests_affected_idlharness(capsys, manifest_dir): commit = "47cea8c38b88c0ddd3854e4edec0c5b6f2697e62" with pytest.raises(SystemExit) as excinfo: wpt.main(argv=["tests-affected", "--metadata", manifest_dir, "%s~..%s" % (commit, commit)]) assert excinfo.value.code == 0 out, err = capsys.readouterr() assert "webrtc-stats/idlharness.window.js\nwebrtc/idlharness.https.window.js\n" == out
def test_run_verify_unstable(temp_test): """Unstable tests should be reported with a non-zero exit status. Stable tests should be reported with a zero exit status.""" if is_port_8000_in_use(): pytest.skip("port 8000 already in use") unstable_test = temp_test(""" test(function() { if (localStorage.getItem('wpt-unstable-test-flag')) { throw new Error(); } localStorage.setItem('wpt-unstable-test-flag', 'x'); }, 'my test'); """) with pytest.raises(SystemExit) as excinfo: wpt.main(argv=["run", "--yes", "--verify", "--channel", "dev", # Taskcluster machines do not have GPUs, so use software rendering via --enable-swiftshader. "--enable-swiftshader", "chrome", unstable_test]) assert excinfo.value.code != 0 stable_test = temp_test("test(function() {}, 'my test');") with pytest.raises(SystemExit) as excinfo: wpt.main(argv=["run", "--yes", "--verify", "--channel", "dev", # Taskcluster machines do not have GPUs, so use software rendering via --enable-swiftshader. "--enable-swiftshader", "chrome", stable_test]) assert excinfo.value.code == 0
def init_manifest(): if sys.version_info >= (3,): pytest.xfail(reason="broken on Py3") with pytest.raises(SystemExit) as excinfo: wpt.main(argv=["manifest", "--no-download", "--path", get_persistent_manifest_path()]) assert excinfo.value.code == 0
def test_run_zero_tests(): """A test execution describing zero tests should be reported as an error even in the presence of the `--no-fail-on-unexpected` option.""" if is_port_8000_in_use(): pytest.skip("port 8000 already in use") with pytest.raises(SystemExit) as excinfo: wpt.main(argv=[ "run", "--yes", "--no-pause", "--channel", "dev", # Taskcluster machines do not have GPUs, so use software rendering via --enable-swiftshader. "--enable-swiftshader", "chrome", "/non-existent-dir/non-existent-file.html" ]) assert excinfo.value.code != 0 with pytest.raises(SystemExit) as excinfo: wpt.main(argv=[ "run", "--yes", "--no-pause", "--no-fail-on-unexpected", "--channel", "dev", # Taskcluster machines do not have GPUs, so use software rendering via --enable-swiftshader. "--enable-swiftshader", "chrome", "/non-existent-dir/non-existent-file.html" ]) assert excinfo.value.code != 0
def test_list_tests_invalid_manifest(manifest_dir): """The `--list-tests` option should not produce an error in the presence of a malformed test manifest file.""" manifest_filename = os.path.join(manifest_dir, "MANIFEST.json") assert os.path.isfile(manifest_filename) with open(manifest_filename, "a+") as handle: handle.write("extra text which invalidates the file") with pytest.raises(SystemExit) as excinfo: wpt.main(argv=["run", # This test triggers the creation of a new manifest # file which is not necessary to ensure successful # process completion. Specifying the current directory # as the tests source via the --tests` option # drastically reduces the time to execute the test. "--tests", here, "--metadata", manifest_dir, "--list-tests", "--yes", "firefox", "/dom/nodes/Element-tagName.html"]) assert excinfo.value.code == 0
def test_run_firefox(manifest_dir): # TODO: It seems like there's a bug in argparse that makes this argument order required # should try to work around that if is_port_8000_in_use(): pytest.skip("port 8000 already in use") if sys.platform == "darwin": fx_path = os.path.join(wpt.localpaths.repo_root, "_venv", "browsers", "nightly", "Firefox Nightly.app") else: fx_path = os.path.join(wpt.localpaths.repo_root, "_venv", "browsers", "nightly", "firefox") if os.path.exists(fx_path): shutil.rmtree(fx_path) with pytest.raises(SystemExit) as excinfo: wpt.main(argv=["run", "--no-pause", "--install-browser", "--yes", # The use of `--binary-args` is intentional: it # demonstrates that internally-managed command-line # arguments are properly merged with those specified by # the user. See # https://github.com/web-platform-tests/wpt/pull/13154 "--binary-arg=-headless", "--metadata", manifest_dir, "firefox", "/dom/nodes/Element-tagName.html"]) assert os.path.exists(fx_path) shutil.rmtree(fx_path) assert excinfo.value.code == 0
def test_run_verify_unstable(temp_test): """Unstable tests should be reported with a non-zero exit status. Stable tests should be reported with a zero exit status.""" if is_port_8000_in_use(): pytest.skip("port 8000 already in use") unstable_test = temp_test(""" test(function() { if (localStorage.getItem('wpt-unstable-test-flag')) { throw new Error(); } localStorage.setItem('wpt-unstable-test-flag', 'x'); }, 'my test'); """) with pytest.raises(SystemExit) as excinfo: wpt.main(argv=["run", "--yes", "--verify", "--binary-arg", "headless", "chrome", unstable_test]) assert excinfo.value.code != 0 stable_test = temp_test("test(function() {}, 'my test');") with pytest.raises(SystemExit) as excinfo: wpt.main(argv=["run", "--yes", "--verify", "--binary-arg", "headless", "chrome", stable_test]) assert excinfo.value.code == 0
def init_manifest(): with pytest.raises(SystemExit) as excinfo: wpt.main(argv=[ "manifest", "--no-download", "--path", get_persistent_manifest_path() ]) assert excinfo.value.code == 0
def update_manifest(): with pytest.raises(SystemExit) as excinfo: wpt.main(argv=[ "manifest", "--no-download", "--path", os.path.join(path, "MANIFEST.json") ]) assert excinfo.value.code == 0
def test_run_firefox(manifest_dir): # TODO: It seems like there's a bug in argparse that makes this argument order required # should try to work around that if is_port_8000_in_use(): pytest.skip("port 8000 already in use") os.environ["MOZ_HEADLESS"] = "1" try: if sys.platform == "darwin": fx_path = os.path.join(wpt.localpaths.repo_root, "_venv", "browsers", "Firefox Nightly.app") else: fx_path = os.path.join(wpt.localpaths.repo_root, "_venv", "browsers", "firefox") if os.path.exists(fx_path): shutil.rmtree(fx_path) with pytest.raises(SystemExit) as excinfo: wpt.main(argv=[ "run", "--no-pause", "--install-browser", "--yes", "--metadata", manifest_dir, "firefox", "/dom/nodes/Element-tagName.html" ]) assert os.path.exists(fx_path) shutil.rmtree(fx_path) assert excinfo.value.code == 0 finally: del os.environ["MOZ_HEADLESS"]
def test_list_tests_missing_manifest(manifest_dir): """The `--list-tests` option should not produce an error in the absence of a test manifest file.""" os.remove(os.path.join(manifest_dir, "MANIFEST.json")) with pytest.raises(SystemExit) as excinfo: wpt.main(argv=[ "run", # This test triggers the creation of a new manifest # file which is not necessary to ensure successful # process completion. Specifying the current directory # as the tests source via the --tests` option # drastically reduces the time to execute the test. "--tests", here, "--metadata", manifest_dir, "--list-tests", "--yes", "firefox", "/dom/nodes/Element-tagName.html" ]) assert excinfo.value.code == 0
def test_run_firefox(manifest_dir): # TODO: It seems like there's a bug in argparse that makes this argument order required # should try to work around that if is_port_8000_in_use(): pytest.skip("port 8000 already in use") if sys.platform == "darwin": fx_path = os.path.join(wpt.localpaths.repo_root, "_venv", "browsers", "nightly", "Firefox Nightly.app") else: fx_path = os.path.join(wpt.localpaths.repo_root, "_venv", "browsers", "nightly", "firefox") if os.path.exists(fx_path): shutil.rmtree(fx_path) with pytest.raises(SystemExit) as excinfo: wpt.main(argv=[ "run", "--no-pause", "--install-browser", "--yes", # The use of `--binary-args` is intentional: it # demonstrates that internally-managed command-line # arguments are properly merged with those specified by # the user. See # https://github.com/web-platform-tests/wpt/pull/13154 "--binary-arg=-headless", "--metadata", manifest_dir, "firefox", "/dom/nodes/Element-tagName.html" ]) assert os.path.exists(fx_path) shutil.rmtree(fx_path) assert excinfo.value.code == 0
def test_run_chrome(): with pytest.raises(SystemExit) as excinfo: wpt.main(argv=[ "run", "--yes", "--no-pause", "--binary-arg", "headless", "--metadata", "~/meta/", "chrome", "/dom/nodes/Element-tagName.html" ]) assert excinfo.value.code == 0
def init_manifest(): # See https://github.com/pypa/virtualenv/issues/1710 if sys.version_info[0] >= 3 and platform.system() == "Windows": pytest.xfail(reason="virtualenv activation fails in Windows for python3") with pytest.raises(SystemExit) as excinfo: wpt.main(argv=["manifest", "--no-download", "--path", get_persistent_manifest_path()]) assert excinfo.value.code == 0
def test_list_tests(manifest_dir): """The `--list-tests` option should not produce an error under normal conditions.""" with pytest.raises(SystemExit) as excinfo: wpt.main(argv=["run", "--metadata", manifest_dir, "--list-tests", "--yes", "chrome", "/dom/nodes/Element-tagName.html"]) assert excinfo.value.code == 0
def test_install_firefox(): fx_path = os.path.join(wpt.localpaths.repo_root, "_venv", "firefox") if os.path.exists(fx_path): shutil.rmtree(fx_path) with pytest.raises(SystemExit) as excinfo: wpt.main(argv=["install", "firefox", "browser"]) assert excinfo.value.code == 0 assert os.path.exists(fx_path) shutil.rmtree(fx_path)
def test_install_chromedriver(): chromedriver_path = os.path.join(wpt.localpaths.repo_root, "_venv", "bin", "chromedriver") if os.path.exists(chromedriver_path): os.unlink(chromedriver_path) with pytest.raises(SystemExit) as excinfo: wpt.main(argv=["install", "chrome", "webdriver"]) assert excinfo.value.code == 0 assert os.path.exists(chromedriver_path) os.unlink(chromedriver_path)
def test_run_chrome(manifest_dir): if is_port_8000_in_use(): pytest.skip("port 8000 already in use") with pytest.raises(SystemExit) as excinfo: wpt.main(argv=["run", "--yes", "--no-pause", "--binary-arg", "headless", "--metadata", manifest_dir, "chrome", "/dom/nodes/Element-tagName.html"]) assert excinfo.value.code == 0
def test_tests_affected(capsys, manifest_dir): # This doesn't really work properly for random commits because we test the files in # the current working directory for references to the changed files, not the ones at # that specific commit. But we can at least test it returns something sensible commit = "9047ac1d9f51b1e9faa4f9fad9c47d109609ab09" with pytest.raises(SystemExit) as excinfo: wpt.main(argv=["tests-affected", "--metadata", manifest_dir, "%s~..%s" % (commit, commit)]) assert excinfo.value.code == 0 out, err = capsys.readouterr() assert "html/browsers/offline/appcache/workers/appcache-worker.html" in out
def spawn_wpt(script_name, wpt_checkout, wpt_args): # Import the WPT python module. sys.path.insert(0, wpt_checkout) try: from tools.wpt import wpt except ImportError: _log.error("Failed to import wpt Python modules from the web-platform-tests directory") sys.exit(-1) wpt.main(script_name, wpt_args)
def test_tests_affected(capsys): # This doesn't really work properly for random commits because we test the files in # the current working directory for references to the changed files, not the ones at # that specific commit. But we can at least test it returns something sensible commit = "9047ac1d9f51b1e9faa4f9fad9c47d109609ab09" with pytest.raises(SystemExit) as excinfo: wpt.main(argv=["tests-affected", "--metadata", "~/meta/", "%s~..%s" % (commit, commit)]) assert excinfo.value.code == 0 out, err = capsys.readouterr() assert "html/browsers/offline/appcache/workers/appcache-worker.html" in out
def test_update(tmp_path, metadata_file): # This has to be a real test so it's in the manifest test_id = "/infrastructure/assumptions/cookie.html" subtest_name = "cookies work in default browse settings" test_path = os.path.join("infrastructure", "assumptions", "cookie.html") run_info_firefox, path_firefox = metadata_file(test_id, subtest_name, "firefox", subtest_status="FAIL", channel="nightly") run_info_chrome, path_chrome = metadata_file(test_id, subtest_name, "chrome", status="ERROR", subtest_status="NOTRUN", channel="dev") metadata_path = str(os.path.join(tmp_path, "metadata")) os.makedirs(metadata_path) wptreport_paths = [path_firefox, path_chrome] update_properties = {"properties": ["product"]} with open(os.path.join(metadata_path, "update_properties.json"), "w") as f: json.dump(update_properties, f) args = [ "update-expectations", "--manifest", os.path.join(repo_root, "MANIFEST.json"), "--metadata", metadata_path, "--log-mach-level", "debug" ] args += wptreport_paths with pytest.raises(SystemExit) as excinfo: wpt.main(argv=args) assert excinfo.value.code == 0 expectation_path = os.path.join(metadata_path, test_path + ".ini") assert os.path.exists(expectation_path) firefox_expected = manifestexpected.get_manifest(metadata_path, test_path, "/", run_info_firefox) # Default expected isn't stored with pytest.raises(KeyError): assert firefox_expected.get_test(test_id).get("expected") assert firefox_expected.get_test(test_id).get_subtest( subtest_name).expected == "FAIL" chrome_expected = manifestexpected.get_manifest(metadata_path, test_path, "/", run_info_chrome) assert chrome_expected.get_test(test_id).expected == "ERROR" assert chrome_expected.get_test(test_id).get_subtest( subtest_name).expected == "NOTRUN"
def test_list_tests(manifest_dir): """The `--list-tests` option should not produce an error under normal conditions.""" with pytest.raises(SystemExit) as excinfo: wpt.main(argv=["run", "--metadata", manifest_dir, "--list-tests", "--channel", "dev", "--yes", # Taskcluster machines do not have GPUs, so use software rendering via --enable-swiftshader. "--enable-swiftshader", "chrome", "/dom/nodes/Element-tagName.html"]) assert excinfo.value.code == 0
def test_tests_affected(capsys, manifest_dir): # This doesn't really work properly for random commits because we test the files in # the current working directory for references to the changed files, not the ones at # that specific commit. But we can at least test it returns something sensible. # The test will fail if the file we assert is renamed, so we choose a stable one. commit = "3a055e818218f548db240c316654f3cc1aeeb733" with pytest.raises(SystemExit) as excinfo: wpt.main(argv=["tests-affected", "--metadata", manifest_dir, "%s~..%s" % (commit, commit)]) assert excinfo.value.code == 0 out, err = capsys.readouterr() assert "infrastructure/reftest-wait.html" in out
def test_install_chromedriver(): if sys.platform == "win32": chromedriver_path = os.path.join(wpt.localpaths.repo_root, wpt.venv_dir(), "Scripts", "chromedriver.exe") else: chromedriver_path = os.path.join(wpt.localpaths.repo_root, wpt.venv_dir(), "bin", "chromedriver") if os.path.exists(chromedriver_path): os.unlink(chromedriver_path) with pytest.raises(SystemExit) as excinfo: wpt.main(argv=["install", "chrome", "webdriver"]) assert excinfo.value.code == 0 assert os.path.exists(chromedriver_path) os.unlink(chromedriver_path)
def test_install_firefox(): if sys.platform == "darwin": fx_path = os.path.join(wpt.localpaths.repo_root, "_venv", "browsers", "nightly", "Firefox Nightly.app") else: fx_path = os.path.join(wpt.localpaths.repo_root, "_venv", "browsers", "nightly", "firefox") if os.path.exists(fx_path): shutil.rmtree(fx_path) with pytest.raises(SystemExit) as excinfo: wpt.main(argv=["install", "firefox", "browser", "--channel=nightly"]) assert excinfo.value.code == 0 assert os.path.exists(fx_path) shutil.rmtree(fx_path)
def test_tests_affected_idlharness(capsys, manifest_dir): commit = "47cea8c38b88c0ddd3854e4edec0c5b6f2697e62" with pytest.raises(SystemExit) as excinfo: wpt.main(argv=["tests-affected", "--metadata", manifest_dir, "%s~..%s" % (commit, commit)]) assert excinfo.value.code == 0 out, err = capsys.readouterr() assert ("mst-content-hint/idlharness.window.js\n" + "webrtc-encoded-transform/idlharness.https.window.js\n" + "webrtc-identity/idlharness.https.window.js\n" + "webrtc-stats/idlharness.window.js\n" + "webrtc-stats/supported-stats.html\n" + "webrtc/idlharness.https.window.js\n") == out
def test_tests_affected_null(capsys, manifest_dir): # This doesn't really work properly for random commits because we test the files in # the current working directory for references to the changed files, not the ones at # that specific commit. But we can at least test it returns something sensible. # The test will fail if the file we assert is renamed, so we choose a stable one. commit = "2614e3316f1d3d1a744ed3af088d19516552a5de" with pytest.raises(SystemExit) as excinfo: wpt.main(argv=["tests-affected", "--null", "--metadata", manifest_dir, "%s~..%s" % (commit, commit)]) assert excinfo.value.code == 0 out, err = capsys.readouterr() tests = out.split("\0") assert "dom/idlharness.any.js" in tests assert "xhr/idlharness.any.js" in tests
def test_tests_affected_null(capsys, manifest_dir): # This doesn't really work properly for random commits because we test the files in # the current working directory for references to the changed files, not the ones at # that specific commit. But we can at least test it returns something sensible. # The test will fail if the file we assert is renamed, so we choose a stable one. commit = "9bf1daa3d8b4425f2354c3ca92c4cf0398d329dd" with pytest.raises(SystemExit) as excinfo: wpt.main(argv=["tests-affected", "--null", "--metadata", manifest_dir, "%s~..%s" % (commit, commit)]) assert excinfo.value.code == 0 out, err = capsys.readouterr() tests = out.split("\0") assert "dom/interfaces.html" in tests assert "html/dom/interfaces.https.html" in tests
def test_files_changed(capsys): commit = "9047ac1d9f51b1e9faa4f9fad9c47d109609ab09" with pytest.raises(SystemExit) as excinfo: wpt.main(argv=["files-changed", "%s~..%s" % (commit, commit)]) assert excinfo.value.code == 0 out, err = capsys.readouterr() assert out == """html/browsers/offline/appcache/workers/appcache-worker.html html/browsers/offline/appcache/workers/resources/appcache-dedicated-worker-not-in-cache.js html/browsers/offline/appcache/workers/resources/appcache-shared-worker-not-in-cache.js html/browsers/offline/appcache/workers/resources/appcache-worker-data.py html/browsers/offline/appcache/workers/resources/appcache-worker-import.py html/browsers/offline/appcache/workers/resources/appcache-worker.manifest html/browsers/offline/appcache/workers/resources/appcache-worker.py """ assert err == ""
def test_files_changed_null(capsys): commit = "9047ac1d9f51b1e9faa4f9fad9c47d109609ab09" with pytest.raises(SystemExit) as excinfo: wpt.main(argv=["files-changed", "--null", "%s~..%s" % (commit, commit)]) assert excinfo.value.code == 0 out, err = capsys.readouterr() expected = "\0".join(["html/browsers/offline/appcache/workers/appcache-worker.html", "html/browsers/offline/appcache/workers/resources/appcache-dedicated-worker-not-in-cache.js", "html/browsers/offline/appcache/workers/resources/appcache-shared-worker-not-in-cache.js", "html/browsers/offline/appcache/workers/resources/appcache-worker-data.py", "html/browsers/offline/appcache/workers/resources/appcache-worker-import.py", "html/browsers/offline/appcache/workers/resources/appcache-worker.manifest", "html/browsers/offline/appcache/workers/resources/appcache-worker.py", ""]).replace("/", os.path.sep) assert out == expected assert err == ""
def test_run_zero_tests(): """A test execution describing zero tests should be reported as an error even in the presence of the `--no-fail-on-unexpected` option.""" if is_port_8000_in_use(): pytest.skip("port 8000 already in use") with pytest.raises(SystemExit) as excinfo: wpt.main(argv=["run", "--yes", "--no-pause", "--binary-arg", "headless", "chrome", "/non-existent-dir/non-existent-file.html"]) assert excinfo.value.code != 0 with pytest.raises(SystemExit) as excinfo: wpt.main(argv=["run", "--yes", "--no-pause", "--binary-arg", "headless", "--no-fail-on-unexpected", "chrome", "/non-existent-dir/non-existent-file.html"]) assert excinfo.value.code != 0
def test_run_firefox(): # TODO: It seems like there's a bug in argparse that makes this argument order required # should try to work around that os.environ["MOZ_HEADLESS"] = "1" try: fx_path = os.path.join(wpt.localpaths.repo_root, "_venv", "firefox") if os.path.exists(fx_path): shutil.rmtree(fx_path) with pytest.raises(SystemExit) as excinfo: wpt.main(argv=["run", "--no-pause", "--install-browser", "--yes", "--metadata", "~/meta/", "firefox", "/dom/nodes/Element-tagName.html"]) assert os.path.exists(fx_path) shutil.rmtree(fx_path) assert excinfo.value.code == 0 finally: del os.environ["MOZ_HEADLESS"]
def test_run_failing_test(): """Failing tests should be reported with a non-zero exit status unless the `--no-fail-on-unexpected` option has been specified.""" if is_port_8000_in_use(): pytest.skip("port 8000 already in use") failing_test = "/infrastructure/expected-fail/failing-test.html" assert os.path.isfile("../../%s" % failing_test) with pytest.raises(SystemExit) as excinfo: wpt.main(argv=["run", "--yes", "--no-pause", "--binary-arg", "headless", "chrome", failing_test]) assert excinfo.value.code != 0 with pytest.raises(SystemExit) as excinfo: wpt.main(argv=["run", "--yes", "--no-pause", "--binary-arg", "headless", "--no-fail-on-unexpected", "chrome", failing_test]) assert excinfo.value.code == 0
def test_list_tests_missing_manifest(manifest_dir): """The `--list-tests` option should not produce an error in the absence of a test manifest file.""" os.remove(os.path.join(manifest_dir, "MANIFEST.json")) with pytest.raises(SystemExit) as excinfo: wpt.main(argv=["run", # This test triggers the creation of a new manifest # file which is not necessary to ensure successful # process completion. Specifying the current directory # as the tests source via the --tests` option # drastically reduces the time to execute the test. "--tests", here, "--metadata", manifest_dir, "--list-tests", "--yes", "firefox", "/dom/nodes/Element-tagName.html"]) assert excinfo.value.code == 0
def test_run_firefox(manifest_dir): # TODO: It seems like there's a bug in argparse that makes this argument order required # should try to work around that if is_port_8000_in_use(): pytest.skip("port 8000 already in use") os.environ["MOZ_HEADLESS"] = "1" try: if sys.platform == "darwin": fx_path = os.path.join(wpt.localpaths.repo_root, "_venv", "browsers", "Firefox Nightly.app") else: fx_path = os.path.join(wpt.localpaths.repo_root, "_venv", "browsers", "firefox") if os.path.exists(fx_path): shutil.rmtree(fx_path) with pytest.raises(SystemExit) as excinfo: wpt.main(argv=["run", "--no-pause", "--install-browser", "--yes", "--metadata", manifest_dir, "firefox", "/dom/nodes/Element-tagName.html"]) assert os.path.exists(fx_path) shutil.rmtree(fx_path) assert excinfo.value.code == 0 finally: del os.environ["MOZ_HEADLESS"]
def test_run_chrome(): with pytest.raises(SystemExit) as excinfo: wpt.main(argv=["run", "--yes", "--no-pause", "--binary-arg", "headless", "--metadata", "~/meta/", "chrome", "/dom/nodes/Element-tagName.html"]) assert excinfo.value.code == 0
def test_help(): # TODO: It seems like there's a bug in argparse that makes this argument order required # should try to work around that with pytest.raises(SystemExit) as excinfo: wpt.main(argv=["--help"]) assert excinfo.value.code == 0
def test_missing(): with pytest.raises(SystemExit): wpt.main(argv=["#missing-command"])
def update_manifest(): with pytest.raises(SystemExit) as excinfo: wpt.main(argv=["manifest", "--no-download", "--path", os.path.join(path, "MANIFEST.json")]) assert excinfo.value.code == 0
def init_manifest(): with pytest.raises(SystemExit) as excinfo: wpt.main(argv=["manifest", "--no-download", "--path", get_persistent_manifest_path()]) assert excinfo.value.code == 0