def test_use_subtest_results_flag_with_baseline_and_timeout(self):
"""If a test has both baseline and a non-default-pass expectation, do not
derive expectation from the baseline"""
# Create a baseline with a failing subtest, and a TIMEOUT expectation
test_name = "external/wpt/test.html"
baseline_filename = self.port.expected_filename(test_name, '.txt')
self.host.filesystem.write_text_file(
baseline_filename,
"This is a test\nFAIL some subtest\nPASS another subtest\n")
expectations = _make_expectation(self.port, test_name, "TIMEOUT")
metadata_builder = WPTMetadataBuilder(expectations, self.port)
metadata_builder.use_subtest_results = True
test_and_status_dict = metadata_builder.get_tests_needing_metadata()
self.assertEqual(1, len(test_and_status_dict))
self.assertTrue(test_name in test_and_status_dict)
self.assertEqual(TEST_TIMEOUT, test_and_status_dict[test_name])
filename, contents = metadata_builder.get_metadata_filename_and_contents(
test_name, test_and_status_dict[test_name])
self.assertEqual("test.html.ini", filename)
self.assertEqual("[test.html]\n expected: [TIMEOUT]\n",
contents)
def main(args):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--additional-expectations",
action="append",
help="Paths to additional expectations files for WPT.")
parser.add_argument("--android-product",
default=None,
choices=PRODUCTS,
help="Android product argument for wpt runner.")
parser.add_argument(
'--ignore-default-expectations',
action='store_true',
help='Do not use the default set of test expectations files.'
' i.e TestExpectations, NeverFixTests, etc...')
known_args, rest_args = parser.parse_known_args(args)
options = optparse.Values(vars(known_args))
host = Host()
if known_args.android_product:
port = AndroidPort(host,
product=known_args.android_product,
options=options)
else:
port = host.port_factory.get(options=options)
expectations = TestExpectations(port)
metadata_builder = WPTMetadataBuilder(expectations, port)
sys.exit(metadata_builder.run(rest_args))
def test_without_use_subtest_results_flag_with_baseline(self):
"""A test has a failing baseline."""
test_name = "external/wpt/test.html"
# Create a baseline with a failing subtest, and a TIMEOUT expectation
baseline_filename = self.port.expected_filename(test_name, '.txt')
self.host.filesystem.write_text_file(
baseline_filename,
"This is a test\nFAIL some subtest\nPASS another subtest\n")
expectations = _make_expectation(self.port, test_name, "TIMEOUT")
metadata_builder = WPTMetadataBuilder(expectations, self.port)
test_and_status_dict = metadata_builder.get_tests_needing_metadata()
self.assertEqual(1, len(test_and_status_dict))
self.assertTrue(test_name in test_and_status_dict)
self.assertEqual(TEST_TIMEOUT,
test_and_status_dict[test_name])
filename, contents = metadata_builder.get_metadata_filename_and_contents(
test_name, test_and_status_dict[test_name])
self.assertEqual("test.html.ini", filename)
self.assertEqual(("[test.html]\n blink_expect_any_subtest_status: "
"True # wpt_metadata_builder.py\n "
"expected: [TIMEOUT]\n"), contents)
def test_use_subtest_results_flag_with_baseline_and_default_pass(self):
"""A test may have a failing baseline because there are subtest failures.
When the wptrunner see's the failing subtest it will return failure
for the test since we are not setting expectations for the subtest in
the metadata. However the expected result will be set to FAIL in the
JSON results and the CI will stay green."""
# Create a baseline with a failing subtest, and a default pass expectation
test_name = "external/wpt/test.html"
baseline_filename = self.port.expected_filename(test_name, '.txt')
self.host.filesystem.write_text_file(
baseline_filename,
"This is a test\nFAIL some subtest\nPASS another subtest\n")
expectations = _make_expectation(self.port, test_name,
"Pass", is_default_pass=True)
metadata_builder = WPTMetadataBuilder(expectations, self.port)
metadata_builder.use_subtest_results = True
test_and_status_dict = metadata_builder.get_tests_needing_metadata()
self.assertEqual(1, len(test_and_status_dict))
self.assertTrue(test_name in test_and_status_dict)
self.assertEqual(SUBTEST_FAIL, test_and_status_dict[test_name])
filename, contents = metadata_builder.get_metadata_filename_and_contents(
test_name, test_and_status_dict[test_name])
self.assertEqual("test.html.ini", filename)
self.assertEqual("[test.html]\n expected: [FAIL, ERROR]\n",
contents)
def test_overriding_skip_expectation_with_annotation(self):
"""A skipped tests is overridden by an annotation.
In this case, the SKIP status gets ignored because the status line in
the override file has the 'wpt_use_checked_in_metadata' annotation. This
forces us to prioritize the checked-in metadata over whatever is in
any of the expectation files.
"""
test_name = "external/wpt/test.html"
expectation_dict = OrderedDict()
_append_to_expectation_dict(expectation_dict, "NeverFixTests",
test_name, "SKIP")
_append_to_expectation_dict(
expectation_dict,
"WPTOverrideExpectations",
test_name,
"PASS",
trailing_comments=" # wpt_use_checked_in_metadata")
expectations = _make_expectation_with_dict(self.port, expectation_dict)
metadata_builder = WPTMetadataBuilder(expectations, self.port)
test_and_status_dict = metadata_builder.get_tests_needing_metadata()
self.assertEqual(1, len(test_and_status_dict))
self.assertTrue(test_name in test_and_status_dict)
# Skip statuses always live alone. The override status is ignored
# because it does not have an annotation to use checked-in metadata.
self.assertEqual(USE_CHECKED_IN_METADATA,
test_and_status_dict[test_name])
def test_wpt_test_not_skipped(self):
"""A WPT test that is not skipped should not get a SKIP metadata."""
test_name = "external/wpt/test.html"
expectations = _make_expectation(self.port, test_name, "TIMEOUT")
metadata_builder = WPTMetadataBuilder(expectations, self.port)
test_names = metadata_builder.get_test_names_to_skip()
self.assertFalse(test_names)
def test_skipped_test(self):
"""A skipped WPT test should get a test-specific metadata file."""
test_name = "external/wpt/test.html"
expectations = _make_expectation(self.port, test_name, "SKIP")
metadata_builder = WPTMetadataBuilder(expectations, self.port)
filename, contents = metadata_builder.get_metadata_filename_and_contents(test_name, 'SKIP')
self.assertEqual("test.html.ini", filename)
self.assertEqual("[test.html]\n disabled: wpt_metadata_builder.py\n", contents)
def test_non_wpt_test(self):
"""A non-WPT test should not get any metadata."""
test_name = "some/other/test.html"
expectations = _make_expectation(self.port, test_name, "SKIP")
metadata_builder = WPTMetadataBuilder(expectations, self.port)
filename, contents = metadata_builder.get_metadata_filename_and_contents(test_name, 'SKIP')
self.assertIsNone(filename)
self.assertIsNone(contents)
def test_wpt_test_without_manifest_entry(self):
"""A WPT test that is not in the manifest should not get a baseline."""
test_name = "external/wpt/test-not-in-manifest.html"
expectations = _make_expectation(self.port, test_name, "SKIP")
metadata_builder = WPTMetadataBuilder(expectations, self.port)
filename, contents = metadata_builder.get_metadata_filename_and_contents(test_name, 'SKIP')
self.assertIsNone(filename)
self.assertIsNone(contents)
def test_skipped_directory(self):
"""A skipped WPT directory should get a dir-wide metadata file."""
test_name = "external/wpt/test_dir/"
expectations = _make_expectation(self.port, test_name, "SKIP")
metadata_builder = WPTMetadataBuilder(expectations, self.port)
filename, contents = metadata_builder.get_metadata_filename_and_contents(test_name, 'SKIP')
self.assertEqual(os.path.join("test_dir", "__dir__.ini"), filename)
self.assertEqual("disabled: wpt_metadata_builder.py\n", contents)
def test_metadata_for_wpt_test_with_harness_error_baseline(self):
"""A WPT test with a baseline file containing a harness error gets metadata."""
test_name = "external/wpt/dir/zzzz.html"
expectations = TestExpectations(self.port)
metadata_builder = WPTMetadataBuilder(expectations, self.port)
filename, contents = metadata_builder.get_metadata_filename_and_contents(
test_name, 'FAIL', HARNESS_ERROR)
self.assertEqual(os.path.join("dir", "zzzz.html.ini"), filename)
self.assertEqual("[zzzz.html]\n expected: ERROR\n", contents)
def test_skipped_test_with_variants(self):
"""A skipped WPT tests with variants should get a test-specific metadata file."""
test_name = "external/wpt/variant.html?foo=bar"
expectations = _make_expectation(self.port, test_name, "SKIP")
metadata_builder = WPTMetadataBuilder(expectations, self.port)
filename, contents = metadata_builder.get_metadata_filename_and_contents(test_name, 'SKIP')
# The metadata file name should not include variants
self.assertEqual("variant.html.ini", filename)
# ..but the contents of the file should include variants in the test name
self.assertEqual("[variant.html?foo=bar]\n disabled: wpt_metadata_builder.py\n", contents)
def test_metadata_for_wpt_test_with_fail_baseline(self):
"""A WPT test with a baseline file containing failures gets metadata."""
test_name = "external/wpt/dir/zzzz.html"
expectations = TestExpectations(self.port)
metadata_builder = WPTMetadataBuilder(expectations, self.port)
filename, contents = metadata_builder.get_metadata_filename_and_contents(
test_name, SUBTEST_FAIL)
self.assertEqual(os.path.join("dir", "zzzz.html.ini"), filename)
self.assertEqual(
"[zzzz.html]\n blink_expect_any_subtest_status: True # wpt_metadata_builder.py\n",
contents)
def test_metadata_for_use_checked_in_metadata_annotation(self):
"""A WPT test annotated to use checked-in metadata."""
test_name = "external/wpt/test.html"
expectations = TestExpectations(self.port)
metadata_builder = WPTMetadataBuilder(expectations, self.port)
filename, contents = metadata_builder.get_metadata_filename_and_contents(
test_name, USE_CHECKED_IN_METADATA)
# If the test is using the metadata that is checked-in, then there is no
# work to be done by the metadata builder.
self.assertIsNone(filename)
self.assertIsNone(contents)
def test_metadata_for_flaky_test(self):
"""A WPT test that is flaky has multiple statuses in metadata."""
test_name = "external/wpt/test.html"
expectations = _make_expectation(self.port, test_name, "PASS FAILURE")
metadata_builder = WPTMetadataBuilder(expectations, self.port)
filename, contents = metadata_builder.get_metadata_filename_and_contents(
test_name, TEST_PASS | TEST_FAIL)
self.assertEqual("test.html.ini", filename)
# The PASS and FAIL expectations fan out to also include OK and ERROR
# to support reftest/testharness test differences.
self.assertEqual("[test.html]\n expected: [PASS, OK, FAIL, ERROR]\n",
contents)
def main(args):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--additional-expectations",
action="append",
help="Paths to additional expectations files for WPT.")
known_args, rest_args = parser.parse_known_args(args)
host = Host()
port = host.port_factory.get(options=optparse.Values(vars(known_args)))
expectations = TestExpectations(port)
metadata_builder = WPTMetadataBuilder(expectations)
sys.exit(metadata_builder.run(rest_args))
def test_precondition_failed(self):
"""A WPT that fails a precondition."""
test_name = "external/wpt/test.html"
expectations = TestExpectations(self.port)
metadata_builder = WPTMetadataBuilder(expectations, self.port)
filename, contents = metadata_builder.get_metadata_filename_and_contents(
test_name, TEST_PRECONDITION_FAILED)
self.assertEqual("test.html.ini", filename)
# The PASS and FAIL expectations fan out to also include OK and ERROR
# to support reftest/testharness test differences.
self.assertEqual("[test.html]\n expected: [PRECONDITION_FAILED]\n",
contents)
def test_parse_baseline_all_pass(self):
"""A WPT test with an all-pass baseline doesn't get metadata."""
# Here we use a test_name that is actually in the test manifest
test_name = "external/wpt/dir/zzzz.html"
# Manually initialize the baseline file and its contents
baseline_filename = self.port.expected_filename(test_name, '.txt')
self.host.filesystem.write_text_file(
baseline_filename,
"This is a test\nPASS some subtest\nPASS another subtest\n")
expectations = TestExpectations(self.port)
metadata_builder = WPTMetadataBuilder(expectations, self.port)
test_and_status_dict = metadata_builder.get_tests_needing_metadata()
self.assertFalse(test_and_status_dict)
def test_wpt_test_not_skipped(self):
"""A WPT test that is not skipped should not get a SKIP metadata."""
test_name = "external/wpt/test.html"
expectations = _make_expectation(self.port, test_name, "TIMEOUT")
metadata_builder = WPTMetadataBuilder(expectations, self.port)
test_names = metadata_builder.get_tests_needing_metadata()
# The test will appear in the result but won't have a SKIP status
found = False
for name_item, status_item in test_names.items():
if name_item == test_name:
found = True
self.assertNotEqual(SKIP_TEST, status_item)
self.assertTrue(found)
def test_wpt_test_with_passing_baseline(self):
"""A WPT test with an all-pass baseline doesn't get metadata."""
# Here we use a test_name that is actually in the test manifest
test_name = "external/wpt/test/zzzz.html"
# Manually initialize the baseline file and its contents
baseline_filename = "external/wpt/test/zzzz-expected.txt"
self.host.filesystem.write_text_file(
os.path.join(self.port.web_tests_dir(), baseline_filename),
"This is a test\nPASS some subtest\nPASS another subtest\n")
expectations = TestExpectations(self.port)
metadata_builder = WPTMetadataBuilder(expectations, self.port)
test_names = metadata_builder.get_test_names_to_fail()
self.assertFalse(test_names)
def test_wpt_test_with_baseline(self):
"""A WPT test with a baseline file containing failures gets metadata."""
test_name = "external/wpt/dir/zzzz.html"
# Manually initialize the baseline file and its contents
baseline_filename = "external/wpt/dir/zzzz-expected.txt"
self.host.filesystem.write_text_file(
os.path.join(self.port.web_tests_dir(), baseline_filename),
"This is a test\nPASS some subtest\nFAIL some failing subtest\n")
expectations = TestExpectations(self.port)
metadata_builder = WPTMetadataBuilder(expectations, self.port)
filename, contents = metadata_builder.get_metadata_filename_and_contents(test_name, 'FAIL')
self.assertEqual(os.path.join("dir", "zzzz.html.ini"), filename)
self.assertEqual("[zzzz.html]\n expected: FAIL # wpt_metadata_builder.py\n", contents)
def test_parse_baseline_harness_error(self):
"""Test parsing a baseline with a harness error."""
# Here we use a test_name that is actually in the test manifest
test_name = "external/wpt/dir/zzzz.html"
# Manually initialize the baseline file and its contents
baseline_filename = self.port.expected_filename(test_name, '.txt')
self.host.filesystem.write_text_file(
baseline_filename, "This is a test\nHarness Error. some stuff\n")
expectations = TestExpectations(self.port)
metadata_builder = WPTMetadataBuilder(expectations, self.port)
test_and_status_dict = metadata_builder.get_tests_needing_metadata()
self.assertEqual(1, len(test_and_status_dict))
self.assertTrue(test_name in test_and_status_dict)
self.assertEqual(HARNESS_ERROR, test_and_status_dict[test_name])
def test_use_subtest_results_flag(self):
"""Test that the --use-subtest-results flag updates metadata correctly.
The --use-subtest-results flag should result in the
blink_expect_any_subtest_status tag not being applied to metadata for
any tests."""
test_name = "external/wpt/test.html"
expectations = _make_expectation(self.port, test_name, "FAILURE")
metadata_builder = WPTMetadataBuilder(expectations, self.port)
metadata_builder.use_subtest_results = True
filename, contents = metadata_builder.get_metadata_filename_and_contents(
test_name, TEST_FAIL)
self.assertEqual("test.html.ini", filename)
self.assertEqual("[test.html]\n expected: [FAIL, ERROR]\n", contents)
def test_inline_test_name_from_test_name(self):
"""Check that we get the correct inline test name in various cases."""
expectations = TestExpectations(self.port)
mb = WPTMetadataBuilder(expectations, self.port)
self.assertEqual(
"test.html",
mb._metadata_inline_test_name_from_test_name("test.html"))
self.assertEqual(
"test.html",
mb._metadata_inline_test_name_from_test_name("dir/test.html"))
self.assertEqual(
"test.html?variant=abc",
mb._metadata_inline_test_name_from_test_name(
"dir/test.html?variant=abc"))
self.assertEqual(
"test.html?variant=abc/def",
mb._metadata_inline_test_name_from_test_name(
"dir/test.html?variant=abc/def"))
self.assertEqual(
"test.worker.html",
mb._metadata_inline_test_name_from_test_name("test.worker.html"))
self.assertEqual(
"test.worker.html?variant=abc",
mb._metadata_inline_test_name_from_test_name(
"dir/test.worker.html?variant=abc"))
def test_baseline_and_expectation(self):
"""A test has a failing baseline and a timeout expectation."""
test_name = "external/wpt/test.html"
# Create a baseline with a failing subtest, and a TIMEOUT expectation
baseline_filename = self.port.expected_filename(test_name, '.txt')
self.host.filesystem.write_text_file(
baseline_filename,
"This is a test\nFAIL some subtest\nPASS another subtest\n")
expectations = _make_expectation(self.port, test_name, "TIMEOUT")
metadata_builder = WPTMetadataBuilder(expectations, self.port)
test_and_status_dict = metadata_builder.get_tests_needing_metadata()
self.assertEqual(1, len(test_and_status_dict))
self.assertTrue(test_name in test_and_status_dict)
self.assertEqual(TEST_TIMEOUT, test_and_status_dict[test_name])
def test_parse_precondition_failure_annotation(self):
"""Check that we parse the wpt_precondition_failed annotation correctly."""
test_name = "external/wpt/test.html"
expectations = _make_expectation(
self.port,
test_name,
"PASS",
trailing_comments=" # wpt_precondition_failed")
metadata_builder = WPTMetadataBuilder(expectations, self.port)
test_and_status_dict = metadata_builder.get_tests_needing_metadata()
self.assertEqual(1, len(test_and_status_dict))
self.assertTrue(test_name in test_and_status_dict)
self.assertEqual(TEST_PASS | TEST_PRECONDITION_FAILED,
test_and_status_dict[test_name])
def test_metadata_for_wpt_multiglobal_test_with_baseline(self):
"""A WPT test with a baseline file containing failures gets metadata."""
test_name = "external/wpt/dir/multiglob.https.any.window.html"
expectations = TestExpectations(self.port)
metadata_builder = WPTMetadataBuilder(expectations, self.port)
filename, contents = metadata_builder.get_metadata_filename_and_contents(
test_name, SUBTEST_FAIL)
# The metadata filename matches the test *filename*, not the test name,
# which in this case is the js file from the manifest.
self.assertEqual(os.path.join("dir", "multiglob.https.any.js.ini"),
filename)
# The metadata contents contains the *test name*
self.assertEqual(
"[multiglob.https.any.window.html]\n blink_expect_any_subtest_status: True # wpt_metadata_builder.py\n",
contents)
def test_parse_baseline_subtest_timeout(self):
"""Test parsing a baseline with a timeout subtest."""
# Here we use a test_name that is actually in the test manifest
test_name = "external/wpt/dir/zzzz.html"
# Manually initialize the baseline file and its contents
baseline_filename = self.port.expected_filename(test_name, '.txt')
self.host.filesystem.write_text_file(
baseline_filename,
"This is a test\nTIMEOUT some subtest\nPASS another subtest\n")
expectations = TestExpectations(self.port)
metadata_builder = WPTMetadataBuilder(expectations, self.port)
test_and_status_dict = metadata_builder.get_tests_needing_metadata()
self.assertEqual(1, len(test_and_status_dict))
self.assertTrue(test_name in test_and_status_dict)
self.assertEqual(SUBTEST_FAIL, test_and_status_dict[test_name])
def test_parse_baseline_subtest_notrun(self):
"""Test parsing a baseline with a notrun subtest."""
# Here we use a test_name that is actually in the test manifest
test_name = "external/wpt/dir/zzzz.html"
# Manually initialize the baseline file and its contents
baseline_filename = self.port.expected_filename(test_name, '.txt')
self.host.filesystem.write_text_file(
baseline_filename,
"This is a test\nPASS some subtest\nNOTRUN another subtest\n")
expectations = TestExpectations(self.port)
metadata_builder = WPTMetadataBuilder(expectations, self.port)
test_and_status_list = metadata_builder.get_tests_with_baselines()
self.assertEqual(1, len(test_and_status_list))
test_entry = test_and_status_list[0]
self.assertEqual(test_name, test_entry[0])
self.assertEqual(SUBTEST_FAIL, test_entry[1])
def test_wpt_multiglobal_test_with_baseline(self):
"""A WPT test with a baseline file containing failures gets metadata."""
test_name = "external/wpt/dir/multiglob.https.any.window.html"
# Manually initialize the baseline file and its contents
baseline_filename = "external/wpt/dir/multiglob.https.any.window-expected.txt"
self.host.filesystem.write_text_file(
os.path.join(self.port.web_tests_dir(), baseline_filename),
"This is a test\nPASS some subtest\nFAIL some failing subtest\n")
expectations = TestExpectations(self.port)
metadata_builder = WPTMetadataBuilder(expectations, self.port)
filename, contents = metadata_builder.get_metadata_filename_and_contents(test_name, 'FAIL')
# The metadata filename matches the test *filename*, not the test name,
# which in this case is the js file from the manifest.
self.assertEqual(os.path.join("dir", "multiglob.https.any.js.ini"), filename)
# The metadata contents contains the *test name*
self.assertEqual("[multiglob.https.any.window.html]\n expected: FAIL # wpt_metadata_builder.py\n", contents)
