def test_without_use_subtest_results_flag_with_baseline(self):
"""A test has a failing baseline."""
test_name = "external/wpt/test.html"
# Create a baseline with a failing subtest, and a TIMEOUT expectation
baseline_filename = self.port.expected_filename(test_name, '.txt')
self.host.filesystem.write_text_file(
baseline_filename,
"This is a test\nFAIL some subtest\nPASS another subtest\n")
expectations = _make_expectation(self.port, test_name, "TIMEOUT")
metadata_builder = WPTMetadataBuilder(expectations, self.port)
test_and_status_dict = metadata_builder.get_tests_needing_metadata()
self.assertEqual(1, len(test_and_status_dict))
self.assertTrue(test_name in test_and_status_dict)
self.assertEqual(TEST_TIMEOUT,
test_and_status_dict[test_name])
filename, contents = metadata_builder.get_metadata_filename_and_contents(
test_name, test_and_status_dict[test_name])
self.assertEqual("test.html.ini", filename)
self.assertEqual(("[test.html]\n blink_expect_any_subtest_status: "
"True # wpt_metadata_builder.py\n "
"expected: [TIMEOUT]\n"), contents)
def test_use_subtest_results_flag_with_baseline_and_default_pass(self):
"""A test may have a failing baseline because there are subtest failures.
When the wptrunner see's the failing subtest it will return failure
for the test since we are not setting expectations for the subtest in
the metadata. However the expected result will be set to FAIL in the
JSON results and the CI will stay green."""
# Create a baseline with a failing subtest, and a default pass expectation
test_name = "external/wpt/test.html"
baseline_filename = self.port.expected_filename(test_name, '.txt')
self.host.filesystem.write_text_file(
baseline_filename,
"This is a test\nFAIL some subtest\nPASS another subtest\n")
expectations = _make_expectation(self.port, test_name,
"Pass", is_default_pass=True)
metadata_builder = WPTMetadataBuilder(expectations, self.port)
metadata_builder.use_subtest_results = True
test_and_status_dict = metadata_builder.get_tests_needing_metadata()
self.assertEqual(1, len(test_and_status_dict))
self.assertTrue(test_name in test_and_status_dict)
self.assertEqual(SUBTEST_FAIL, test_and_status_dict[test_name])
filename, contents = metadata_builder.get_metadata_filename_and_contents(
test_name, test_and_status_dict[test_name])
self.assertEqual("test.html.ini", filename)
self.assertEqual("[test.html]\n expected: [FAIL, ERROR]\n",
contents)
def test_use_subtest_results_flag_with_baseline_and_timeout(self):
"""If a test has both baseline and a non-default-pass expectation, do not
derive expectation from the baseline"""
# Create a baseline with a failing subtest, and a TIMEOUT expectation
test_name = "external/wpt/test.html"
baseline_filename = self.port.expected_filename(test_name, '.txt')
self.host.filesystem.write_text_file(
baseline_filename,
"This is a test\nFAIL some subtest\nPASS another subtest\n")
expectations = _make_expectation(self.port, test_name, "TIMEOUT")
metadata_builder = WPTMetadataBuilder(expectations, self.port)
metadata_builder.use_subtest_results = True
test_and_status_dict = metadata_builder.get_tests_needing_metadata()
self.assertEqual(1, len(test_and_status_dict))
self.assertTrue(test_name in test_and_status_dict)
self.assertEqual(TEST_TIMEOUT, test_and_status_dict[test_name])
filename, contents = metadata_builder.get_metadata_filename_and_contents(
test_name, test_and_status_dict[test_name])
self.assertEqual("test.html.ini", filename)
self.assertEqual("[test.html]\n expected: [TIMEOUT]\n",
contents)
def test_non_wpt_test(self):
"""A non-WPT test should not get any metadata."""
test_name = "some/other/test.html"
expectations = _make_expectation(self.port, test_name, "SKIP")
metadata_builder = WPTMetadataBuilder(expectations, self.port)
filename, contents = metadata_builder.get_metadata_filename_and_contents(test_name, 'SKIP')
self.assertIsNone(filename)
self.assertIsNone(contents)
def test_skipped_directory(self):
"""A skipped WPT directory should get a dir-wide metadata file."""
test_name = "external/wpt/test_dir/"
expectations = _make_expectation(self.port, test_name, "SKIP")
metadata_builder = WPTMetadataBuilder(expectations, self.port)
filename, contents = metadata_builder.get_metadata_filename_and_contents(test_name, 'SKIP')
self.assertEqual(os.path.join("test_dir", "__dir__.ini"), filename)
self.assertEqual("disabled: wpt_metadata_builder.py\n", contents)
def test_skipped_test(self):
"""A skipped WPT test should get a test-specific metadata file."""
test_name = "external/wpt/test.html"
expectations = _make_expectation(self.port, test_name, "SKIP")
metadata_builder = WPTMetadataBuilder(expectations, self.port)
filename, contents = metadata_builder.get_metadata_filename_and_contents(test_name, 'SKIP')
self.assertEqual("test.html.ini", filename)
self.assertEqual("[test.html]\n disabled: wpt_metadata_builder.py\n", contents)
def test_wpt_test_without_manifest_entry(self):
"""A WPT test that is not in the manifest should not get a baseline."""
test_name = "external/wpt/test-not-in-manifest.html"
expectations = _make_expectation(self.port, test_name, "SKIP")
metadata_builder = WPTMetadataBuilder(expectations, self.port)
filename, contents = metadata_builder.get_metadata_filename_and_contents(test_name, 'SKIP')
self.assertIsNone(filename)
self.assertIsNone(contents)
def test_metadata_for_wpt_test_with_harness_error_baseline(self):
"""A WPT test with a baseline file containing a harness error gets metadata."""
test_name = "external/wpt/dir/zzzz.html"
expectations = TestExpectations(self.port)
metadata_builder = WPTMetadataBuilder(expectations, self.port)
filename, contents = metadata_builder.get_metadata_filename_and_contents(
test_name, 'FAIL', HARNESS_ERROR)
self.assertEqual(os.path.join("dir", "zzzz.html.ini"), filename)
self.assertEqual("[zzzz.html]\n expected: ERROR\n", contents)
def test_skipped_test_with_variants(self):
"""A skipped WPT tests with variants should get a test-specific metadata file."""
test_name = "external/wpt/variant.html?foo=bar"
expectations = _make_expectation(self.port, test_name, "SKIP")
metadata_builder = WPTMetadataBuilder(expectations, self.port)
filename, contents = metadata_builder.get_metadata_filename_and_contents(test_name, 'SKIP')
# The metadata file name should not include variants
self.assertEqual("variant.html.ini", filename)
# ..but the contents of the file should include variants in the test name
self.assertEqual("[variant.html?foo=bar]\n disabled: wpt_metadata_builder.py\n", contents)
def test_metadata_for_wpt_test_with_fail_baseline(self):
"""A WPT test with a baseline file containing failures gets metadata."""
test_name = "external/wpt/dir/zzzz.html"
expectations = TestExpectations(self.port)
metadata_builder = WPTMetadataBuilder(expectations, self.port)
filename, contents = metadata_builder.get_metadata_filename_and_contents(
test_name, SUBTEST_FAIL)
self.assertEqual(os.path.join("dir", "zzzz.html.ini"), filename)
self.assertEqual(
"[zzzz.html]\n blink_expect_any_subtest_status: True # wpt_metadata_builder.py\n",
contents)
def test_metadata_for_use_checked_in_metadata_annotation(self):
"""A WPT test annotated to use checked-in metadata."""
test_name = "external/wpt/test.html"
expectations = TestExpectations(self.port)
metadata_builder = WPTMetadataBuilder(expectations, self.port)
filename, contents = metadata_builder.get_metadata_filename_and_contents(
test_name, USE_CHECKED_IN_METADATA)
# If the test is using the metadata that is checked-in, then there is no
# work to be done by the metadata builder.
self.assertIsNone(filename)
self.assertIsNone(contents)
def test_precondition_failed(self):
"""A WPT that fails a precondition."""
test_name = "external/wpt/test.html"
expectations = TestExpectations(self.port)
metadata_builder = WPTMetadataBuilder(expectations, self.port)
filename, contents = metadata_builder.get_metadata_filename_and_contents(
test_name, TEST_PRECONDITION_FAILED)
self.assertEqual("test.html.ini", filename)
# The PASS and FAIL expectations fan out to also include OK and ERROR
# to support reftest/testharness test differences.
self.assertEqual("[test.html]\n expected: [PRECONDITION_FAILED]\n",
contents)
def test_metadata_for_flaky_test(self):
"""A WPT test that is flaky has multiple statuses in metadata."""
test_name = "external/wpt/test.html"
expectations = _make_expectation(self.port, test_name, "PASS FAILURE")
metadata_builder = WPTMetadataBuilder(expectations, self.port)
filename, contents = metadata_builder.get_metadata_filename_and_contents(
test_name, TEST_PASS | TEST_FAIL)
self.assertEqual("test.html.ini", filename)
# The PASS and FAIL expectations fan out to also include OK and ERROR
# to support reftest/testharness test differences.
self.assertEqual("[test.html]\n expected: [PASS, OK, FAIL, ERROR]\n",
contents)
def test_wpt_test_with_baseline(self):
"""A WPT test with a baseline file containing failures gets metadata."""
test_name = "external/wpt/dir/zzzz.html"
# Manually initialize the baseline file and its contents
baseline_filename = "external/wpt/dir/zzzz-expected.txt"
self.host.filesystem.write_text_file(
os.path.join(self.port.web_tests_dir(), baseline_filename),
"This is a test\nPASS some subtest\nFAIL some failing subtest\n")
expectations = TestExpectations(self.port)
metadata_builder = WPTMetadataBuilder(expectations, self.port)
filename, contents = metadata_builder.get_metadata_filename_and_contents(test_name, 'FAIL')
self.assertEqual(os.path.join("dir", "zzzz.html.ini"), filename)
self.assertEqual("[zzzz.html]\n expected: FAIL # wpt_metadata_builder.py\n", contents)
def test_use_subtest_results_flag(self):
"""Test that the --use-subtest-results flag updates metadata correctly.
The --use-subtest-results flag should result in the
blink_expect_any_subtest_status tag not being applied to metadata for
any tests."""
test_name = "external/wpt/test.html"
expectations = _make_expectation(self.port, test_name, "FAILURE")
metadata_builder = WPTMetadataBuilder(expectations, self.port)
metadata_builder.use_subtest_results = True
filename, contents = metadata_builder.get_metadata_filename_and_contents(
test_name, TEST_FAIL)
self.assertEqual("test.html.ini", filename)
self.assertEqual("[test.html]\n expected: [FAIL, ERROR]\n", contents)
def test_metadata_for_wpt_multiglobal_test_with_baseline(self):
"""A WPT test with a baseline file containing failures gets metadata."""
test_name = "external/wpt/dir/multiglob.https.any.window.html"
expectations = TestExpectations(self.port)
metadata_builder = WPTMetadataBuilder(expectations, self.port)
filename, contents = metadata_builder.get_metadata_filename_and_contents(
test_name, SUBTEST_FAIL)
# The metadata filename matches the test *filename*, not the test name,
# which in this case is the js file from the manifest.
self.assertEqual(os.path.join("dir", "multiglob.https.any.js.ini"),
filename)
# The metadata contents contains the *test name*
self.assertEqual(
"[multiglob.https.any.window.html]\n blink_expect_any_subtest_status: True # wpt_metadata_builder.py\n",
contents)
def test_wpt_multiglobal_test_with_baseline(self):
"""A WPT test with a baseline file containing failures gets metadata."""
test_name = "external/wpt/dir/multiglob.https.any.window.html"
# Manually initialize the baseline file and its contents
baseline_filename = "external/wpt/dir/multiglob.https.any.window-expected.txt"
self.host.filesystem.write_text_file(
os.path.join(self.port.web_tests_dir(), baseline_filename),
"This is a test\nPASS some subtest\nFAIL some failing subtest\n")
expectations = TestExpectations(self.port)
metadata_builder = WPTMetadataBuilder(expectations, self.port)
filename, contents = metadata_builder.get_metadata_filename_and_contents(test_name, 'FAIL')
# The metadata filename matches the test *filename*, not the test name,
# which in this case is the js file from the manifest.
self.assertEqual(os.path.join("dir", "multiglob.https.any.js.ini"), filename)
# The metadata contents contains the *test name*
self.assertEqual("[multiglob.https.any.window.html]\n expected: FAIL # wpt_metadata_builder.py\n", contents)
