def test_without_use_subtest_results_flag_with_baseline(self):
"""A test has a failing baseline."""
test_name = "external/wpt/test.html"
# Create a baseline with a failing subtest, and a TIMEOUT expectation
baseline_filename = self.port.expected_filename(test_name, '.txt')
self.host.filesystem.write_text_file(
baseline_filename,
"This is a test\nFAIL some subtest\nPASS another subtest\n")
expectations = _make_expectation(self.port, test_name, "TIMEOUT")
metadata_builder = WPTMetadataBuilder(expectations, self.port)
test_and_status_dict = metadata_builder.get_tests_needing_metadata()
self.assertEqual(1, len(test_and_status_dict))
self.assertTrue(test_name in test_and_status_dict)
self.assertEqual(TEST_TIMEOUT,
test_and_status_dict[test_name])
filename, contents = metadata_builder.get_metadata_filename_and_contents(
test_name, test_and_status_dict[test_name])
self.assertEqual("test.html.ini", filename)
self.assertEqual(("[test.html]\n blink_expect_any_subtest_status: "
"True # wpt_metadata_builder.py\n "
"expected: [TIMEOUT]\n"), contents)
def test_overriding_skip_expectation_with_annotation(self):
"""A skipped tests is overridden by an annotation.
In this case, the SKIP status gets ignored because the status line in
the override file has the 'wpt_use_checked_in_metadata' annotation. This
forces us to prioritize the checked-in metadata over whatever is in
any of the expectation files.
"""
test_name = "external/wpt/test.html"
expectation_dict = OrderedDict()
_append_to_expectation_dict(expectation_dict, "NeverFixTests",
test_name, "SKIP")
_append_to_expectation_dict(
expectation_dict,
"WPTOverrideExpectations",
test_name,
"PASS",
trailing_comments=" # wpt_use_checked_in_metadata")
expectations = _make_expectation_with_dict(self.port, expectation_dict)
metadata_builder = WPTMetadataBuilder(expectations, self.port)
test_and_status_dict = metadata_builder.get_tests_needing_metadata()
self.assertEqual(1, len(test_and_status_dict))
self.assertTrue(test_name in test_and_status_dict)
# Skip statuses always live alone. The override status is ignored
# because it does not have an annotation to use checked-in metadata.
self.assertEqual(USE_CHECKED_IN_METADATA,
test_and_status_dict[test_name])
def test_use_subtest_results_flag_with_baseline_and_default_pass(self):
"""A test may have a failing baseline because there are subtest failures.
When the wptrunner see's the failing subtest it will return failure
for the test since we are not setting expectations for the subtest in
the metadata. However the expected result will be set to FAIL in the
JSON results and the CI will stay green."""
# Create a baseline with a failing subtest, and a default pass expectation
test_name = "external/wpt/test.html"
baseline_filename = self.port.expected_filename(test_name, '.txt')
self.host.filesystem.write_text_file(
baseline_filename,
"This is a test\nFAIL some subtest\nPASS another subtest\n")
expectations = _make_expectation(self.port, test_name,
"Pass", is_default_pass=True)
metadata_builder = WPTMetadataBuilder(expectations, self.port)
metadata_builder.use_subtest_results = True
test_and_status_dict = metadata_builder.get_tests_needing_metadata()
self.assertEqual(1, len(test_and_status_dict))
self.assertTrue(test_name in test_and_status_dict)
self.assertEqual(SUBTEST_FAIL, test_and_status_dict[test_name])
filename, contents = metadata_builder.get_metadata_filename_and_contents(
test_name, test_and_status_dict[test_name])
self.assertEqual("test.html.ini", filename)
self.assertEqual("[test.html]\n expected: [FAIL, ERROR]\n",
contents)
def test_use_subtest_results_flag_with_baseline_and_timeout(self):
"""If a test has both baseline and a non-default-pass expectation, do not
derive expectation from the baseline"""
# Create a baseline with a failing subtest, and a TIMEOUT expectation
test_name = "external/wpt/test.html"
baseline_filename = self.port.expected_filename(test_name, '.txt')
self.host.filesystem.write_text_file(
baseline_filename,
"This is a test\nFAIL some subtest\nPASS another subtest\n")
expectations = _make_expectation(self.port, test_name, "TIMEOUT")
metadata_builder = WPTMetadataBuilder(expectations, self.port)
metadata_builder.use_subtest_results = True
test_and_status_dict = metadata_builder.get_tests_needing_metadata()
self.assertEqual(1, len(test_and_status_dict))
self.assertTrue(test_name in test_and_status_dict)
self.assertEqual(TEST_TIMEOUT, test_and_status_dict[test_name])
filename, contents = metadata_builder.get_metadata_filename_and_contents(
test_name, test_and_status_dict[test_name])
self.assertEqual("test.html.ini", filename)
self.assertEqual("[test.html]\n expected: [TIMEOUT]\n",
contents)
def test_wpt_test_not_skipped(self):
"""A WPT test that is not skipped should not get a SKIP metadata."""
test_name = "external/wpt/test.html"
expectations = _make_expectation(self.port, test_name, "TIMEOUT")
metadata_builder = WPTMetadataBuilder(expectations, self.port)
test_names = metadata_builder.get_tests_needing_metadata()
self.assertFalse(test_names)
def test_parse_baseline_all_pass(self):
"""A WPT test with an all-pass baseline doesn't get metadata."""
# Here we use a test_name that is actually in the test manifest
test_name = "external/wpt/dir/zzzz.html"
# Manually initialize the baseline file and its contents
baseline_filename = self.port.expected_filename(test_name, '.txt')
self.host.filesystem.write_text_file(
baseline_filename,
"This is a test\nPASS some subtest\nPASS another subtest\n")
expectations = TestExpectations(self.port)
metadata_builder = WPTMetadataBuilder(expectations, self.port)
test_and_status_dict = metadata_builder.get_tests_needing_metadata()
self.assertFalse(test_and_status_dict)
def test_wpt_test_not_skipped(self):
"""A WPT test that is not skipped should not get a SKIP metadata."""
test_name = "external/wpt/test.html"
expectations = _make_expectation(self.port, test_name, "TIMEOUT")
metadata_builder = WPTMetadataBuilder(expectations, self.port)
test_names = metadata_builder.get_tests_needing_metadata()
# The test will appear in the result but won't have a SKIP status
found = False
for name_item, status_item in test_names.items():
if name_item == test_name:
found = True
self.assertNotEqual(SKIP_TEST, status_item)
self.assertTrue(found)
def test_parse_precondition_failure_annotation(self):
"""Check that we parse the wpt_precondition_failed annotation correctly."""
test_name = "external/wpt/test.html"
expectations = _make_expectation(
self.port,
test_name,
"PASS",
trailing_comments=" # wpt_precondition_failed")
metadata_builder = WPTMetadataBuilder(expectations, self.port)
test_and_status_dict = metadata_builder.get_tests_needing_metadata()
self.assertEqual(1, len(test_and_status_dict))
self.assertTrue(test_name in test_and_status_dict)
self.assertEqual(TEST_PASS | TEST_PRECONDITION_FAILED,
test_and_status_dict[test_name])
def test_parse_baseline_harness_error(self):
"""Test parsing a baseline with a harness error."""
# Here we use a test_name that is actually in the test manifest
test_name = "external/wpt/dir/zzzz.html"
# Manually initialize the baseline file and its contents
baseline_filename = self.port.expected_filename(test_name, '.txt')
self.host.filesystem.write_text_file(
baseline_filename, "This is a test\nHarness Error. some stuff\n")
expectations = TestExpectations(self.port)
metadata_builder = WPTMetadataBuilder(expectations, self.port)
test_and_status_dict = metadata_builder.get_tests_needing_metadata()
self.assertEqual(1, len(test_and_status_dict))
self.assertTrue(test_name in test_and_status_dict)
self.assertEqual(HARNESS_ERROR, test_and_status_dict[test_name])
def test_baseline_and_expectation(self):
"""A test has a failing baseline and a timeout expectation."""
test_name = "external/wpt/test.html"
# Create a baseline with a failing subtest, and a TIMEOUT expectation
baseline_filename = self.port.expected_filename(test_name, '.txt')
self.host.filesystem.write_text_file(
baseline_filename,
"This is a test\nFAIL some subtest\nPASS another subtest\n")
expectations = _make_expectation(self.port, test_name, "TIMEOUT")
metadata_builder = WPTMetadataBuilder(expectations, self.port)
test_and_status_dict = metadata_builder.get_tests_needing_metadata()
self.assertEqual(1, len(test_and_status_dict))
self.assertTrue(test_name in test_and_status_dict)
self.assertEqual(TEST_TIMEOUT, test_and_status_dict[test_name])
def test_parse_baseline_subtest_timeout(self):
"""Test parsing a baseline with a timeout subtest."""
# Here we use a test_name that is actually in the test manifest
test_name = "external/wpt/dir/zzzz.html"
# Manually initialize the baseline file and its contents
baseline_filename = self.port.expected_filename(test_name, '.txt')
self.host.filesystem.write_text_file(
baseline_filename,
"This is a test\nTIMEOUT some subtest\nPASS another subtest\n")
expectations = TestExpectations(self.port)
metadata_builder = WPTMetadataBuilder(expectations, self.port)
test_and_status_dict = metadata_builder.get_tests_needing_metadata()
self.assertEqual(1, len(test_and_status_dict))
self.assertTrue(test_name in test_and_status_dict)
self.assertEqual(SUBTEST_FAIL, test_and_status_dict[test_name])
def test_expectations_across_files(self):
"""Check the inheritance order of expectations across several files."""
test_name = "external/wpt/test.html"
expectation_dict = OrderedDict()
_append_to_expectation_dict(expectation_dict, "TestExpectations",
test_name, "FAILURE")
_append_to_expectation_dict(expectation_dict,
"WPTOverrideExpectations", test_name,
"TIMEOUT")
expectations = _make_expectation_with_dict(self.port, expectation_dict)
metadata_builder = WPTMetadataBuilder(expectations, self.port)
test_and_status_dict = metadata_builder.get_tests_needing_metadata()
self.assertEqual(1, len(test_and_status_dict))
self.assertTrue(test_name in test_and_status_dict)
# The test statuses from the two files are combined
self.assertEqual(TEST_FAIL | TEST_TIMEOUT,
test_and_status_dict[test_name])
def test_overriding_skip_expectation_no_annotation(self):
"""Check how a skipped test (in NeverFix) interacts with an override.
In this case, only the SKIP status is translated since skipped tests are
not combined with any other statuses. This is because wpt metadata
requires a special 'disabled' keyword to skip tests, it's not just
another status in the expected status list like in Chromium.
"""
test_name = "external/wpt/test.html"
expectation_dict = OrderedDict()
_append_to_expectation_dict(expectation_dict, "NeverFixTests",
test_name, "SKIP")
_append_to_expectation_dict(expectation_dict,
"WPTOverrideExpectations", test_name,
"FAILURE")
expectations = _make_expectation_with_dict(self.port, expectation_dict)
metadata_builder = WPTMetadataBuilder(expectations, self.port)
test_and_status_dict = metadata_builder.get_tests_needing_metadata()
self.assertEqual(1, len(test_and_status_dict))
self.assertTrue(test_name in test_and_status_dict)
# Skip statuses always live alone. The override status is ignored
# because it does not have an annotation to use checked-in metadata.
self.assertEqual(SKIP_TEST, test_and_status_dict[test_name])
