def test_use_subtest_results_flag_with_baseline_and_default_pass(self):
"""A test may have a failing baseline because there are subtest failures.
When the wptrunner see's the failing subtest it will return failure
for the test since we are not setting expectations for the subtest in
the metadata. However the expected result will be set to FAIL in the
JSON results and the CI will stay green."""
# Create a baseline with a failing subtest, and a default pass expectation
test_name = "external/wpt/test.html"
baseline_filename = self.port.expected_filename(test_name, '.txt')
self.host.filesystem.write_text_file(
baseline_filename,
"This is a test\nFAIL some subtest\nPASS another subtest\n")
expectations = _make_expectation(self.port, test_name,
"Pass", is_default_pass=True)
metadata_builder = WPTMetadataBuilder(expectations, self.port)
metadata_builder.use_subtest_results = True
test_and_status_dict = metadata_builder.get_tests_needing_metadata()
self.assertEqual(1, len(test_and_status_dict))
self.assertTrue(test_name in test_and_status_dict)
self.assertEqual(SUBTEST_FAIL, test_and_status_dict[test_name])
filename, contents = metadata_builder.get_metadata_filename_and_contents(
test_name, test_and_status_dict[test_name])
self.assertEqual("test.html.ini", filename)
self.assertEqual("[test.html]\n expected: [FAIL, ERROR]\n",
contents)
def test_use_subtest_results_flag_with_baseline_and_timeout(self):
"""If a test has both baseline and a non-default-pass expectation, do not
derive expectation from the baseline"""
# Create a baseline with a failing subtest, and a TIMEOUT expectation
test_name = "external/wpt/test.html"
baseline_filename = self.port.expected_filename(test_name, '.txt')
self.host.filesystem.write_text_file(
baseline_filename,
"This is a test\nFAIL some subtest\nPASS another subtest\n")
expectations = _make_expectation(self.port, test_name, "TIMEOUT")
metadata_builder = WPTMetadataBuilder(expectations, self.port)
metadata_builder.use_subtest_results = True
test_and_status_dict = metadata_builder.get_tests_needing_metadata()
self.assertEqual(1, len(test_and_status_dict))
self.assertTrue(test_name in test_and_status_dict)
self.assertEqual(TEST_TIMEOUT, test_and_status_dict[test_name])
filename, contents = metadata_builder.get_metadata_filename_and_contents(
test_name, test_and_status_dict[test_name])
self.assertEqual("test.html.ini", filename)
self.assertEqual("[test.html]\n expected: [TIMEOUT]\n",
contents)
def test_use_subtest_results_flag(self):
"""Test that the --use-subtest-results flag updates metadata correctly.
The --use-subtest-results flag should result in the
blink_expect_any_subtest_status tag not being applied to metadata for
any tests."""
test_name = "external/wpt/test.html"
expectations = _make_expectation(self.port, test_name, "FAILURE")
metadata_builder = WPTMetadataBuilder(expectations, self.port)
metadata_builder.use_subtest_results = True
filename, contents = metadata_builder.get_metadata_filename_and_contents(
test_name, TEST_FAIL)
self.assertEqual("test.html.ini", filename)
self.assertEqual("[test.html]\n expected: [FAIL, ERROR]\n", contents)
