def test_use_subtest_results_flag_with_baseline_and_default_pass(self): """A test may have a failing baseline because there are subtest failures. When the wptrunner see's the failing subtest it will return failure for the test since we are not setting expectations for the subtest in the metadata. However the expected result will be set to FAIL in the JSON results and the CI will stay green.""" # Create a baseline with a failing subtest, and a default pass expectation test_name = "external/wpt/test.html" baseline_filename = self.port.expected_filename(test_name, '.txt') self.host.filesystem.write_text_file( baseline_filename, "This is a test\nFAIL some subtest\nPASS another subtest\n") expectations = _make_expectation(self.port, test_name, "Pass", is_default_pass=True) metadata_builder = WPTMetadataBuilder(expectations, self.port) metadata_builder.use_subtest_results = True test_and_status_dict = metadata_builder.get_tests_needing_metadata() self.assertEqual(1, len(test_and_status_dict)) self.assertTrue(test_name in test_and_status_dict) self.assertEqual(SUBTEST_FAIL, test_and_status_dict[test_name]) filename, contents = metadata_builder.get_metadata_filename_and_contents( test_name, test_and_status_dict[test_name]) self.assertEqual("test.html.ini", filename) self.assertEqual("[test.html]\n expected: [FAIL, ERROR]\n", contents)
def test_use_subtest_results_flag_with_baseline_and_timeout(self): """If a test has both baseline and a non-default-pass expectation, do not derive expectation from the baseline""" # Create a baseline with a failing subtest, and a TIMEOUT expectation test_name = "external/wpt/test.html" baseline_filename = self.port.expected_filename(test_name, '.txt') self.host.filesystem.write_text_file( baseline_filename, "This is a test\nFAIL some subtest\nPASS another subtest\n") expectations = _make_expectation(self.port, test_name, "TIMEOUT") metadata_builder = WPTMetadataBuilder(expectations, self.port) metadata_builder.use_subtest_results = True test_and_status_dict = metadata_builder.get_tests_needing_metadata() self.assertEqual(1, len(test_and_status_dict)) self.assertTrue(test_name in test_and_status_dict) self.assertEqual(TEST_TIMEOUT, test_and_status_dict[test_name]) filename, contents = metadata_builder.get_metadata_filename_and_contents( test_name, test_and_status_dict[test_name]) self.assertEqual("test.html.ini", filename) self.assertEqual("[test.html]\n expected: [TIMEOUT]\n", contents)
def test_use_subtest_results_flag(self): """Test that the --use-subtest-results flag updates metadata correctly. The --use-subtest-results flag should result in the blink_expect_any_subtest_status tag not being applied to metadata for any tests.""" test_name = "external/wpt/test.html" expectations = _make_expectation(self.port, test_name, "FAILURE") metadata_builder = WPTMetadataBuilder(expectations, self.port) metadata_builder.use_subtest_results = True filename, contents = metadata_builder.get_metadata_filename_and_contents( test_name, TEST_FAIL) self.assertEqual("test.html.ini", filename) self.assertEqual("[test.html]\n expected: [FAIL, ERROR]\n", contents)