def test_new_manual_tests_get_skip_expectation(self): host = self.mock_host() updater = WPTExpectationsUpdater(host) results = { 'external/wpt/x-manual.html': { ( 'test-linux-precise', 'test-linux-trusty', 'test-mac-mac10.10', 'test-mac-mac10.11', 'test-win-win7', 'test-win-win10', ): SimpleTestResult(expected='PASS', actual='MISSING', bug='crbug.com/test') } } tests_to_rebaseline, _ = updater.get_tests_to_rebaseline(results) self.assertEqual(tests_to_rebaseline, []) self.assertEqual( updater.create_line_dict(results), { 'external/wpt/x-manual.html': ['crbug.com/test external/wpt/x-manual.html [ Skip ]'] })
def test_get_tests_to_rebaseline_also_returns_slow_tests(self): host = self.mock_host() test_results_dict = { 'external/fake/test/path.html': { 'one': { 'expected': 'SLOW', 'actual': 'TEXT' }, 'two': { 'expected': 'SLOW', 'actual': 'TIMEOUT' }, }, } test_results_dict_copy = copy.deepcopy(test_results_dict) host.filesystem.files[ '/mock-checkout/third_party/WebKit/LayoutTests/external/fake/test/path.html'] = ( '<script src="/resources/testharness.js"></script>') updater = WPTExpectationsUpdater(host) tests_to_rebaseline, modified_test_results = updater.get_tests_to_rebaseline( test_results_dict) self.assertEqual(tests_to_rebaseline, ['external/fake/test/path.html']) # The record for the builder with a timeout is kept, but not with a text mismatch, # since that should be covered by downloading a new baseline. self.assertEqual( modified_test_results, { 'external/fake/test/path.html': { 'two': { 'expected': 'SLOW', 'actual': 'TIMEOUT' }, }, }) # The original dict isn't modified. self.assertEqual(test_results_dict, test_results_dict_copy)
def test_get_tests_to_rebaseline_also_returns_slow_tests(self): host = self.mock_host() results = { 'external/wpt/test/path.html': { 'one': SimpleTestResult(expected='SLOW', actual='TEXT', bug='bug'), 'two': SimpleTestResult(expected='SLOW', actual='TIMEOUT', bug='bug'), }, } results_copy = copy.deepcopy(results) updater = WPTExpectationsUpdater(host) tests_to_rebaseline, modified_test_results = updater.get_tests_to_rebaseline( results) self.assertEqual(tests_to_rebaseline, ['external/wpt/test/path.html']) # The record for the builder with a timeout is kept, but not with a text mismatch, # since that should be covered by downloading a new baseline. self.assertEqual( modified_test_results, { 'external/wpt/test/path.html': { 'two': SimpleTestResult( expected='SLOW', actual='TIMEOUT', bug='bug'), }, }) # The original dict isn't modified. self.assertEqual(results, results_copy)
def test_get_test_to_rebaseline_returns_only_js_tests(self): host = self.mock_host() host.filesystem.files[ '/mock-checkout/third_party/WebKit/LayoutTests/external/fake/test/path.html'] = ( 'this file does not look like a testharness JS test.') updater = WPTExpectationsUpdater(host) two = { 'external/fake/test/path.html': { 'one': { 'expected': 'FAIL', 'actual': 'PASS', 'bug': 'crbug.com/test' }, 'two': { 'expected': 'FAIL', 'actual': 'TIMEOUT', 'bug': 'crbug.com/test' }, 'three': { 'expected': 'FAIL', 'actual': 'PASS', 'bug': 'crbug.com/test' }, } } tests_to_rebaseline, _ = updater.get_tests_to_rebaseline(two) self.assertEqual(tests_to_rebaseline, [])
def test_get_test_to_rebaseline_returns_only_tests_with_failures(self): host = self.mock_host() host.filesystem.files[ '/mock-checkout/third_party/WebKit/LayoutTests/external/fake/test/path.html'] = ( '<script src="/resources/testharness.js"></script>') host.filesystem.files[ '/mock-checkout/third_party/WebKit/LayoutTests/external/other/test/path.html'] = ( '<script src="/resources/testharness.js"></script>') updater = WPTExpectationsUpdater(host) two = { 'external/fake/test/path.html': { 'one': { 'expected': 'FAIL', 'actual': 'PASS' }, 'two': { 'expected': 'FAIL', 'actual': 'TIMEOUT' }, 'three': { 'expected': 'FAIL', 'actual': 'PASS' }, } } tests_to_rebaseline, _ = updater.get_tests_to_rebaseline(two) # The other test doesn't have an entry in the test results dict, so it is not listed as a test to rebaseline. self.assertEqual(tests_to_rebaseline, ['external/fake/test/path.html'])
def test_get_test_to_rebaseline_does_not_return_ref_tests(self): host = self.mock_host() updater = WPTExpectationsUpdater(host) two = { 'external/wpt/reftest.html': { 'one': {'expected': 'FAIL', 'actual': 'PASS', 'bug': 'crbug.com/test'}, 'two': {'expected': 'FAIL', 'actual': 'TIMEOUT', 'bug': 'crbug.com/test'}, 'three': {'expected': 'FAIL', 'actual': 'PASS', 'bug': 'crbug.com/test'}, } } tests_to_rebaseline, _ = updater.get_tests_to_rebaseline(two) self.assertEqual(tests_to_rebaseline, [])
def test_get_test_to_rebaseline_returns_only_tests_with_failures(self): host = self.mock_host() updater = WPTExpectationsUpdater(host) two = { 'external/wpt/test/path.html': { 'one': {'expected': 'FAIL', 'actual': 'PASS'}, 'two': {'expected': 'FAIL', 'actual': 'TIMEOUT'}, 'three': {'expected': 'FAIL', 'actual': 'PASS'}, } } tests_to_rebaseline, _ = updater.get_tests_to_rebaseline(two) # external/wpt/test/zzzz.html is another possible candidate, but it # is not listed in the results dict, so it shall not be rebaselined. self.assertEqual(tests_to_rebaseline, ['external/wpt/test/path.html'])
def test_get_test_to_rebaseline_does_not_return_ref_tests(self): host = self.mock_host() updater = WPTExpectationsUpdater(host) two = { 'external/wpt/reftest.html': { 'one': SimpleTestResult(expected='FAIL', actual='PASS', bug='crbug.com/test'), 'two': SimpleTestResult(expected='FAIL', actual='TIMEOUT', bug='crbug.com/test'), 'three': SimpleTestResult(expected='FAIL', actual='PASS', bug='crbug.com/test'), } } tests_to_rebaseline, _ = updater.get_tests_to_rebaseline(two) self.assertEqual(tests_to_rebaseline, [])