def test_new_manual_tests_get_skip_expectation(self):
host = self.mock_host()
updater = WPTExpectationsUpdater(host)
results = {
'external/wpt/x-manual.html': {
(
'test-linux-precise',
'test-linux-trusty',
'test-mac-mac10.10',
'test-mac-mac10.11',
'test-win-win7',
'test-win-win10',
):
SimpleTestResult(expected='PASS',
actual='MISSING',
bug='crbug.com/test')
}
}
tests_to_rebaseline, _ = updater.get_tests_to_rebaseline(results)
self.assertEqual(tests_to_rebaseline, [])
self.assertEqual(
updater.create_line_dict(results), {
'external/wpt/x-manual.html':
['crbug.com/test external/wpt/x-manual.html [ Skip ]']
})
def test_get_tests_to_rebaseline_also_returns_slow_tests(self):
host = self.mock_host()
test_results_dict = {
'external/fake/test/path.html': {
'one': {
'expected': 'SLOW',
'actual': 'TEXT'
},
'two': {
'expected': 'SLOW',
'actual': 'TIMEOUT'
},
},
}
test_results_dict_copy = copy.deepcopy(test_results_dict)
host.filesystem.files[
'/mock-checkout/third_party/WebKit/LayoutTests/external/fake/test/path.html'] = (
'<script src="/resources/testharness.js"></script>')
updater = WPTExpectationsUpdater(host)
tests_to_rebaseline, modified_test_results = updater.get_tests_to_rebaseline(
test_results_dict)
self.assertEqual(tests_to_rebaseline, ['external/fake/test/path.html'])
# The record for the builder with a timeout is kept, but not with a text mismatch,
# since that should be covered by downloading a new baseline.
self.assertEqual(
modified_test_results, {
'external/fake/test/path.html': {
'two': {
'expected': 'SLOW',
'actual': 'TIMEOUT'
},
},
})
# The original dict isn't modified.
self.assertEqual(test_results_dict, test_results_dict_copy)
def test_get_tests_to_rebaseline_also_returns_slow_tests(self):
host = self.mock_host()
results = {
'external/wpt/test/path.html': {
'one':
SimpleTestResult(expected='SLOW', actual='TEXT', bug='bug'),
'two':
SimpleTestResult(expected='SLOW', actual='TIMEOUT', bug='bug'),
},
}
results_copy = copy.deepcopy(results)
updater = WPTExpectationsUpdater(host)
tests_to_rebaseline, modified_test_results = updater.get_tests_to_rebaseline(
results)
self.assertEqual(tests_to_rebaseline, ['external/wpt/test/path.html'])
# The record for the builder with a timeout is kept, but not with a text mismatch,
# since that should be covered by downloading a new baseline.
self.assertEqual(
modified_test_results, {
'external/wpt/test/path.html': {
'two':
SimpleTestResult(
expected='SLOW', actual='TIMEOUT', bug='bug'),
},
})
# The original dict isn't modified.
self.assertEqual(results, results_copy)
def test_get_test_to_rebaseline_returns_only_js_tests(self):
host = self.mock_host()
host.filesystem.files[
'/mock-checkout/third_party/WebKit/LayoutTests/external/fake/test/path.html'] = (
'this file does not look like a testharness JS test.')
updater = WPTExpectationsUpdater(host)
two = {
'external/fake/test/path.html': {
'one': {
'expected': 'FAIL',
'actual': 'PASS',
'bug': 'crbug.com/test'
},
'two': {
'expected': 'FAIL',
'actual': 'TIMEOUT',
'bug': 'crbug.com/test'
},
'three': {
'expected': 'FAIL',
'actual': 'PASS',
'bug': 'crbug.com/test'
},
}
}
tests_to_rebaseline, _ = updater.get_tests_to_rebaseline(two)
self.assertEqual(tests_to_rebaseline, [])
def test_get_test_to_rebaseline_returns_only_tests_with_failures(self):
host = self.mock_host()
host.filesystem.files[
'/mock-checkout/third_party/WebKit/LayoutTests/external/fake/test/path.html'] = (
'<script src="/resources/testharness.js"></script>')
host.filesystem.files[
'/mock-checkout/third_party/WebKit/LayoutTests/external/other/test/path.html'] = (
'<script src="/resources/testharness.js"></script>')
updater = WPTExpectationsUpdater(host)
two = {
'external/fake/test/path.html': {
'one': {
'expected': 'FAIL',
'actual': 'PASS'
},
'two': {
'expected': 'FAIL',
'actual': 'TIMEOUT'
},
'three': {
'expected': 'FAIL',
'actual': 'PASS'
},
}
}
tests_to_rebaseline, _ = updater.get_tests_to_rebaseline(two)
# The other test doesn't have an entry in the test results dict, so it is not listed as a test to rebaseline.
self.assertEqual(tests_to_rebaseline, ['external/fake/test/path.html'])
def test_get_test_to_rebaseline_does_not_return_ref_tests(self):
host = self.mock_host()
updater = WPTExpectationsUpdater(host)
two = {
'external/wpt/reftest.html': {
'one': {'expected': 'FAIL', 'actual': 'PASS', 'bug': 'crbug.com/test'},
'two': {'expected': 'FAIL', 'actual': 'TIMEOUT', 'bug': 'crbug.com/test'},
'three': {'expected': 'FAIL', 'actual': 'PASS', 'bug': 'crbug.com/test'},
}
}
tests_to_rebaseline, _ = updater.get_tests_to_rebaseline(two)
self.assertEqual(tests_to_rebaseline, [])
def test_get_test_to_rebaseline_returns_only_tests_with_failures(self):
host = self.mock_host()
updater = WPTExpectationsUpdater(host)
two = {
'external/wpt/test/path.html': {
'one': {'expected': 'FAIL', 'actual': 'PASS'},
'two': {'expected': 'FAIL', 'actual': 'TIMEOUT'},
'three': {'expected': 'FAIL', 'actual': 'PASS'},
}
}
tests_to_rebaseline, _ = updater.get_tests_to_rebaseline(two)
# external/wpt/test/zzzz.html is another possible candidate, but it
# is not listed in the results dict, so it shall not be rebaselined.
self.assertEqual(tests_to_rebaseline, ['external/wpt/test/path.html'])
def test_get_test_to_rebaseline_does_not_return_ref_tests(self):
host = self.mock_host()
updater = WPTExpectationsUpdater(host)
two = {
'external/wpt/reftest.html': {
'one':
SimpleTestResult(expected='FAIL',
actual='PASS',
bug='crbug.com/test'),
'two':
SimpleTestResult(expected='FAIL',
actual='TIMEOUT',
bug='crbug.com/test'),
'three':
SimpleTestResult(expected='FAIL',
actual='PASS',
bug='crbug.com/test'),
}
}
tests_to_rebaseline, _ = updater.get_tests_to_rebaseline(two)
self.assertEqual(tests_to_rebaseline, [])