def test_harness_remove_all(self):
"""Tests that removing all expectations doesn't delete the file.
Make sure we're prepared for the day when we exterminated flakes.
"""
self._define_builders({
'WebKit Linux Trusty': {
'port_name': 'linux-trusty',
'specifiers': ['Trusty', 'Release']
},
'WebKit Linux Trusty (dbg)': {
'port_name': 'linux-trusty',
'specifiers': ['Trusty', 'Debug']
},
})
# Set up the mock host and port.
host = self._host
host.port_factory = FakePortFactory(host,
all_build_types=('release',
'debug'),
all_systems=(('trusty',
'x86_64'), ))
# Write out a fake TestExpectations file.
test_expectation_path = (
host.port_factory.get().path_to_generic_test_expectations_file())
test_expectations = """
# Remove since passing on both bots.
Bug(test) [ Linux ] test/a.html [ Failure Pass ]"""
files = {test_expectation_path: test_expectations}
host.filesystem = MockFileSystem(files)
self._write_tests_into_filesystem(host.filesystem)
# Write out the fake builder bot results.
expectation_factory = FakeBotTestExpectationsFactory()
expectation_factory.all_results_by_builder = {
'WebKit Linux Trusty': {
'test/a.html': ['PASS', 'PASS', 'PASS'],
},
'WebKit Linux Trusty (dbg)': {
'test/a.html': ['PASS', 'PASS', 'PASS'],
},
}
main(host, expectation_factory, [])
self.assertTrue(host.filesystem.isfile(test_expectation_path))
self.assertEqual(host.filesystem.files[test_expectation_path], '')
def test_harness_no_expectations(self):
"""Tests behavior when TestExpectations file doesn't exist.
Tests that a warning is outputted if the TestExpectations file
doesn't exist.
"""
# Set up the mock host and port.
host = MockHost()
host.port_factory = FakePortFactory(host)
# Write the test file but not the TestExpectations file.
test_expectation_path = (
host.port_factory.get().path_to_generic_test_expectations_file())
host.filesystem = MockFileSystem()
self._write_tests_into_filesystem(host.filesystem)
# Write out the fake builder bot results.
expectation_factory = FakeBotTestExpectationsFactory()
expectation_factory.all_results_by_builder = {}
self.assertFalse(host.filesystem.isfile(test_expectation_path))
return_code = main(host, expectation_factory, [])
self.assertEqual(return_code, 1)
self.assertLog([
"WARNING: Didn't find generic expectations file at: %s\n" %
test_expectation_path
])
self.assertFalse(host.filesystem.isfile(test_expectation_path))
def test_harness_updates_file(self):
"""Tests that the call harness updates the TestExpectations file."""
self._define_builders({
'WebKit Linux Trusty': {
'port_name': 'linux-trusty',
'specifiers': ['Trusty', 'Release']
},
'WebKit Linux Trusty (dbg)': {
'port_name': 'linux-trusty',
'specifiers': ['Trusty', 'Debug']
},
})
# Setup the mock host and port.
host = self._host
host.port_factory = FakePortFactory(host,
all_build_types=('release',
'debug'),
all_systems=(('trusty',
'x86_64'), ))
# Write out a fake TestExpectations file.
test_expectation_path = (
host.port_factory.get().path_to_generic_test_expectations_file())
test_expectations = """
# Remove since passing on both bots.
Bug(test) [ Linux ] test/a.html [ Failure Pass ]
# Keep since there's a failure on release bot.
Bug(test) [ Linux Release ] test/b.html [ Failure Pass ]
# Remove since it's passing on both builders.
Bug(test) test/c.html [ Failure Pass ]
# Keep since there's a failure on debug bot.
Bug(test) [ Linux ] test/d.html [ Failure ]"""
files = {test_expectation_path: test_expectations}
host.filesystem = MockFileSystem(files)
self._write_tests_into_filesystem(host.filesystem)
# Write out the fake builder bot results.
expectation_factory = FakeBotTestExpectationsFactory()
expectation_factory.all_results_by_builder = {
'WebKit Linux Trusty': {
'test/a.html': ['PASS', 'PASS', 'PASS'],
'test/b.html': ['PASS', 'IMAGE', 'PASS'],
'test/c.html': ['PASS', 'PASS', 'PASS'],
'test/d.html': ['PASS', 'PASS', 'PASS'],
},
'WebKit Linux Trusty (dbg)': {
'test/a.html': ['PASS', 'PASS', 'PASS'],
'test/b.html': ['PASS', 'PASS', 'PASS'],
'test/c.html': ['PASS', 'PASS', 'PASS'],
'test/d.html': ['IMAGE', 'PASS', 'PASS'],
},
}
main(host, expectation_factory, [])
self.assertEqual(
host.filesystem.files[test_expectation_path],
(""" # Keep since there's a failure on release bot.
Bug(test) [ Linux Release ] test/b.html [ Failure Pass ]
# Keep since there's a failure on debug bot.
Bug(test) [ Linux ] test/d.html [ Failure ]"""))
#!/usr/bin/env vpython
#
# Copyright 2016 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import sys
from blinkpy.common import add_webkitpy # pylint: disable=unused-import
from webkitpy.common import host
from webkitpy.layout_tests import update_flaky_expectations
from webkitpy.layout_tests.layout_package.bot_test_expectations import BotTestExpectationsFactory
if __name__ == "__main__":
host = host.Host()
return_code = update_flaky_expectations.main(
host, BotTestExpectationsFactory(host.builders), sys.argv[1:])
sys.exit(return_code)