def test_harness_remove_all(self): """Tests that removing all expectations doesn't delete the file. Make sure we're prepared for the day when we exterminated flakes. """ self._define_builders({ 'WebKit Linux Trusty': { 'port_name': 'linux-trusty', 'specifiers': ['Trusty', 'Release'] }, 'WebKit Linux Trusty (dbg)': { 'port_name': 'linux-trusty', 'specifiers': ['Trusty', 'Debug'] }, }) # Set up the mock host and port. host = self._host host.port_factory = FakePortFactory(host, all_build_types=('release', 'debug'), all_systems=(('trusty', 'x86_64'), )) # Write out a fake TestExpectations file. test_expectation_path = ( host.port_factory.get().path_to_generic_test_expectations_file()) test_expectations = """ # Remove since passing on both bots. Bug(test) [ Linux ] test/a.html [ Failure Pass ]""" files = {test_expectation_path: test_expectations} host.filesystem = MockFileSystem(files) self._write_tests_into_filesystem(host.filesystem) # Write out the fake builder bot results. expectation_factory = FakeBotTestExpectationsFactory() expectation_factory.all_results_by_builder = { 'WebKit Linux Trusty': { 'test/a.html': ['PASS', 'PASS', 'PASS'], }, 'WebKit Linux Trusty (dbg)': { 'test/a.html': ['PASS', 'PASS', 'PASS'], }, } main(host, expectation_factory, []) self.assertTrue(host.filesystem.isfile(test_expectation_path)) self.assertEqual(host.filesystem.files[test_expectation_path], '')
def test_harness_no_expectations(self): """Tests behavior when TestExpectations file doesn't exist. Tests that a warning is outputted if the TestExpectations file doesn't exist. """ # Set up the mock host and port. host = MockHost() host.port_factory = FakePortFactory(host) # Write the test file but not the TestExpectations file. test_expectation_path = ( host.port_factory.get().path_to_generic_test_expectations_file()) host.filesystem = MockFileSystem() self._write_tests_into_filesystem(host.filesystem) # Write out the fake builder bot results. expectation_factory = FakeBotTestExpectationsFactory() expectation_factory.all_results_by_builder = {} self.assertFalse(host.filesystem.isfile(test_expectation_path)) return_code = main(host, expectation_factory, []) self.assertEqual(return_code, 1) self.assertLog([ "WARNING: Didn't find generic expectations file at: %s\n" % test_expectation_path ]) self.assertFalse(host.filesystem.isfile(test_expectation_path))
def test_harness_updates_file(self): """Tests that the call harness updates the TestExpectations file.""" self._define_builders({ 'WebKit Linux Trusty': { 'port_name': 'linux-trusty', 'specifiers': ['Trusty', 'Release'] }, 'WebKit Linux Trusty (dbg)': { 'port_name': 'linux-trusty', 'specifiers': ['Trusty', 'Debug'] }, }) # Setup the mock host and port. host = self._host host.port_factory = FakePortFactory(host, all_build_types=('release', 'debug'), all_systems=(('trusty', 'x86_64'), )) # Write out a fake TestExpectations file. test_expectation_path = ( host.port_factory.get().path_to_generic_test_expectations_file()) test_expectations = """ # Remove since passing on both bots. Bug(test) [ Linux ] test/a.html [ Failure Pass ] # Keep since there's a failure on release bot. Bug(test) [ Linux Release ] test/b.html [ Failure Pass ] # Remove since it's passing on both builders. Bug(test) test/c.html [ Failure Pass ] # Keep since there's a failure on debug bot. Bug(test) [ Linux ] test/d.html [ Failure ]""" files = {test_expectation_path: test_expectations} host.filesystem = MockFileSystem(files) self._write_tests_into_filesystem(host.filesystem) # Write out the fake builder bot results. expectation_factory = FakeBotTestExpectationsFactory() expectation_factory.all_results_by_builder = { 'WebKit Linux Trusty': { 'test/a.html': ['PASS', 'PASS', 'PASS'], 'test/b.html': ['PASS', 'IMAGE', 'PASS'], 'test/c.html': ['PASS', 'PASS', 'PASS'], 'test/d.html': ['PASS', 'PASS', 'PASS'], }, 'WebKit Linux Trusty (dbg)': { 'test/a.html': ['PASS', 'PASS', 'PASS'], 'test/b.html': ['PASS', 'PASS', 'PASS'], 'test/c.html': ['PASS', 'PASS', 'PASS'], 'test/d.html': ['IMAGE', 'PASS', 'PASS'], }, } main(host, expectation_factory, []) self.assertEqual( host.filesystem.files[test_expectation_path], (""" # Keep since there's a failure on release bot. Bug(test) [ Linux Release ] test/b.html [ Failure Pass ] # Keep since there's a failure on debug bot. Bug(test) [ Linux ] test/d.html [ Failure ]"""))
#!/usr/bin/env vpython # # Copyright 2016 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import sys from blinkpy.common import add_webkitpy # pylint: disable=unused-import from webkitpy.common import host from webkitpy.layout_tests import update_flaky_expectations from webkitpy.layout_tests.layout_package.bot_test_expectations import BotTestExpectationsFactory if __name__ == "__main__": host = host.Host() return_code = update_flaky_expectations.main( host, BotTestExpectationsFactory(host.builders), sys.argv[1:]) sys.exit(return_code)