def test_harness_remove_all(self): """Tests that removing all expectations doesn't delete the file. Make sure we're prepared for the day when we exterminated flakes. """ self._define_builders({ "WebKit Linux Trusty": { "port_name": "linux-trusty", "specifiers": ['Trusty', 'Release'] }, "WebKit Linux Trusty (dbg)": { "port_name": "linux-trusty", "specifiers": ['Trusty', 'Debug'] }, }) # Set up the mock host and port. host = MockHost() host.port_factory = FakePortFactory( host, all_build_types=('release', 'debug'), all_systems=(('trusty', 'x86_64'),)) # Write out a fake TestExpectations file. test_expectation_path = ( host.port_factory.get().path_to_generic_test_expectations_file()) test_expectations = """ # Remove since passing on both bots. Bug(test) [ Linux ] test/a.html [ Failure Pass ]""" files = { test_expectation_path: test_expectations } host.filesystem = MockFileSystem(files) self._write_tests_into_filesystem(host.filesystem) # Write out the fake builder bot results. expectation_factory = FakeBotTestExpectationsFactory() expectation_factory._all_results_by_builder = { 'WebKit Linux Trusty': { "test/a.html": ["PASS", "PASS", "PASS"], }, 'WebKit Linux Trusty (dbg)': { "test/a.html": ["PASS", "PASS", "PASS"], }, } main(host, expectation_factory, []) self.assertTrue(host.filesystem.isfile(test_expectation_path)) self.assertEqual(host.filesystem.files[test_expectation_path], '')
def test_harness_remove_all(self): """Tests that removing all expectations doesn't delete the file. Make sure we're prepared for the day when we exterminated flakes. """ self._define_builders({ "WebKit Linux Trusty": { "port_name": "linux-trusty", "specifiers": ['Trusty', 'Release'] }, "WebKit Linux Trusty (dbg)": { "port_name": "linux-trusty", "specifiers": ['Trusty', 'Debug'] }, }) # Setup the mock host and port. host = MockHost() host.port_factory = FakePortFactory( host, all_build_types=('release', 'debug'), all_systems=(('trusty', 'x86_64'),)) # Write out a fake TestExpectations file. test_expectation_path = ( host.port_factory.get().path_to_generic_test_expectations_file()) test_expectations = """ # Remove since passing on both bots. Bug(test) [ Linux ] test/a.html [ Failure Pass ]""" files = { test_expectation_path: test_expectations } host.filesystem = MockFileSystem(files) self._write_tests_into_filesystem(host.filesystem) # Write out the fake builder bot results. expectation_factory = FakeBotTestExpectationsFactory() expectation_factory._all_results_by_builder = { 'WebKit Linux Trusty': { "test/a.html": ["PASS", "PASS", "PASS"], }, 'WebKit Linux Trusty (dbg)': { "test/a.html": ["PASS", "PASS", "PASS"], }, } main(host, expectation_factory, []) self.assertTrue(host.filesystem.isfile(test_expectation_path)) self.assertEqual(host.filesystem.files[test_expectation_path], '')
def test_harness_no_expectations(self): """Tests behavior when TestExpectations file doesn't exist. Tests that a warning is outputted if the TestExpectations file doesn't exist. """ # Setup the mock host and port. host = MockHost() host.port_factory = FakePortFactory(host) # Write the test file but not the TestExpectations file. test_expectation_path = ( host.port_factory.get().path_to_generic_test_expectations_file()) host.filesystem = MockFileSystem() self._write_tests_into_filesystem(host.filesystem) # Write out the fake builder bot results. expectation_factory = FakeBotTestExpectationsFactory() expectation_factory._all_results_by_builder = {} self.assertFalse(host.filesystem.isfile(test_expectation_path)) return_code = main(host, expectation_factory, []) self.assertEqual(return_code, 1) self.assertLog([ "WARNING: Didn't find generic expectations file at: %s\n" % test_expectation_path ]) self.assertFalse(host.filesystem.isfile(test_expectation_path))
def test_harness_no_expectations(self): """Tests behavior when TestExpectations file doesn't exist. Tests that a warning is outputted if the TestExpectations file doesn't exist. """ # Set up the mock host and port. host = MockHost() host.port_factory = FakePortFactory(host) # Write the test file but not the TestExpectations file. test_expectation_path = ( host.port_factory.get().path_to_generic_test_expectations_file()) host.filesystem = MockFileSystem() self._write_tests_into_filesystem(host.filesystem) # Write out the fake builder bot results. expectation_factory = FakeBotTestExpectationsFactory() expectation_factory._all_results_by_builder = {} self.assertFalse(host.filesystem.isfile(test_expectation_path)) return_code = main(host, expectation_factory, []) self.assertEqual(return_code, 1) self.assertLog([ "WARNING: Didn't find generic expectations file at: %s\n" % test_expectation_path ]) self.assertFalse(host.filesystem.isfile(test_expectation_path))
def test_harness_updates_file(self): """Tests that the call harness updates the TestExpectations file.""" self._define_builders({ "WebKit Linux Precise": { "port_name": "linux-precise", "specifiers": ['Precise', 'Release'] }, "WebKit Linux Precise (dbg)": { "port_name": "linux-precise", "specifiers": ['Precise', 'Debug'] }, }) # Setup the mock host and port. host = MockHost() host.port_factory = FakePortFactory(host) host.port_factory._all_build_types = ('release', 'debug') host.port_factory._all_systems = (('precise', 'x86_64'), ) # Write out a fake TestExpectations file. test_expectation_path = ( host.port_factory.get().path_to_generic_test_expectations_file()) test_expectations = """ # Remove since passing on both bots. Bug(test) [ Linux ] test/a.html [ Failure Pass ] # Keep since there's a failure on release bot. Bug(test) [ Linux Release ] test/b.html [ Failure Pass ] # Remove since it's passing on both builders. Bug(test) test/c.html [ Failure Pass ] # Keep since there's a failure on debug bot. Bug(test) [ Linux ] test/d.html [ Failure ]""" files = {test_expectation_path: test_expectations} host.filesystem = MockFileSystem(files) self._write_tests_into_filesystem(host.filesystem) # Write out the fake builder bot results. expectation_factory = FakeBotTestExpectationsFactory() expectation_factory._all_results_by_builder = { 'WebKit Linux Precise': { "test/a.html": ["PASS", "PASS", "PASS"], "test/b.html": ["PASS", "IMAGE", "PASS"], "test/c.html": ["PASS", "PASS", "PASS"], "test/d.html": ["PASS", "PASS", "PASS"], }, 'WebKit Linux Precise (dbg)': { "test/a.html": ["PASS", "PASS", "PASS"], "test/b.html": ["PASS", "PASS", "PASS"], "test/c.html": ["PASS", "PASS", "PASS"], "test/d.html": ["IMAGE", "PASS", "PASS"], }, } main(host, expectation_factory, []) self.assertEqual( host.filesystem.files[test_expectation_path], (""" # Keep since there's a failure on release bot. Bug(test) [ Linux Release ] test/b.html [ Failure Pass ] # Keep since there's a failure on debug bot. Bug(test) [ Linux ] test/d.html [ Failure ]"""))
def test_harness_updates_file(self): """Tests that the call harness updates the TestExpectations file.""" self._define_builders({ "WebKit Linux Trusty": { "port_name": "linux-trusty", "specifiers": ['Trusty', 'Release'] }, "WebKit Linux Trusty (dbg)": { "port_name": "linux-trusty", "specifiers": ['Trusty', 'Debug'] }, }) # Setup the mock host and port. host = MockHost() host.port_factory = FakePortFactory( host, all_build_types=('release', 'debug'), all_systems=(('trusty', 'x86_64'),)) # Write out a fake TestExpectations file. test_expectation_path = ( host.port_factory.get().path_to_generic_test_expectations_file()) test_expectations = """ # Remove since passing on both bots. Bug(test) [ Linux ] test/a.html [ Failure Pass ] # Keep since there's a failure on release bot. Bug(test) [ Linux Release ] test/b.html [ Failure Pass ] # Remove since it's passing on both builders. Bug(test) test/c.html [ Failure Pass ] # Keep since there's a failure on debug bot. Bug(test) [ Linux ] test/d.html [ Failure ]""" files = { test_expectation_path: test_expectations } host.filesystem = MockFileSystem(files) self._write_tests_into_filesystem(host.filesystem) # Write out the fake builder bot results. expectation_factory = FakeBotTestExpectationsFactory() expectation_factory._all_results_by_builder = { 'WebKit Linux Trusty': { "test/a.html": ["PASS", "PASS", "PASS"], "test/b.html": ["PASS", "IMAGE", "PASS"], "test/c.html": ["PASS", "PASS", "PASS"], "test/d.html": ["PASS", "PASS", "PASS"], }, 'WebKit Linux Trusty (dbg)': { "test/a.html": ["PASS", "PASS", "PASS"], "test/b.html": ["PASS", "PASS", "PASS"], "test/c.html": ["PASS", "PASS", "PASS"], "test/d.html": ["IMAGE", "PASS", "PASS"], }, } main(host, expectation_factory, []) self.assertEqual(host.filesystem.files[test_expectation_path], ( """ # Keep since there's a failure on release bot. Bug(test) [ Linux Release ] test/b.html [ Failure Pass ] # Keep since there's a failure on debug bot. Bug(test) [ Linux ] test/d.html [ Failure ]"""))