def test_harness_remove_all(self):
"""Tests that removing all expectations doesn't delete the file.
Make sure we're prepared for the day when we exterminated flakes.
"""
self._define_builders({
"WebKit Linux Trusty": {
"port_name": "linux-trusty",
"specifiers": ['Trusty', 'Release']
},
"WebKit Linux Trusty (dbg)": {
"port_name": "linux-trusty",
"specifiers": ['Trusty', 'Debug']
},
})
# Set up the mock host and port.
host = MockHost()
host.port_factory = FakePortFactory(
host,
all_build_types=('release', 'debug'),
all_systems=(('trusty', 'x86_64'),))
# Write out a fake TestExpectations file.
test_expectation_path = (
host.port_factory.get().path_to_generic_test_expectations_file())
test_expectations = """
# Remove since passing on both bots.
Bug(test) [ Linux ] test/a.html [ Failure Pass ]"""
files = {
test_expectation_path: test_expectations
}
host.filesystem = MockFileSystem(files)
self._write_tests_into_filesystem(host.filesystem)
# Write out the fake builder bot results.
expectation_factory = FakeBotTestExpectationsFactory()
expectation_factory._all_results_by_builder = {
'WebKit Linux Trusty': {
"test/a.html": ["PASS", "PASS", "PASS"],
},
'WebKit Linux Trusty (dbg)': {
"test/a.html": ["PASS", "PASS", "PASS"],
},
}
main(host, expectation_factory, [])
self.assertTrue(host.filesystem.isfile(test_expectation_path))
self.assertEqual(host.filesystem.files[test_expectation_path], '')
def test_harness_remove_all(self):
"""Tests that removing all expectations doesn't delete the file.
Make sure we're prepared for the day when we exterminated flakes.
"""
self._define_builders({
"WebKit Linux Trusty": {
"port_name": "linux-trusty",
"specifiers": ['Trusty', 'Release']
},
"WebKit Linux Trusty (dbg)": {
"port_name": "linux-trusty",
"specifiers": ['Trusty', 'Debug']
},
})
# Setup the mock host and port.
host = MockHost()
host.port_factory = FakePortFactory(
host,
all_build_types=('release', 'debug'),
all_systems=(('trusty', 'x86_64'),))
# Write out a fake TestExpectations file.
test_expectation_path = (
host.port_factory.get().path_to_generic_test_expectations_file())
test_expectations = """
# Remove since passing on both bots.
Bug(test) [ Linux ] test/a.html [ Failure Pass ]"""
files = {
test_expectation_path: test_expectations
}
host.filesystem = MockFileSystem(files)
self._write_tests_into_filesystem(host.filesystem)
# Write out the fake builder bot results.
expectation_factory = FakeBotTestExpectationsFactory()
expectation_factory._all_results_by_builder = {
'WebKit Linux Trusty': {
"test/a.html": ["PASS", "PASS", "PASS"],
},
'WebKit Linux Trusty (dbg)': {
"test/a.html": ["PASS", "PASS", "PASS"],
},
}
main(host, expectation_factory, [])
self.assertTrue(host.filesystem.isfile(test_expectation_path))
self.assertEqual(host.filesystem.files[test_expectation_path], '')
def test_harness_no_expectations(self):
"""Tests behavior when TestExpectations file doesn't exist.
Tests that a warning is outputted if the TestExpectations file
doesn't exist.
"""
# Setup the mock host and port.
host = MockHost()
host.port_factory = FakePortFactory(host)
# Write the test file but not the TestExpectations file.
test_expectation_path = (
host.port_factory.get().path_to_generic_test_expectations_file())
host.filesystem = MockFileSystem()
self._write_tests_into_filesystem(host.filesystem)
# Write out the fake builder bot results.
expectation_factory = FakeBotTestExpectationsFactory()
expectation_factory._all_results_by_builder = {}
self.assertFalse(host.filesystem.isfile(test_expectation_path))
return_code = main(host, expectation_factory, [])
self.assertEqual(return_code, 1)
self.assertLog([
"WARNING: Didn't find generic expectations file at: %s\n" %
test_expectation_path
])
self.assertFalse(host.filesystem.isfile(test_expectation_path))
def test_harness_no_expectations(self):
"""Tests behavior when TestExpectations file doesn't exist.
Tests that a warning is outputted if the TestExpectations file
doesn't exist.
"""
# Set up the mock host and port.
host = MockHost()
host.port_factory = FakePortFactory(host)
# Write the test file but not the TestExpectations file.
test_expectation_path = (
host.port_factory.get().path_to_generic_test_expectations_file())
host.filesystem = MockFileSystem()
self._write_tests_into_filesystem(host.filesystem)
# Write out the fake builder bot results.
expectation_factory = FakeBotTestExpectationsFactory()
expectation_factory._all_results_by_builder = {}
self.assertFalse(host.filesystem.isfile(test_expectation_path))
return_code = main(host, expectation_factory, [])
self.assertEqual(return_code, 1)
self.assertLog([
"WARNING: Didn't find generic expectations file at: %s\n" % test_expectation_path
])
self.assertFalse(host.filesystem.isfile(test_expectation_path))
def test_harness_updates_file(self):
"""Tests that the call harness updates the TestExpectations file."""
self._define_builders({
"WebKit Linux Precise": {
"port_name": "linux-precise",
"specifiers": ['Precise', 'Release']
},
"WebKit Linux Precise (dbg)": {
"port_name": "linux-precise",
"specifiers": ['Precise', 'Debug']
},
})
# Setup the mock host and port.
host = MockHost()
host.port_factory = FakePortFactory(host)
host.port_factory._all_build_types = ('release', 'debug')
host.port_factory._all_systems = (('precise', 'x86_64'), )
# Write out a fake TestExpectations file.
test_expectation_path = (
host.port_factory.get().path_to_generic_test_expectations_file())
test_expectations = """
# Remove since passing on both bots.
Bug(test) [ Linux ] test/a.html [ Failure Pass ]
# Keep since there's a failure on release bot.
Bug(test) [ Linux Release ] test/b.html [ Failure Pass ]
# Remove since it's passing on both builders.
Bug(test) test/c.html [ Failure Pass ]
# Keep since there's a failure on debug bot.
Bug(test) [ Linux ] test/d.html [ Failure ]"""
files = {test_expectation_path: test_expectations}
host.filesystem = MockFileSystem(files)
self._write_tests_into_filesystem(host.filesystem)
# Write out the fake builder bot results.
expectation_factory = FakeBotTestExpectationsFactory()
expectation_factory._all_results_by_builder = {
'WebKit Linux Precise': {
"test/a.html": ["PASS", "PASS", "PASS"],
"test/b.html": ["PASS", "IMAGE", "PASS"],
"test/c.html": ["PASS", "PASS", "PASS"],
"test/d.html": ["PASS", "PASS", "PASS"],
},
'WebKit Linux Precise (dbg)': {
"test/a.html": ["PASS", "PASS", "PASS"],
"test/b.html": ["PASS", "PASS", "PASS"],
"test/c.html": ["PASS", "PASS", "PASS"],
"test/d.html": ["IMAGE", "PASS", "PASS"],
},
}
main(host, expectation_factory, [])
self.assertEqual(
host.filesystem.files[test_expectation_path],
(""" # Keep since there's a failure on release bot.
Bug(test) [ Linux Release ] test/b.html [ Failure Pass ]
# Keep since there's a failure on debug bot.
Bug(test) [ Linux ] test/d.html [ Failure ]"""))
def test_harness_updates_file(self):
"""Tests that the call harness updates the TestExpectations file."""
self._define_builders({
"WebKit Linux Trusty": {
"port_name": "linux-trusty",
"specifiers": ['Trusty', 'Release']
},
"WebKit Linux Trusty (dbg)": {
"port_name": "linux-trusty",
"specifiers": ['Trusty', 'Debug']
},
})
# Setup the mock host and port.
host = MockHost()
host.port_factory = FakePortFactory(
host,
all_build_types=('release', 'debug'),
all_systems=(('trusty', 'x86_64'),))
# Write out a fake TestExpectations file.
test_expectation_path = (
host.port_factory.get().path_to_generic_test_expectations_file())
test_expectations = """
# Remove since passing on both bots.
Bug(test) [ Linux ] test/a.html [ Failure Pass ]
# Keep since there's a failure on release bot.
Bug(test) [ Linux Release ] test/b.html [ Failure Pass ]
# Remove since it's passing on both builders.
Bug(test) test/c.html [ Failure Pass ]
# Keep since there's a failure on debug bot.
Bug(test) [ Linux ] test/d.html [ Failure ]"""
files = {
test_expectation_path: test_expectations
}
host.filesystem = MockFileSystem(files)
self._write_tests_into_filesystem(host.filesystem)
# Write out the fake builder bot results.
expectation_factory = FakeBotTestExpectationsFactory()
expectation_factory._all_results_by_builder = {
'WebKit Linux Trusty': {
"test/a.html": ["PASS", "PASS", "PASS"],
"test/b.html": ["PASS", "IMAGE", "PASS"],
"test/c.html": ["PASS", "PASS", "PASS"],
"test/d.html": ["PASS", "PASS", "PASS"],
},
'WebKit Linux Trusty (dbg)': {
"test/a.html": ["PASS", "PASS", "PASS"],
"test/b.html": ["PASS", "PASS", "PASS"],
"test/c.html": ["PASS", "PASS", "PASS"],
"test/d.html": ["IMAGE", "PASS", "PASS"],
},
}
main(host, expectation_factory, [])
self.assertEqual(host.filesystem.files[test_expectation_path], (
""" # Keep since there's a failure on release bot.
Bug(test) [ Linux Release ] test/b.html [ Failure Pass ]
# Keep since there's a failure on debug bot.
Bug(test) [ Linux ] test/d.html [ Failure ]"""))