def mock_host(self):
"""Returns a mock host with fake values set up for testing."""
host = MockHost()
host.port_factory = MockPortFactory(host)
# Set up a fake list of try builders.
host.builders = BuilderList({
'MOCK Try Mac10.10': {
'port_name': 'test-mac-mac10.10',
'specifiers': ['Mac10.10', 'Release'],
'is_try_builder': True,
},
'MOCK Try Mac10.11': {
'port_name': 'test-mac-mac10.11',
'specifiers': ['Mac10.11', 'Release'],
'is_try_builder': True,
},
'MOCK Try Trusty': {
'port_name': 'test-linux-trusty',
'specifiers': ['Trusty', 'Release'],
'is_try_builder': True,
},
'MOCK Try Precise': {
'port_name': 'test-linux-precise',
'specifiers': ['Precise', 'Release'],
'is_try_builder': True,
},
'MOCK Try Win10': {
'port_name': 'test-win-win10',
'specifiers': ['Win10', 'Release'],
'is_try_builder': True,
},
'MOCK Try Win7': {
'port_name': 'test-win-win7',
'specifiers': ['Win7', 'Release'],
'is_try_builder': True,
},
})
# Write a dummy manifest file, describing what tests exist.
host.filesystem.write_text_file(
host.port_factory.get().layout_tests_dir() +
'/external/wpt/MANIFEST.json',
json.dumps({
'items': {
'reftest': {
'reftest.html':
[['/reftest.html', [['/reftest-ref.html', '==']], {}]]
},
'testharness': {
'test/path.html': [['/test/path.html', {}]],
'test/zzzz.html': [['/test/zzzz.html', {}]],
},
'manual': {
'x-manual.html': [['/x-manual.html', {}]],
},
},
}))
return host
def test_harness_no_expectations(self):
"""Tests behavior when TestExpectations file doesn't exist.
Tests that a warning is outputted if the TestExpectations file
doesn't exist.
"""
# Set up the mock host and port.
host = MockHost()
host.port_factory = FakePortFactory(host)
# Write the test file but not the TestExpectations file.
test_expectation_path = (
host.port_factory.get().path_to_generic_test_expectations_file())
host.filesystem = MockFileSystem()
self._write_tests_into_filesystem(host.filesystem)
# Write out the fake builder bot results.
expectation_factory = FakeBotTestExpectationsFactory()
expectation_factory._all_results_by_builder = {}
self.assertFalse(host.filesystem.isfile(test_expectation_path))
return_code = main(host, expectation_factory, [])
self.assertEqual(return_code, 1)
self.assertLog([
"WARNING: Didn't find generic expectations file at: %s\n" % test_expectation_path
])
self.assertFalse(host.filesystem.isfile(test_expectation_path))
def test_harness_no_expectations(self):
"""Tests behavior when TestExpectations file doesn't exist.
Tests that a warning is outputted if the TestExpectations file
doesn't exist.
"""
# Setup the mock host and port.
host = MockHost()
host.port_factory = FakePortFactory(host)
# Write the test file but not the TestExpectations file.
test_expectation_path = (
host.port_factory.get().path_to_generic_test_expectations_file())
host.filesystem = MockFileSystem()
self._write_tests_into_filesystem(host.filesystem)
# Write out the fake builder bot results.
expectation_factory = FakeBotTestExpectationsFactory()
expectation_factory._all_results_by_builder = {}
self.assertFalse(host.filesystem.isfile(test_expectation_path))
return_code = main(host, expectation_factory, [])
self.assertEqual(return_code, 1)
self.assertLog([
"WARNING: Didn't find generic expectations file at: %s\n" %
test_expectation_path
])
self.assertFalse(host.filesystem.isfile(test_expectation_path))
def test_all_configurations(self):
class FakePort(object):
def __init__(self, host, name, path):
self.host = host
self.name = name
self.path = path
def path_to_test_expectations_file(self):
return self.path
def test_configuration(self):
return None
def expectations_dict(self):
self.host.ports_parsed.append(self.name)
return {self.path: ''}
def skipped_layout_tests(self, tests):
return set([])
def all_test_configurations(self):
return []
def configuration_specifier_macros(self):
return []
def path_from_webkit_base(self):
return ''
def get_option(self, name, val):
return val
class FakeFactory(object):
def __init__(self, host, ports):
self.host = host
self.ports = {}
for port in ports:
self.ports[port.name] = port
def get(self, port_name, *args, **kwargs):
return self.ports[port_name]
def all_port_names(self):
return sorted(self.ports.keys())
host = MockHost()
host.ports_parsed = []
host.port_factory = FakeFactory(host, (FakePort(host, 'a', 'path-to-a'),
FakePort(host, 'b', 'path-to-b'),
FakePort(host, 'b-win', 'path-to-b')))
self.assertEquals(run_webkit_tests.lint(host.port_factory.ports['a'], MockOptions(platform=None)), 0)
self.assertEquals(host.ports_parsed, ['a', 'b'])
host.ports_parsed = []
self.assertEquals(run_webkit_tests.lint(host.port_factory.ports['a'], MockOptions(platform='a')), 0)
self.assertEquals(host.ports_parsed, ['a'])
def mock_host(self):
super(WPTExpectationsUpdaterTest, self).setUp()
host = MockHost()
host.port_factory = MockPortFactory(host)
host.builders = BuilderList({
'MOCK Try Mac10.10': {
'port_name': 'test-mac-mac10.10',
'specifiers': ['Mac10.10', 'Release'],
'is_try_builder': True,
},
'MOCK Try Mac10.11': {
'port_name': 'test-mac-mac10.11',
'specifiers': ['Mac10.11', 'Release'],
'is_try_builder': True,
},
'MOCK Try Trusty': {
'port_name': 'test-linux-trusty',
'specifiers': ['Trusty', 'Release'],
'is_try_builder': True,
},
'MOCK Try Precise': {
'port_name': 'test-linux-precise',
'specifiers': ['Precise', 'Release'],
'is_try_builder': True,
},
'MOCK Try Win10': {
'port_name': 'test-win-win10',
'specifiers': ['Win10', 'Release'],
'is_try_builder': True,
},
'MOCK Try Win7': {
'port_name': 'test-win-win7',
'specifiers': ['Win7', 'Release'],
'is_try_builder': True,
},
})
host.filesystem.write_text_file(
host.port_factory.get().layout_tests_dir() +
'/external/wpt/MANIFEST.json',
json.dumps({
'items': {
'reftest': {
'reftest.html':
[['/reftest.html', [['/reftest-ref.html', '==']], {}]]
},
'testharness': {
'test/path.html': [['/test/path.html', {}]],
'test/zzzz.html': [['/test/zzzz.html', {}]],
},
'manual': {
'x-manual.html': [['/x-manual.html', {}]],
},
},
}))
return host
def test_all_configurations(self):
host = MockHost()
host.ports_parsed = []
host.port_factory = FakeFactory(host, (FakePort(host, 'a', 'path-to-a'),
FakePort(host, 'b', 'path-to-b'),
FakePort(host, 'b-win', 'path-to-b')))
logging_stream = StringIO.StringIO()
options = optparse.Values({'platform': None})
res = lint_test_expectations.lint(host, options, logging_stream)
self.assertEqual(res, 0)
self.assertEqual(host.ports_parsed, ['a', 'b', 'b-win'])
def test_all_configurations(self):
host = MockHost()
host.ports_parsed = []
host.port_factory = FakeFactory(host, (FakePort(host, 'a', 'path-to-a'),
FakePort(host, 'b', 'path-to-b'),
FakePort(host, 'b-win', 'path-to-b')))
logging_stream = StringIO.StringIO()
options = optparse.Values({'platform': None})
res = lint_test_expectations.lint(host, options, logging_stream)
self.assertEqual(res, 0)
self.assertEqual(host.ports_parsed, ['a', 'b', 'b-win'])
def test_harness_remove_all(self):
"""Tests that removing all expectations doesn't delete the file.
Make sure we're prepared for the day when we exterminated flakes.
"""
self._define_builders({
"WebKit Linux Trusty": {
"port_name": "linux-trusty",
"specifiers": ['Trusty', 'Release']
},
"WebKit Linux Trusty (dbg)": {
"port_name": "linux-trusty",
"specifiers": ['Trusty', 'Debug']
},
})
# Set up the mock host and port.
host = MockHost()
host.port_factory = FakePortFactory(
host,
all_build_types=('release', 'debug'),
all_systems=(('trusty', 'x86_64'),))
# Write out a fake TestExpectations file.
test_expectation_path = (
host.port_factory.get().path_to_generic_test_expectations_file())
test_expectations = """
# Remove since passing on both bots.
Bug(test) [ Linux ] test/a.html [ Failure Pass ]"""
files = {
test_expectation_path: test_expectations
}
host.filesystem = MockFileSystem(files)
self._write_tests_into_filesystem(host.filesystem)
# Write out the fake builder bot results.
expectation_factory = FakeBotTestExpectationsFactory()
expectation_factory._all_results_by_builder = {
'WebKit Linux Trusty': {
"test/a.html": ["PASS", "PASS", "PASS"],
},
'WebKit Linux Trusty (dbg)': {
"test/a.html": ["PASS", "PASS", "PASS"],
},
}
main(host, expectation_factory, [])
self.assertTrue(host.filesystem.isfile(test_expectation_path))
self.assertEqual(host.filesystem.files[test_expectation_path], '')
def test_harness_remove_all(self):
"""Tests that removing all expectations doesn't delete the file.
Make sure we're prepared for the day when we exterminated flakes.
"""
self._define_builders({
"WebKit Linux Trusty": {
"port_name": "linux-trusty",
"specifiers": ['Trusty', 'Release']
},
"WebKit Linux Trusty (dbg)": {
"port_name": "linux-trusty",
"specifiers": ['Trusty', 'Debug']
},
})
# Setup the mock host and port.
host = MockHost()
host.port_factory = FakePortFactory(
host,
all_build_types=('release', 'debug'),
all_systems=(('trusty', 'x86_64'),))
# Write out a fake TestExpectations file.
test_expectation_path = (
host.port_factory.get().path_to_generic_test_expectations_file())
test_expectations = """
# Remove since passing on both bots.
Bug(test) [ Linux ] test/a.html [ Failure Pass ]"""
files = {
test_expectation_path: test_expectations
}
host.filesystem = MockFileSystem(files)
self._write_tests_into_filesystem(host.filesystem)
# Write out the fake builder bot results.
expectation_factory = FakeBotTestExpectationsFactory()
expectation_factory._all_results_by_builder = {
'WebKit Linux Trusty': {
"test/a.html": ["PASS", "PASS", "PASS"],
},
'WebKit Linux Trusty (dbg)': {
"test/a.html": ["PASS", "PASS", "PASS"],
},
}
main(host, expectation_factory, [])
self.assertTrue(host.filesystem.isfile(test_expectation_path))
self.assertEqual(host.filesystem.files[test_expectation_path], '')
def test_all_configurations(self):
host = MockHost()
host.ports_parsed = []
host.port_factory = FakeFactory(
host,
(FakePort(host, "a", "path-to-a"), FakePort(host, "b", "path-to-b"), FakePort(host, "b-win", "path-to-b")),
)
logging_stream = StringIO.StringIO()
options = optparse.Values({"platform": None})
res = lint_test_expectations.lint(host, options, logging_stream)
self.assertEqual(res, 0)
self.assertEqual(host.ports_parsed, ["a", "b", "b-win"])
def test_all_configurations(self):
class FakePort(object):
def __init__(self, name, path):
self.name = name
self.path = path
def test_expectations(self):
return ''
def path_to_test_expectations_file(self):
return self.path
def test_configuration(self):
return None
def test_expectations_overrides(self):
return None
class FakeFactory(object):
def __init__(self, host, ports):
self.host = host
self.ports = {}
for port in ports:
self.ports[port.name] = port
port.host = host
port.factory = self
def get(self, port_name, *args, **kwargs):
return self.ports[port_name]
def all_port_names(self):
return sorted(self.ports.keys())
class FakeExpectationsParser(object):
def __init__(self, port, *args, **kwargs):
port.host.ports_parsed.append(port.name)
host = MockHost()
host.ports_parsed = []
host.port_factory = FakeFactory(host, (FakePort('a', 'path-to-a'),
FakePort('b', 'path-to-b'),
FakePort('b-win', 'path-to-b')))
self.assertEquals(run_webkit_tests.lint(host.port_factory.ports['a'], MockOptions(platform=None), FakeExpectationsParser), 0)
self.assertEquals(host.ports_parsed, ['a', 'b'])
host.ports_parsed = []
self.assertEquals(run_webkit_tests.lint(host.port_factory.ports['a'], MockOptions(platform='a'), FakeExpectationsParser), 0)
self.assertEquals(host.ports_parsed, ['a'])
def test_all_configurations(self):
host = MockHost()
host.ports_parsed = []
host.port_factory = FakeFactory(host, (FakePort(host, 'a', 'path-to-a'),
FakePort(host, 'b', 'path-to-b'),
FakePort(host, 'b-win', 'path-to-b')))
logging_stream = StringIO.StringIO()
options = optparse.Values({'platform': None})
logger, handler = lint_test_expectations.set_up_logging(logging_stream)
try:
res = lint_test_expectations.lint(host, options)
finally:
lint_test_expectations.tear_down_logging(logger, handler)
self.assertEqual(res, 0)
self.assertEqual(host.ports_parsed, ['a', 'b', 'b-win'])
def test_all_configurations(self):
host = MockHost()
host.ports_parsed = []
host.port_factory = FakeFactory(host, (FakePort(host, 'a', 'path-to-a'),
FakePort(host, 'b', 'path-to-b'),
FakePort(host, 'b-win', 'path-to-b')))
logging_stream = StringIO.StringIO()
options = optparse.Values({'platform': None})
logger, handler = lint_test_expectations.set_up_logging(logging_stream)
try:
res = lint_test_expectations.lint(host, options)
finally:
lint_test_expectations.tear_down_logging(logger, handler)
self.assertEqual(res, 0)
self.assertEqual(host.ports_parsed, ['a', 'b', 'b-win'])
def test_all_configurations(self):
host = MockHost()
host.ports_parsed = []
host.port_factory = FakeFactory(
host,
(FakePort(host, "a", "path-to-a"), FakePort(host, "b", "path-to-b"), FakePort(host, "b-win", "path-to-b")),
)
logging_stream = StringIO.StringIO()
options = optparse.Values({"platform": None})
logger, handler = lint_test_expectations.set_up_logging(logging_stream)
try:
res = lint_test_expectations.lint(host, options)
finally:
lint_test_expectations.tear_down_logging(logger, handler)
self.assertEqual(res, [])
self.assertEqual(host.ports_parsed, ["a", "b", "b-win"])
def mock_host(self):
super(WPTExpectationsUpdaterTest, self).setUp()
host = MockHost()
host.port_factory = MockPortFactory(host)
host.builders = BuilderList({
'MOCK Try Mac10.10': {
'port_name': 'test-mac-mac10.10',
'specifiers': ['Mac10.10', 'Release'],
'is_try_builder': True,
},
'MOCK Try Mac10.11': {
'port_name': 'test-mac-mac10.11',
'specifiers': ['Mac10.11', 'Release'],
'is_try_builder': True,
},
'MOCK Try Trusty': {
'port_name': 'test-linux-trusty',
'specifiers': ['Trusty', 'Release'],
'is_try_builder': True,
},
'MOCK Try Precise': {
'port_name': 'test-linux-precise',
'specifiers': ['Precise', 'Release'],
'is_try_builder': True,
},
'MOCK Try Win10': {
'port_name': 'test-win-win10',
'specifiers': ['Win10', 'Release'],
'is_try_builder': True,
},
'MOCK Try Win7': {
'port_name': 'test-win-win7',
'specifiers': ['Win7', 'Release'],
'is_try_builder': True,
},
})
return host
def test_harness_updates_file(self):
"""Tests that the call harness updates the TestExpectations file."""
self._define_builders({
"WebKit Linux Precise": {
"port_name": "linux-precise",
"specifiers": ['Precise', 'Release']
},
"WebKit Linux Precise (dbg)": {
"port_name": "linux-precise",
"specifiers": ['Precise', 'Debug']
},
})
# Setup the mock host and port.
host = MockHost()
host.port_factory = FakePortFactory(host)
host.port_factory._all_build_types = ('release', 'debug')
host.port_factory._all_systems = (('precise', 'x86_64'), )
# Write out a fake TestExpectations file.
test_expectation_path = (
host.port_factory.get().path_to_generic_test_expectations_file())
test_expectations = """
# Remove since passing on both bots.
Bug(test) [ Linux ] test/a.html [ Failure Pass ]
# Keep since there's a failure on release bot.
Bug(test) [ Linux Release ] test/b.html [ Failure Pass ]
# Remove since it's passing on both builders.
Bug(test) test/c.html [ Failure Pass ]
# Keep since there's a failure on debug bot.
Bug(test) [ Linux ] test/d.html [ Failure ]"""
files = {test_expectation_path: test_expectations}
host.filesystem = MockFileSystem(files)
self._write_tests_into_filesystem(host.filesystem)
# Write out the fake builder bot results.
expectation_factory = FakeBotTestExpectationsFactory()
expectation_factory._all_results_by_builder = {
'WebKit Linux Precise': {
"test/a.html": ["PASS", "PASS", "PASS"],
"test/b.html": ["PASS", "IMAGE", "PASS"],
"test/c.html": ["PASS", "PASS", "PASS"],
"test/d.html": ["PASS", "PASS", "PASS"],
},
'WebKit Linux Precise (dbg)': {
"test/a.html": ["PASS", "PASS", "PASS"],
"test/b.html": ["PASS", "PASS", "PASS"],
"test/c.html": ["PASS", "PASS", "PASS"],
"test/d.html": ["IMAGE", "PASS", "PASS"],
},
}
main(host, expectation_factory, [])
self.assertEqual(
host.filesystem.files[test_expectation_path],
(""" # Keep since there's a failure on release bot.
Bug(test) [ Linux Release ] test/b.html [ Failure Pass ]
# Keep since there's a failure on debug bot.
Bug(test) [ Linux ] test/d.html [ Failure ]"""))
def test_harness_updates_file(self):
"""Tests that the call harness updates the TestExpectations file."""
self._define_builders({
"WebKit Linux Trusty": {
"port_name": "linux-trusty",
"specifiers": ['Trusty', 'Release']
},
"WebKit Linux Trusty (dbg)": {
"port_name": "linux-trusty",
"specifiers": ['Trusty', 'Debug']
},
})
# Setup the mock host and port.
host = MockHost()
host.port_factory = FakePortFactory(
host,
all_build_types=('release', 'debug'),
all_systems=(('trusty', 'x86_64'),))
# Write out a fake TestExpectations file.
test_expectation_path = (
host.port_factory.get().path_to_generic_test_expectations_file())
test_expectations = """
# Remove since passing on both bots.
Bug(test) [ Linux ] test/a.html [ Failure Pass ]
# Keep since there's a failure on release bot.
Bug(test) [ Linux Release ] test/b.html [ Failure Pass ]
# Remove since it's passing on both builders.
Bug(test) test/c.html [ Failure Pass ]
# Keep since there's a failure on debug bot.
Bug(test) [ Linux ] test/d.html [ Failure ]"""
files = {
test_expectation_path: test_expectations
}
host.filesystem = MockFileSystem(files)
self._write_tests_into_filesystem(host.filesystem)
# Write out the fake builder bot results.
expectation_factory = FakeBotTestExpectationsFactory()
expectation_factory._all_results_by_builder = {
'WebKit Linux Trusty': {
"test/a.html": ["PASS", "PASS", "PASS"],
"test/b.html": ["PASS", "IMAGE", "PASS"],
"test/c.html": ["PASS", "PASS", "PASS"],
"test/d.html": ["PASS", "PASS", "PASS"],
},
'WebKit Linux Trusty (dbg)': {
"test/a.html": ["PASS", "PASS", "PASS"],
"test/b.html": ["PASS", "PASS", "PASS"],
"test/c.html": ["PASS", "PASS", "PASS"],
"test/d.html": ["IMAGE", "PASS", "PASS"],
},
}
main(host, expectation_factory, [])
self.assertEqual(host.filesystem.files[test_expectation_path], (
""" # Keep since there's a failure on release bot.
Bug(test) [ Linux Release ] test/b.html [ Failure Pass ]
# Keep since there's a failure on debug bot.
Bug(test) [ Linux ] test/d.html [ Failure ]"""))
