class FailureMapTest(unittest.TestCase):
builder1 = MockBuilder("Builder1")
builder2 = MockBuilder("Builder2")
build1a = Build(builder1, build_number=22, revision=1233, is_green=True)
build1b = Build(builder1, build_number=23, revision=1234, is_green=False)
build2a = Build(builder2, build_number=89, revision=1233, is_green=True)
build2b = Build(builder2, build_number=90, revision=1235, is_green=False)
regression_window1 = RegressionWindow(build1a,
build1b,
failing_tests=[u'test1', u'test1'])
regression_window2 = RegressionWindow(build2a,
build2b,
failing_tests=[u'test1'])
def _make_failure_map(self):
failure_map = FailureMap()
failure_map.add_regression_window(self.builder1,
self.regression_window1)
failure_map.add_regression_window(self.builder2,
self.regression_window2)
return failure_map
def test_failing_revisions(self):
failure_map = self._make_failure_map()
self.assertEqual(failure_map.failing_revisions(), [1234, 1235])
def test_new_failures(self):
failure_map = self._make_failure_map()
failure_map.filter_out_old_failures(lambda revision: False)
self.assertEqual(failure_map.failing_revisions(), [1234, 1235])
def test_new_failures_with_old_revisions(self):
failure_map = self._make_failure_map()
failure_map.filter_out_old_failures(lambda revision: revision == 1234)
self.assertEqual(failure_map.failing_revisions(), [])
def test_new_failures_with_more_old_revisions(self):
failure_map = self._make_failure_map()
failure_map.filter_out_old_failures(lambda revision: revision == 1235)
self.assertEqual(failure_map.failing_revisions(), [1234])
def test_tests_failing_for(self):
failure_map = self._make_failure_map()
self.assertEqual(failure_map.tests_failing_for(1234), [u'test1'])
def test_failing_tests(self):
failure_map = self._make_failure_map()
self.assertEqual(failure_map.failing_tests(), set([u'test1']))
def test_rebaseline(self):
# This test basically tests the path from command.execute() to command._rebaseline();
# it doesn't test that _rebaseline() actually does anything (that is tested in TestRebaselineJson.
self.test_list = {}
def rebaseline_stub(options, test_list):
self.test_list = test_list
self.command._builders_to_pull_from = lambda: [
MockBuilder('MOCK builder')
]
self.command._tests_to_update = lambda builder: [
'mock/path/to/test.html'
]
self.command._rebaseline = rebaseline_stub
self._zero_out_test_expectations()
old_exact_matches = builders._exact_matches
oc = OutputCapture()
try:
builders._exact_matches = {
"MOCK builder": {
"port_name": "test-mac-leopard",
"specifiers": set(["mock-specifier"])
},
}
oc.capture_output()
self.command.execute(
MockOptions(optimize=True,
builders=None,
suffixes=["txt"],
verbose=True), [], self.tool)
finally:
oc.restore_output()
builders._exact_matches = old_exact_matches
self.assertEquals(
self.test_list,
{'mock/path/to/test.html': {
'MOCK builder': ['txt']
}})
def test_rebaseline(self):
self.command._builders_to_pull_from = lambda: [
MockBuilder('MOCK builder')
]
self.command._tests_to_update = lambda builder: [
'mock/path/to/test.html'
]
self._zero_out_test_expectations()
old_exact_matches = builders._exact_matches
oc = OutputCapture()
try:
builders._exact_matches = {
"MOCK builder": {
"port_name": "test-mac-leopard",
"specifiers": set(["mock-specifier"])
},
}
oc.capture_output()
self.command.execute(
MockOptions(optimize=False,
builders=None,
suffixes="txt,png",
verbose=True,
move_overwritten_baselines=False,
update_expectations=True), [], self.tool)
finally:
oc.restore_output()
builders._exact_matches = old_exact_matches
calls = list(
filter(
lambda x: x[
0] not in ['perl', '/usr/bin/xcrun', '/usr/bin/ulimit'],
self.tool.executive.calls))
self.assertEqual(calls, [[[
'echo', 'rebaseline-test-internal', '--suffixes', 'txt,png',
'--builder', 'MOCK builder', '--test', 'mock/path/to/test.html',
'--update-expectations', 'True', '--verbose'
]]])
def test_rebaseline(self):
self.command._builders_to_pull_from = lambda: [MockBuilder('MOCK builder')]
self._write("userscripts/first-test.html", "test data")
self._zero_out_test_expectations()
self._setup_mock_builder_data()
old_exact_matches = builders._exact_matches
try:
builders._exact_matches = {
"MOCK builder": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])},
}
self.command.execute(MockOptions(results_directory=False, optimize=False, builders=None, suffixes="txt,png", verbose=True), ['userscripts/first-test.html'], self.tool)
finally:
builders._exact_matches = old_exact_matches
calls = filter(lambda x: x != ['qmake', '-v'] and x[0] != 'perl', self.tool.executive.calls)
self.assertEqual(calls,
[[['echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.html', '--verbose']],
[['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.html', '--verbose']]])
def test_rebaseline_directory(self):
self.command._builders_to_pull_from = lambda: [
MockBuilder('MOCK builder')
]
self.command._tests_to_update = lambda builder: ['userscripts']
self._write("userscripts/first-test.html", "test data")
self._write("userscripts/second-test.html", "test data")
old_exact_matches = builders._exact_matches
try:
builders._exact_matches = {
"MOCK builder": {
"port_name": "test-mac-leopard",
"specifiers": set(["mock-specifier"])
},
}
self.command.execute(
MockOptions(optimize=False,
builders=None,
suffixes="txt,png",
verbose=True,
move_overwritten_baselines=False), [], self.tool)
finally:
builders._exact_matches = old_exact_matches
calls = filter(lambda x: x != ['qmake', '-v'] and x[0] != 'perl',
self.tool.executive.calls)
self.assertEqual(
calls, [[[
'echo', 'rebaseline-test-internal', '--suffixes', 'txt,png',
'--builder', 'MOCK builder', '--test',
'userscripts/first-test.html', '--verbose'
],
[
'echo', 'rebaseline-test-internal', '--suffixes',
'txt,png', '--builder', 'MOCK builder', '--test',
'userscripts/second-test.html', '--verbose'
]]])
def mock_builders_to_pull_from():
return [
MockBuilder('MOCK builder'),
MockBuilder('MOCK builder2')
]