class BuilderTest(unittest.TestCase):
def _mock_test_result(self, testname):
return test_results.TestResult(testname,
[test_failures.FailureTextMismatch()])
def _install_fetch_build(self, failure):
def _mock_fetch_build(build_number):
build = Build(builder=self.builder,
build_number=build_number,
revision=build_number + 1000,
is_green=build_number < 4)
return build
self.builder._fetch_build = _mock_fetch_build
def setUp(self):
self.buildbot = BuildBot()
self.builder = Builder(u"Test Builder \u2661", self.buildbot)
self._install_fetch_build(lambda build_number: ["test1", "test2"])
def test_latest_layout_test_results(self):
self.builder.fetch_layout_test_results = lambda results_url: LayoutTestResults(
None)
self.builder.accumulated_results_url = lambda: "http://dummy_url.org"
self.assertTrue(self.builder.latest_layout_test_results())
def test_build_caching(self):
self.assertEqual(self.builder.build(10), self.builder.build(10))
def test_build_and_revision_for_filename(self):
expectations = {
"r47483 (1)/": (47483, 1),
"r47483 (1).zip": (47483, 1),
"random junk": None,
}
for filename, revision_and_build in expectations.items():
self.assertEqual(
self.builder._revision_and_build_for_filename(filename),
revision_and_build)
def test_file_info_list_to_revision_to_build_list(self):
file_info_list = [
{
"filename": "r47483 (1)/"
},
{
"filename": "r47483 (1).zip"
},
{
"filename": "random junk"
},
]
builds_and_revisions_list = [(47483, 1), (47483, 1)]
self.assertEqual(
self.builder._file_info_list_to_revision_to_build_list(
file_info_list), builds_and_revisions_list)
def test_fetch_build(self):
buildbot = BuildBot()
builder = Builder(u"Test Builder \u2661", buildbot)
def mock_fetch_build_dictionary(self, build_number):
build_dictionary = {
"sourceStamp": {
"revision":
None, # revision=None means a trunk build started from the force-build button on the builder page.
},
"number": int(build_number),
# Intentionally missing the 'results' key, meaning it's a "pass" build.
}
return build_dictionary
buildbot._fetch_build_dictionary = mock_fetch_build_dictionary
self.assertIsNotNone(builder._fetch_build(1))
def test_results_url(self):
builder = BuildBot().builder_with_name('WebKit Mac10.8 (dbg)')
self.assertEqual(
builder.results_url(),
'https://storage.googleapis.com/chromium-layout-test-archives/WebKit_Mac10_8__dbg_'
)
def test_accumulated_results_url(self):
builder = BuildBot().builder_with_name('WebKit Mac10.8 (dbg)')
self.assertEqual(
builder.accumulated_results_url(),
'https://storage.googleapis.com/chromium-layout-test-archives/WebKit_Mac10_8__dbg_/results/layout-test-results'
)
class BuilderTest(unittest.TestCase):
def _mock_test_result(self, testname):
return test_results.TestResult(testname,
[test_failures.FailureTextMismatch()])
def _install_fetch_build(self, failure):
def _mock_fetch_build(build_number):
build = Build(builder=self.builder,
build_number=build_number,
revision=build_number + 1000,
is_green=build_number < 4)
results = [
self._mock_test_result(testname)
for testname in failure(build_number)
]
build._layout_test_results = LayoutTestResults(results)
return build
self.builder._fetch_build = _mock_fetch_build
def setUp(self):
self.buildbot = BuildBot()
self.builder = Builder(u"Test Builder \u2661", self.buildbot)
self._install_fetch_build(lambda build_number: ["test1", "test2"])
def test_find_regression_window(self):
regression_window = self.builder.find_regression_window(
self.builder.build(10))
self.assertEqual(regression_window.build_before_failure().revision(),
1003)
self.assertEqual(regression_window.failing_build().revision(), 1004)
regression_window = self.builder.find_regression_window(
self.builder.build(10), look_back_limit=2)
self.assertEqual(regression_window.build_before_failure(), None)
self.assertEqual(regression_window.failing_build().revision(), 1008)
def test_none_build(self):
self.builder._fetch_build = lambda build_number: None
regression_window = self.builder.find_regression_window(
self.builder.build(10))
self.assertEqual(regression_window.build_before_failure(), None)
self.assertEqual(regression_window.failing_build(), None)
def test_flaky_tests(self):
self._install_fetch_build(lambda build_number: ["test1"]
if build_number % 2 else ["test2"])
regression_window = self.builder.find_regression_window(
self.builder.build(10))
self.assertEqual(regression_window.build_before_failure().revision(),
1009)
self.assertEqual(regression_window.failing_build().revision(), 1010)
def test_failure_and_flaky(self):
self._install_fetch_build(lambda build_number: ["test1", "test2"]
if build_number % 2 else ["test2"])
regression_window = self.builder.find_regression_window(
self.builder.build(10))
self.assertEqual(regression_window.build_before_failure().revision(),
1003)
self.assertEqual(regression_window.failing_build().revision(), 1004)
def test_no_results(self):
self._install_fetch_build(lambda build_number: ["test1", "test2"]
if build_number % 2 else ["test2"])
regression_window = self.builder.find_regression_window(
self.builder.build(10))
self.assertEqual(regression_window.build_before_failure().revision(),
1003)
self.assertEqual(regression_window.failing_build().revision(), 1004)
def test_failure_after_flaky(self):
self._install_fetch_build(lambda build_number: ["test1", "test2"]
if build_number > 6 else ["test3"])
regression_window = self.builder.find_regression_window(
self.builder.build(10))
self.assertEqual(regression_window.build_before_failure().revision(),
1006)
self.assertEqual(regression_window.failing_build().revision(), 1007)
def test_find_blameworthy_regression_window(self):
self.assertEqual(
self.builder.find_blameworthy_regression_window(10).revisions(),
[1004])
self.assertEqual(
self.builder.find_blameworthy_regression_window(10,
look_back_limit=2),
None)
# Flakey test avoidance requires at least 2 red builds:
self.assertEqual(self.builder.find_blameworthy_regression_window(4),
None)
self.assertEqual(
self.builder.find_blameworthy_regression_window(
4, avoid_flakey_tests=False).revisions(), [1004])
# Green builder:
self.assertEqual(self.builder.find_blameworthy_regression_window(3),
None)
def test_build_caching(self):
self.assertEqual(self.builder.build(10), self.builder.build(10))
def test_build_and_revision_for_filename(self):
expectations = {
"r47483 (1)/": (47483, 1),
"r47483 (1).zip": (47483, 1),
}
for filename, revision_and_build in expectations.items():
self.assertEqual(
self.builder._revision_and_build_for_filename(filename),
revision_and_build)
class BuilderTest(unittest.TestCase):
def _mock_test_result(self, testname):
return test_results.TestResult(testname,
[test_failures.FailureTextMismatch()])
def _install_fetch_build(self, failure):
def _mock_fetch_build(build_number):
build = Build(builder=self.builder,
build_number=build_number,
revision=build_number + 1000,
is_green=build_number < 4)
results = [
self._mock_test_result(testname)
for testname in failure(build_number)
]
layout_test_results = LayoutTestResults(results)
def mock_layout_test_results():
return layout_test_results
build.layout_test_results = mock_layout_test_results
return build
self.builder._fetch_build = _mock_fetch_build
def setUp(self):
self.buildbot = BuildBot()
self.builder = Builder(u"Test Builder \u2661", self.buildbot)
self._install_fetch_build(lambda build_number: ["test1", "test2"])
def test_latest_layout_test_results(self):
self.builder.fetch_layout_test_results = lambda results_url: LayoutTestResults(
[
self._mock_test_result(testname)
for testname in ["test1", "test2"]
])
self.builder.accumulated_results_url = lambda: "http://dummy_url.org"
self.assertTrue(self.builder.latest_layout_test_results())
def test_find_regression_window(self):
regression_window = self.builder.find_regression_window(
self.builder.build(10))
self.assertEqual(regression_window.build_before_failure().revision(),
1003)
self.assertEqual(regression_window.failing_build().revision(), 1004)
regression_window = self.builder.find_regression_window(
self.builder.build(10), look_back_limit=2)
self.assertEqual(regression_window.build_before_failure(), None)
self.assertEqual(regression_window.failing_build().revision(), 1008)
def test_none_build(self):
self.builder._fetch_build = lambda build_number: None
regression_window = self.builder.find_regression_window(
self.builder.build(10))
self.assertEqual(regression_window.build_before_failure(), None)
self.assertEqual(regression_window.failing_build(), None)
def test_flaky_tests(self):
self._install_fetch_build(lambda build_number: ["test1"]
if build_number % 2 else ["test2"])
regression_window = self.builder.find_regression_window(
self.builder.build(10))
self.assertEqual(regression_window.build_before_failure().revision(),
1009)
self.assertEqual(regression_window.failing_build().revision(), 1010)
def test_failure_and_flaky(self):
self._install_fetch_build(lambda build_number: ["test1", "test2"]
if build_number % 2 else ["test2"])
regression_window = self.builder.find_regression_window(
self.builder.build(10))
self.assertEqual(regression_window.build_before_failure().revision(),
1003)
self.assertEqual(regression_window.failing_build().revision(), 1004)
def test_no_results(self):
self._install_fetch_build(lambda build_number: ["test1", "test2"]
if build_number % 2 else ["test2"])
regression_window = self.builder.find_regression_window(
self.builder.build(10))
self.assertEqual(regression_window.build_before_failure().revision(),
1003)
self.assertEqual(regression_window.failing_build().revision(), 1004)
def test_failure_after_flaky(self):
self._install_fetch_build(lambda build_number: ["test1", "test2"]
if build_number > 6 else ["test3"])
regression_window = self.builder.find_regression_window(
self.builder.build(10))
self.assertEqual(regression_window.build_before_failure().revision(),
1006)
self.assertEqual(regression_window.failing_build().revision(), 1007)
def test_find_blameworthy_regression_window(self):
self.assertEqual(
self.builder.find_blameworthy_regression_window(10).revisions(),
[1004])
self.assertEqual(
self.builder.find_blameworthy_regression_window(10,
look_back_limit=2),
None)
# Flakey test avoidance requires at least 2 red builds:
self.assertEqual(self.builder.find_blameworthy_regression_window(4),
None)
self.assertEqual(
self.builder.find_blameworthy_regression_window(
4, avoid_flakey_tests=False).revisions(), [1004])
# Green builder:
self.assertEqual(self.builder.find_blameworthy_regression_window(3),
None)
def test_build_caching(self):
self.assertEqual(self.builder.build(10), self.builder.build(10))
def test_build_and_revision_for_filename(self):
expectations = {
"r47483 (1)/": (47483, 1),
"r47483 (1).zip": (47483, 1),
"random junk": None,
}
for filename, revision_and_build in expectations.items():
self.assertEqual(
self.builder._revision_and_build_for_filename(filename),
revision_and_build)
def test_file_info_list_to_revision_to_build_list(self):
file_info_list = [
{
"filename": "r47483 (1)/"
},
{
"filename": "r47483 (1).zip"
},
{
"filename": "random junk"
},
]
builds_and_revisions_list = [(47483, 1), (47483, 1)]
self.assertEqual(
self.builder._file_info_list_to_revision_to_build_list(
file_info_list), builds_and_revisions_list)
def test_fetch_build(self):
buildbot = BuildBot()
builder = Builder(u"Test Builder \u2661", buildbot)
def mock_fetch_build_dictionary(self, build_number):
build_dictionary = {
"sourceStamp": {
"revision":
None, # revision=None means a trunk build started from the force-build button on the builder page.
},
"number": int(build_number),
# Intentionally missing the 'results' key, meaning it's a "pass" build.
}
return build_dictionary
buildbot._fetch_build_dictionary = mock_fetch_build_dictionary
self.assertNotEqual(builder._fetch_build(1), None)
class BuilderTest(unittest.TestCase):
def _mock_test_result(self, testname):
return test_results.TestResult(testname, [test_failures.FailureTextMismatch()])
def _install_fetch_build(self, failure):
def _mock_fetch_build(build_number):
build = Build(
builder=self.builder,
build_number=build_number,
revision=build_number + 1000,
is_green=build_number < 4
)
return build
self.builder._fetch_build = _mock_fetch_build
def setUp(self):
self.buildbot = BuildBot()
self.builder = Builder(u"Test Builder \u2661", self.buildbot)
self._install_fetch_build(lambda build_number: ["test1", "test2"])
def test_latest_layout_test_results(self):
self.builder.fetch_layout_test_results = lambda results_url: LayoutTestResults(None)
self.builder.accumulated_results_url = lambda: "http://dummy_url.org"
self.assertTrue(self.builder.latest_layout_test_results())
def test_build_caching(self):
self.assertEqual(self.builder.build(10), self.builder.build(10))
def test_build_and_revision_for_filename(self):
expectations = {
"r47483 (1)/" : (47483, 1),
"r47483 (1).zip" : (47483, 1),
"random junk": None,
}
for filename, revision_and_build in expectations.items():
self.assertEqual(self.builder._revision_and_build_for_filename(filename), revision_and_build)
def test_file_info_list_to_revision_to_build_list(self):
file_info_list = [
{"filename": "r47483 (1)/"},
{"filename": "r47483 (1).zip"},
{"filename": "random junk"},
]
builds_and_revisions_list = [(47483, 1), (47483, 1)]
self.assertEqual(self.builder._file_info_list_to_revision_to_build_list(file_info_list), builds_and_revisions_list)
def test_fetch_build(self):
buildbot = BuildBot()
builder = Builder(u"Test Builder \u2661", buildbot)
def mock_fetch_build_dictionary(self, build_number):
build_dictionary = {
"sourceStamp": {
"revision": None, # revision=None means a trunk build started from the force-build button on the builder page.
},
"number": int(build_number),
# Intentionally missing the 'results' key, meaning it's a "pass" build.
}
return build_dictionary
buildbot._fetch_build_dictionary = mock_fetch_build_dictionary
self.assertIsNotNone(builder._fetch_build(1))
def test_results_url(self):
builder = BuildBot().builder_with_name('WebKit Mac10.8 (dbg)')
self.assertEqual(builder.results_url(),
'https://storage.googleapis.com/chromium-layout-test-archives/WebKit_Mac10_8__dbg_')
def test_accumulated_results_url(self):
builder = BuildBot().builder_with_name('WebKit Mac10.8 (dbg)')
self.assertEqual(builder.accumulated_results_url(),
'https://storage.googleapis.com/chromium-layout-test-archives/WebKit_Mac10_8__dbg_/results/layout-test-results')
class BuilderTest(unittest.TestCase):
def _mock_test_result(self, testname):
return test_results.TestResult(testname, [test_failures.FailureTextMismatch()])
def _install_fetch_build(self, failure):
def _mock_fetch_build(build_number):
build = Build(
builder=self.builder,
build_number=build_number,
revision=build_number + 1000,
is_green=build_number < 4
)
results = [self._mock_test_result(testname) for testname in failure(build_number)]
layout_test_results = LayoutTestResults(results)
def mock_layout_test_results():
return layout_test_results
build.layout_test_results = mock_layout_test_results
return build
self.builder._fetch_build = _mock_fetch_build
def setUp(self):
self.buildbot = BuildBot()
self.builder = Builder(u"Test Builder \u2661", self.buildbot)
self._install_fetch_build(lambda build_number: ["test1", "test2"])
def test_latest_layout_test_results(self):
self.builder.fetch_layout_test_results = lambda results_url: LayoutTestResults([self._mock_test_result(testname) for testname in ["test1", "test2"]])
self.builder.accumulated_results_url = lambda: "http://dummy_url.org"
self.assertTrue(self.builder.latest_layout_test_results())
def test_find_regression_window(self):
regression_window = self.builder.find_regression_window(self.builder.build(10))
self.assertEqual(regression_window.build_before_failure().revision(), 1003)
self.assertEqual(regression_window.failing_build().revision(), 1004)
regression_window = self.builder.find_regression_window(self.builder.build(10), look_back_limit=2)
self.assertEqual(regression_window.build_before_failure(), None)
self.assertEqual(regression_window.failing_build().revision(), 1008)
def test_none_build(self):
self.builder._fetch_build = lambda build_number: None
regression_window = self.builder.find_regression_window(self.builder.build(10))
self.assertEqual(regression_window.build_before_failure(), None)
self.assertEqual(regression_window.failing_build(), None)
def test_flaky_tests(self):
self._install_fetch_build(lambda build_number: ["test1"] if build_number % 2 else ["test2"])
regression_window = self.builder.find_regression_window(self.builder.build(10))
self.assertEqual(regression_window.build_before_failure().revision(), 1009)
self.assertEqual(regression_window.failing_build().revision(), 1010)
def test_failure_and_flaky(self):
self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number % 2 else ["test2"])
regression_window = self.builder.find_regression_window(self.builder.build(10))
self.assertEqual(regression_window.build_before_failure().revision(), 1003)
self.assertEqual(regression_window.failing_build().revision(), 1004)
def test_no_results(self):
self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number % 2 else ["test2"])
regression_window = self.builder.find_regression_window(self.builder.build(10))
self.assertEqual(regression_window.build_before_failure().revision(), 1003)
self.assertEqual(regression_window.failing_build().revision(), 1004)
def test_failure_after_flaky(self):
self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number > 6 else ["test3"])
regression_window = self.builder.find_regression_window(self.builder.build(10))
self.assertEqual(regression_window.build_before_failure().revision(), 1006)
self.assertEqual(regression_window.failing_build().revision(), 1007)
def test_find_blameworthy_regression_window(self):
self.assertEqual(self.builder.find_blameworthy_regression_window(10).revisions(), [1004])
self.assertEqual(self.builder.find_blameworthy_regression_window(10, look_back_limit=2), None)
# Flakey test avoidance requires at least 2 red builds:
self.assertEqual(self.builder.find_blameworthy_regression_window(4), None)
self.assertEqual(self.builder.find_blameworthy_regression_window(4, avoid_flakey_tests=False).revisions(), [1004])
# Green builder:
self.assertEqual(self.builder.find_blameworthy_regression_window(3), None)
def test_build_caching(self):
self.assertEqual(self.builder.build(10), self.builder.build(10))
def test_build_and_revision_for_filename(self):
expectations = {
"r47483 (1)/" : (47483, 1),
"r47483 (1).zip" : (47483, 1),
"random junk": None,
}
for filename, revision_and_build in expectations.items():
self.assertEqual(self.builder._revision_and_build_for_filename(filename), revision_and_build)
def test_file_info_list_to_revision_to_build_list(self):
file_info_list = [
{"filename": "r47483 (1)/"},
{"filename": "r47483 (1).zip"},
{"filename": "random junk"},
]
builds_and_revisions_list = [(47483, 1), (47483, 1)]
self.assertEqual(self.builder._file_info_list_to_revision_to_build_list(file_info_list), builds_and_revisions_list)
def test_fetch_build(self):
buildbot = BuildBot()
builder = Builder(u"Test Builder \u2661", buildbot)
def mock_fetch_build_dictionary(self, build_number):
build_dictionary = {
"sourceStamp": {
"revision": None, # revision=None means a trunk build started from the force-build button on the builder page.
},
"number": int(build_number),
# Intentionally missing the 'results' key, meaning it's a "pass" build.
}
return build_dictionary
buildbot._fetch_build_dictionary = mock_fetch_build_dictionary
self.assertNotEqual(builder._fetch_build(1), None)
class BuilderTest(unittest.TestCase):
def _install_fetch_build(self, failure):
def _mock_fetch_build(build_number):
build = Build(
builder=self.builder,
build_number=build_number,
revision=build_number + 1000,
is_green=build_number < 4
)
build._layout_test_results = LayoutTestResults(
"http://buildbot.example.com/foo", {
LayoutTestResults.fail_key: failure(build_number),
})
return build
self.builder._fetch_build = _mock_fetch_build
def setUp(self):
self.buildbot = BuildBot()
self.builder = Builder(u"Test Builder \u2661", self.buildbot)
self._install_fetch_build(lambda build_number: ["test1", "test2"])
def test_find_failure_transition(self):
(green_build, red_build) = self.builder.find_failure_transition(self.builder.build(10))
self.assertEqual(green_build.revision(), 1003)
self.assertEqual(red_build.revision(), 1004)
(green_build, red_build) = self.builder.find_failure_transition(self.builder.build(10), look_back_limit=2)
self.assertEqual(green_build, None)
self.assertEqual(red_build.revision(), 1008)
def test_none_build(self):
self.builder._fetch_build = lambda build_number: None
(green_build, red_build) = self.builder.find_failure_transition(self.builder.build(10))
self.assertEqual(green_build, None)
self.assertEqual(red_build, None)
def test_flaky_tests(self):
self._install_fetch_build(lambda build_number: ["test1"] if build_number % 2 else ["test2"])
(green_build, red_build) = self.builder.find_failure_transition(self.builder.build(10))
self.assertEqual(green_build.revision(), 1009)
self.assertEqual(red_build.revision(), 1010)
def test_failure_and_flaky(self):
self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number % 2 else ["test2"])
(green_build, red_build) = self.builder.find_failure_transition(self.builder.build(10))
self.assertEqual(green_build.revision(), 1003)
self.assertEqual(red_build.revision(), 1004)
def test_no_results(self):
self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number % 2 else ["test2"])
(green_build, red_build) = self.builder.find_failure_transition(self.builder.build(10))
self.assertEqual(green_build.revision(), 1003)
self.assertEqual(red_build.revision(), 1004)
def test_failure_after_flaky(self):
self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number > 6 else ["test3"])
(green_build, red_build) = self.builder.find_failure_transition(self.builder.build(10))
self.assertEqual(green_build.revision(), 1006)
self.assertEqual(red_build.revision(), 1007)
def test_blameworthy_revisions(self):
self.assertEqual(self.builder.blameworthy_revisions(10), [1004])
self.assertEqual(self.builder.blameworthy_revisions(10, look_back_limit=2), [])
# Flakey test avoidance requires at least 2 red builds:
self.assertEqual(self.builder.blameworthy_revisions(4), [])
self.assertEqual(self.builder.blameworthy_revisions(4, avoid_flakey_tests=False), [1004])
# Green builder:
self.assertEqual(self.builder.blameworthy_revisions(3), [])
def test_build_caching(self):
self.assertEqual(self.builder.build(10), self.builder.build(10))
def test_build_and_revision_for_filename(self):
expectations = {
"r47483 (1)/" : (47483, 1),
"r47483 (1).zip" : (47483, 1),
}
for filename, revision_and_build in expectations.items():
self.assertEqual(self.builder._revision_and_build_for_filename(filename), revision_and_build)
class BuilderTest(unittest.TestCase):
def _install_fetch_build(self, failure):
def _mock_fetch_build(build_number):
build = Build(builder=self.builder,
build_number=build_number,
revision=build_number + 1000,
is_green=build_number < 4)
build._layout_test_results = LayoutTestResults(
"http://buildbot.example.com/foo", {
LayoutTestResults.fail_key: failure(build_number),
})
return build
self.builder._fetch_build = _mock_fetch_build
def setUp(self):
self.buildbot = BuildBot()
self.builder = Builder(u"Test Builder \u2661", self.buildbot)
self._install_fetch_build(lambda build_number: ["test1", "test2"])
def test_find_failure_transition(self):
(green_build, red_build) = self.builder.find_failure_transition(
self.builder.build(10))
self.assertEqual(green_build.revision(), 1003)
self.assertEqual(red_build.revision(), 1004)
(green_build, red_build) = self.builder.find_failure_transition(
self.builder.build(10), look_back_limit=2)
self.assertEqual(green_build, None)
self.assertEqual(red_build.revision(), 1008)
def test_none_build(self):
self.builder._fetch_build = lambda build_number: None
(green_build, red_build) = self.builder.find_failure_transition(
self.builder.build(10))
self.assertEqual(green_build, None)
self.assertEqual(red_build, None)
def test_flaky_tests(self):
self._install_fetch_build(lambda build_number: ["test1"]
if build_number % 2 else ["test2"])
(green_build, red_build) = self.builder.find_failure_transition(
self.builder.build(10))
self.assertEqual(green_build.revision(), 1009)
self.assertEqual(red_build.revision(), 1010)
def test_failure_and_flaky(self):
self._install_fetch_build(lambda build_number: ["test1", "test2"]
if build_number % 2 else ["test2"])
(green_build, red_build) = self.builder.find_failure_transition(
self.builder.build(10))
self.assertEqual(green_build.revision(), 1003)
self.assertEqual(red_build.revision(), 1004)
def test_no_results(self):
self._install_fetch_build(lambda build_number: ["test1", "test2"]
if build_number % 2 else ["test2"])
(green_build, red_build) = self.builder.find_failure_transition(
self.builder.build(10))
self.assertEqual(green_build.revision(), 1003)
self.assertEqual(red_build.revision(), 1004)
def test_failure_after_flaky(self):
self._install_fetch_build(lambda build_number: ["test1", "test2"]
if build_number > 6 else ["test3"])
(green_build, red_build) = self.builder.find_failure_transition(
self.builder.build(10))
self.assertEqual(green_build.revision(), 1006)
self.assertEqual(red_build.revision(), 1007)
def test_blameworthy_revisions(self):
self.assertEqual(self.builder.blameworthy_revisions(10), [1004])
self.assertEqual(
self.builder.blameworthy_revisions(10, look_back_limit=2), [])
# Flakey test avoidance requires at least 2 red builds:
self.assertEqual(self.builder.blameworthy_revisions(4), [])
self.assertEqual(
self.builder.blameworthy_revisions(4, avoid_flakey_tests=False),
[1004])
# Green builder:
self.assertEqual(self.builder.blameworthy_revisions(3), [])
def test_build_caching(self):
self.assertEqual(self.builder.build(10), self.builder.build(10))
def test_build_and_revision_for_filename(self):
expectations = {
"r47483 (1)/": (47483, 1),
"r47483 (1).zip": (47483, 1),
}
for filename, revision_and_build in expectations.items():
self.assertEqual(
self.builder._revision_and_build_for_filename(filename),
revision_and_build)
class BuilderTest(unittest.TestCase):
def _mock_test_result(self, testname):
return test_results.TestResult(testname, [test_failures.FailureTextMismatch()])
def _install_fetch_build(self, failure):
def _mock_fetch_build(build_number):
build = Build(
builder=self.builder,
build_number=build_number,
revision=build_number + 1000,
is_green=build_number < 4
)
results = [self._mock_test_result(testname) for testname in failure(build_number)]
build._layout_test_results = LayoutTestResults(results)
return build
self.builder._fetch_build = _mock_fetch_build
def setUp(self):
self.buildbot = BuildBot()
self.builder = Builder(u"Test Builder \u2661", self.buildbot)
self._install_fetch_build(lambda build_number: ["test1", "test2"])
def test_find_regression_window(self):
regression_window = self.builder.find_regression_window(self.builder.build(10))
self.assertEqual(regression_window.build_before_failure().revision(), 1003)
self.assertEqual(regression_window.failing_build().revision(), 1004)
regression_window = self.builder.find_regression_window(self.builder.build(10), look_back_limit=2)
self.assertEqual(regression_window.build_before_failure(), None)
self.assertEqual(regression_window.failing_build().revision(), 1008)
def test_none_build(self):
self.builder._fetch_build = lambda build_number: None
regression_window = self.builder.find_regression_window(self.builder.build(10))
self.assertEqual(regression_window.build_before_failure(), None)
self.assertEqual(regression_window.failing_build(), None)
def test_flaky_tests(self):
self._install_fetch_build(lambda build_number: ["test1"] if build_number % 2 else ["test2"])
regression_window = self.builder.find_regression_window(self.builder.build(10))
self.assertEqual(regression_window.build_before_failure().revision(), 1009)
self.assertEqual(regression_window.failing_build().revision(), 1010)
def test_failure_and_flaky(self):
self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number % 2 else ["test2"])
regression_window = self.builder.find_regression_window(self.builder.build(10))
self.assertEqual(regression_window.build_before_failure().revision(), 1003)
self.assertEqual(regression_window.failing_build().revision(), 1004)
def test_no_results(self):
self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number % 2 else ["test2"])
regression_window = self.builder.find_regression_window(self.builder.build(10))
self.assertEqual(regression_window.build_before_failure().revision(), 1003)
self.assertEqual(regression_window.failing_build().revision(), 1004)
def test_failure_after_flaky(self):
self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number > 6 else ["test3"])
regression_window = self.builder.find_regression_window(self.builder.build(10))
self.assertEqual(regression_window.build_before_failure().revision(), 1006)
self.assertEqual(regression_window.failing_build().revision(), 1007)
def test_find_blameworthy_regression_window(self):
self.assertEqual(self.builder.find_blameworthy_regression_window(10).revisions(), [1004])
self.assertEqual(self.builder.find_blameworthy_regression_window(10, look_back_limit=2), None)
# Flakey test avoidance requires at least 2 red builds:
self.assertEqual(self.builder.find_blameworthy_regression_window(4), None)
self.assertEqual(self.builder.find_blameworthy_regression_window(4, avoid_flakey_tests=False).revisions(), [1004])
# Green builder:
self.assertEqual(self.builder.find_blameworthy_regression_window(3), None)
def test_build_caching(self):
self.assertEqual(self.builder.build(10), self.builder.build(10))
def test_build_and_revision_for_filename(self):
expectations = {
"r47483 (1)/" : (47483, 1),
"r47483 (1).zip" : (47483, 1),
}
for filename, revision_and_build in expectations.items():
self.assertEqual(self.builder._revision_and_build_for_filename(filename), revision_and_build)