def test_last_green_revision(self):
buildbot = BuildBot()
def mock_builds_from_builders():
return self._fake_builds_at_index(0)
# Revision, is_green
# Ordered from newest (highest number) to oldest.
fake_builder1 = Builder("Fake Builder 1", None)
fake_builder1.revisions = [(1, True), (3, False), (5, True),
(10, True), (12, False)]
fake_builder2 = Builder("Fake Builder 2", None)
fake_builder2.revisions = [(1, True), (3, False), (7, True), (9, True),
(12, False)]
some_builder = Builder("Some Builder", None)
some_builder.revisions = [(1, True), (3, True), (7, True), (11, False),
(12, True)]
buildbot.builders = lambda: [
fake_builder1, fake_builder2, some_builder
]
buildbot._revisions_for_builder = lambda builder: builder.revisions
buildbot._latest_builds_from_builders = mock_builds_from_builders
self.assertEqual(
buildbot.last_green_revision(''),
"The last known green revision is 7\nFake Builder 1: 10\nFake Builder 2: 9\nSome Builder: 12\n"
)
some_builder.revisions = [(1, False), (3, False)]
self.assertEqual(
buildbot.last_green_revision(''),
"Fake Builder 1: 10\nFake Builder 2: 9\nSome Builder has had no green revision in the last 2 runs\n"
)
def test_layout_test_results(self):
buildbot = BuildBot()
builder = Builder(u"Foo Builder (test)", buildbot)
builder._fetch_file_from_results = lambda results_url, file_name: None
build = Build(builder, None, None, None)
# Test that layout_test_results() returns None if the fetch fails.
self.assertEqual(build.layout_test_results(), None)
def test_layout_test_results(self):
buildbot = BuildBot()
builder = Builder(u"Foo Builder (test)", buildbot)
builder._fetch_file_from_results = lambda results_url, file_name: None
build = Build(builder, None, None, None)
# Test that layout_test_results() returns None if the fetch fails.
self.assertEqual(build.layout_test_results(), None)
def test_rollout_reason(self):
sheriff = Sheriff(MockTool(), MockSheriffBot())
builders = [
Builder("Foo", None),
Builder("Bar", None),
]
reason = "Caused builders Foo and Bar to fail."
self.assertEquals(sheriff._rollout_reason(builders), reason)
def test_revisions_for_builder(self):
buildbot = BuildBot()
buildbot._fetch_builder_page = lambda builder: builder.page
builder_with_success = Builder('Some builder', None)
builder_with_success.page = self._fake_builder_page
self.assertEqual(buildbot._revisions_for_builder(builder_with_success), [(104643, False), (104636, False), (104635, True), (104633, False)])
builder_without_success = Builder('Some builder', None)
builder_without_success.page = self._fake_builder_page_without_success
self.assertEqual(buildbot._revisions_for_builder(builder_without_success), [(104643, False), (104636, False), (104635, False), (104633, False)])
def setUp(self):
self.buildbot = BuildBot()
self.builder = Builder(u"Test Builder \u2661", self.buildbot)
def _mock_fetch_build(build_number):
build = Build(builder=self.builder,
build_number=build_number,
revision=build_number + 1000,
is_green=build_number < 4)
return build
self.builder._fetch_build = _mock_fetch_build
def test_post_blame_comment_on_bug(self):
sheriff = Sheriff(MockTool(), MockSheriffBot())
builders = [
Builder("Foo", None),
Builder("Bar", None),
]
commit_info = Mock()
commit_info.bug_id = lambda: None
commit_info.revision = lambda: 4321
commit_info.committer = lambda: None
commit_info.committer_email = lambda: "*****@*****.**"
commit_info.reviewer = lambda: None
commit_info.author = lambda: None
sheriff.post_automatic_rollout_patch(commit_info, builders)
def test_fetch_build(self):
buildbot = BuildBot()
builder = Builder(u"Test Builder \u2661", buildbot)
def mock_fetch_build_dictionary(self, build_number):
build_dictionary = {
"sourceStamp": {
"revision": None, # revision=None means a trunk build started from the force-build button on the builder page.
},
"number": int(build_number),
# Intentionally missing the 'results' key, meaning it's a "pass" build.
}
return build_dictionary
buildbot._fetch_build_dictionary = mock_fetch_build_dictionary
self.assertNotEqual(builder._fetch_build(1), None)
def test_fetch_build(self):
buildbot = BuildBot()
builder = Builder(u"Test Builder \u2661", buildbot)
def mock_fetch_build_dictionary(self, build_number):
build_dictionary = {
"sourceStamp": {
"revision": None, # revision=None means a trunk build started from the force-build button on the builder page.
},
"number": int(build_number),
# Intentionally missing the 'results' key, meaning it's a "pass" build.
}
return build_dictionary
buildbot._fetch_build_dictionary = mock_fetch_build_dictionary
self.assertIsNotNone(builder._fetch_build(1))
def run():
sheriff = Sheriff(MockTool(), MockSheriffBot())
builders = [
Builder("Foo", None),
Builder("Bar", None),
]
commit_info = Mock()
commit_info.bug_id = lambda: None
commit_info.revision = lambda: 4321
# Should do nothing with no bug_id
sheriff.post_blame_comment_on_bug(commit_info, builders, [])
sheriff.post_blame_comment_on_bug(commit_info, builders, ["mock-test-1", "mock-test-2"])
# Should try to post a comment to the bug, but MockTool.bugs does nothing.
commit_info.bug_id = lambda: 1234
sheriff.post_blame_comment_on_bug(commit_info, builders, [])
sheriff.post_blame_comment_on_bug(commit_info, builders, ["mock-test-1"])
sheriff.post_blame_comment_on_bug(commit_info, builders, ["mock-test-1", "mock-test-2"])
def test_revisions_for_builder(self):
buildbot = BuildBot()
buildbot._fetch_builder_page = lambda builder: builder.page
builder_with_success = Builder('Some builder', None)
builder_with_success.page = self._fake_builder_page
self.assertEqual(buildbot._revisions_for_builder(builder_with_success), [(104643, False), (104636, False), (104635, True), (104633, False)])
builder_without_success = Builder('Some builder', None)
builder_without_success.page = self._fake_builder_page_without_success
self.assertEqual(buildbot._revisions_for_builder(builder_without_success), [(104643, False), (104636, False), (104635, False), (104633, False)])
def test_last_green_revision(self):
buildbot = BuildBot()
def mock_builds_from_builders():
return self._fake_builds_at_index(0)
# Revision, is_green
# Ordered from newest (highest number) to oldest.
fake_builder1 = Builder("Fake Builder 1", None)
fake_builder1.revisions = [(1, True), (3, False), (5, True), (10, True), (12, False)]
fake_builder2 = Builder("Fake Builder 2", None)
fake_builder2.revisions = [(1, True), (3, False), (7, True), (9, True), (12, False)]
some_builder = Builder("Some Builder", None)
some_builder.revisions = [(1, True), (3, True), (7, True), (11, False), (12, True)]
buildbot.builders = lambda: [fake_builder1, fake_builder2, some_builder]
buildbot._revisions_for_builder = lambda builder: builder.revisions
buildbot._latest_builds_from_builders = mock_builds_from_builders
self.assertEqual(buildbot.last_green_revision(''),
"The last known green revision is 7\nFake Builder 1: 10\nFake Builder 2: 9\nSome Builder: 12\n")
some_builder.revisions = [(1, False), (3, False)]
self.assertEqual(buildbot.last_green_revision(''),
"Fake Builder 1: 10\nFake Builder 2: 9\nSome Builder has had no green revision in the last 2 runs\n")
def setUp(self):
self.buildbot = BuildBot()
self.builder = Builder(u"Test Builder \u2661", self.buildbot)
self._install_fetch_build(lambda build_number: ["test1", "test2"])
def test_latest_cached_build(self):
b = Builder('builder', BuildBot())
b._fetch_build = self._fetch_build
b._fetch_revision_to_build_map = self._fetch_revision_to_build_map
self.assertEqual("correct build", b.latest_cached_build())
class BuilderTest(unittest.TestCase):
def _mock_test_result(self, testname):
return test_results.TestResult(testname,
[test_failures.FailureTextMismatch()])
def _install_fetch_build(self, failure):
def _mock_fetch_build(build_number):
build = Build(builder=self.builder,
build_number=build_number,
revision=build_number + 1000,
is_green=build_number < 4)
return build
self.builder._fetch_build = _mock_fetch_build
def setUp(self):
self.buildbot = BuildBot()
self.builder = Builder(u"Test Builder \u2661", self.buildbot)
self._install_fetch_build(lambda build_number: ["test1", "test2"])
def test_latest_layout_test_results(self):
self.builder.fetch_layout_test_results = lambda results_url: LayoutTestResults(
None)
self.builder.accumulated_results_url = lambda: "http://dummy_url.org"
self.assertTrue(self.builder.latest_layout_test_results())
def test_build_caching(self):
self.assertEqual(self.builder.build(10), self.builder.build(10))
def test_build_and_revision_for_filename(self):
expectations = {
"r47483 (1)/": (47483, 1),
"r47483 (1).zip": (47483, 1),
"random junk": None,
}
for filename, revision_and_build in expectations.items():
self.assertEqual(
self.builder._revision_and_build_for_filename(filename),
revision_and_build)
def test_file_info_list_to_revision_to_build_list(self):
file_info_list = [
{
"filename": "r47483 (1)/"
},
{
"filename": "r47483 (1).zip"
},
{
"filename": "random junk"
},
]
builds_and_revisions_list = [(47483, 1), (47483, 1)]
self.assertEqual(
self.builder._file_info_list_to_revision_to_build_list(
file_info_list), builds_and_revisions_list)
def test_fetch_build(self):
buildbot = BuildBot()
builder = Builder(u"Test Builder \u2661", buildbot)
def mock_fetch_build_dictionary(self, build_number):
build_dictionary = {
"sourceStamp": {
"revision":
None, # revision=None means a trunk build started from the force-build button on the builder page.
},
"number": int(build_number),
# Intentionally missing the 'results' key, meaning it's a "pass" build.
}
return build_dictionary
buildbot._fetch_build_dictionary = mock_fetch_build_dictionary
self.assertIsNotNone(builder._fetch_build(1))
def test_results_url(self):
builder = BuildBot().builder_with_name('WebKit Mac10.8 (dbg)')
self.assertEqual(
builder.results_url(),
'https://storage.googleapis.com/chromium-layout-test-archives/WebKit_Mac10_8__dbg_'
)
def test_accumulated_results_url(self):
builder = BuildBot().builder_with_name('WebKit Mac10.8 (dbg)')
self.assertEqual(
builder.accumulated_results_url(),
'https://storage.googleapis.com/chromium-layout-test-archives/WebKit_Mac10_8__dbg_/results/layout-test-results'
)
class BuilderTest(unittest.TestCase):
def _mock_test_result(self, testname):
return test_results.TestResult(testname,
[test_failures.FailureTextMismatch()])
def _install_fetch_build(self, failure):
def _mock_fetch_build(build_number):
build = Build(builder=self.builder,
build_number=build_number,
revision=build_number + 1000,
is_green=build_number < 4)
results = [
self._mock_test_result(testname)
for testname in failure(build_number)
]
layout_test_results = LayoutTestResults(results)
def mock_layout_test_results():
return layout_test_results
build.layout_test_results = mock_layout_test_results
return build
self.builder._fetch_build = _mock_fetch_build
def setUp(self):
self.buildbot = BuildBot()
self.builder = Builder(u"Test Builder \u2661", self.buildbot)
self._install_fetch_build(lambda build_number: ["test1", "test2"])
def test_latest_layout_test_results(self):
self.builder.fetch_layout_test_results = lambda results_url: LayoutTestResults(
[
self._mock_test_result(testname)
for testname in ["test1", "test2"]
])
self.builder.accumulated_results_url = lambda: "http://dummy_url.org"
self.assertTrue(self.builder.latest_layout_test_results())
def test_find_regression_window(self):
regression_window = self.builder.find_regression_window(
self.builder.build(10))
self.assertEqual(regression_window.build_before_failure().revision(),
1003)
self.assertEqual(regression_window.failing_build().revision(), 1004)
regression_window = self.builder.find_regression_window(
self.builder.build(10), look_back_limit=2)
self.assertEqual(regression_window.build_before_failure(), None)
self.assertEqual(regression_window.failing_build().revision(), 1008)
def test_none_build(self):
self.builder._fetch_build = lambda build_number: None
regression_window = self.builder.find_regression_window(
self.builder.build(10))
self.assertEqual(regression_window.build_before_failure(), None)
self.assertEqual(regression_window.failing_build(), None)
def test_flaky_tests(self):
self._install_fetch_build(lambda build_number: ["test1"]
if build_number % 2 else ["test2"])
regression_window = self.builder.find_regression_window(
self.builder.build(10))
self.assertEqual(regression_window.build_before_failure().revision(),
1009)
self.assertEqual(regression_window.failing_build().revision(), 1010)
def test_failure_and_flaky(self):
self._install_fetch_build(lambda build_number: ["test1", "test2"]
if build_number % 2 else ["test2"])
regression_window = self.builder.find_regression_window(
self.builder.build(10))
self.assertEqual(regression_window.build_before_failure().revision(),
1003)
self.assertEqual(regression_window.failing_build().revision(), 1004)
def test_no_results(self):
self._install_fetch_build(lambda build_number: ["test1", "test2"]
if build_number % 2 else ["test2"])
regression_window = self.builder.find_regression_window(
self.builder.build(10))
self.assertEqual(regression_window.build_before_failure().revision(),
1003)
self.assertEqual(regression_window.failing_build().revision(), 1004)
def test_failure_after_flaky(self):
self._install_fetch_build(lambda build_number: ["test1", "test2"]
if build_number > 6 else ["test3"])
regression_window = self.builder.find_regression_window(
self.builder.build(10))
self.assertEqual(regression_window.build_before_failure().revision(),
1006)
self.assertEqual(regression_window.failing_build().revision(), 1007)
def test_find_blameworthy_regression_window(self):
self.assertEqual(
self.builder.find_blameworthy_regression_window(10).revisions(),
[1004])
self.assertEqual(
self.builder.find_blameworthy_regression_window(10,
look_back_limit=2),
None)
# Flakey test avoidance requires at least 2 red builds:
self.assertEqual(self.builder.find_blameworthy_regression_window(4),
None)
self.assertEqual(
self.builder.find_blameworthy_regression_window(
4, avoid_flakey_tests=False).revisions(), [1004])
# Green builder:
self.assertEqual(self.builder.find_blameworthy_regression_window(3),
None)
def test_build_caching(self):
self.assertEqual(self.builder.build(10), self.builder.build(10))
def test_build_and_revision_for_filename(self):
expectations = {
"r47483 (1)/": (47483, 1),
"r47483 (1).zip": (47483, 1),
"random junk": None,
}
for filename, revision_and_build in expectations.items():
self.assertEqual(
self.builder._revision_and_build_for_filename(filename),
revision_and_build)
def test_file_info_list_to_revision_to_build_list(self):
file_info_list = [
{
"filename": "r47483 (1)/"
},
{
"filename": "r47483 (1).zip"
},
{
"filename": "random junk"
},
]
builds_and_revisions_list = [(47483, 1), (47483, 1)]
self.assertEqual(
self.builder._file_info_list_to_revision_to_build_list(
file_info_list), builds_and_revisions_list)
def test_fetch_build(self):
buildbot = BuildBot()
builder = Builder(u"Test Builder \u2661", buildbot)
def mock_fetch_build_dictionary(self, build_number):
build_dictionary = {
"sourceStamp": {
"revision":
None, # revision=None means a trunk build started from the force-build button on the builder page.
},
"number": int(build_number),
# Intentionally missing the 'results' key, meaning it's a "pass" build.
}
return build_dictionary
buildbot._fetch_build_dictionary = mock_fetch_build_dictionary
self.assertNotEqual(builder._fetch_build(1), None)
class BuilderTest(unittest.TestCase):
def _install_fetch_build(self, failure):
def _mock_fetch_build(build_number):
build = Build(builder=self.builder,
build_number=build_number,
revision=build_number + 1000,
is_green=build_number < 4)
build._layout_test_results = LayoutTestResults(
"http://buildbot.example.com/foo", {
LayoutTestResults.fail_key: failure(build_number),
})
return build
self.builder._fetch_build = _mock_fetch_build
def setUp(self):
self.buildbot = BuildBot()
self.builder = Builder(u"Test Builder \u2661", self.buildbot)
self._install_fetch_build(lambda build_number: ["test1", "test2"])
def test_find_failure_transition(self):
(green_build, red_build) = self.builder.find_failure_transition(
self.builder.build(10))
self.assertEqual(green_build.revision(), 1003)
self.assertEqual(red_build.revision(), 1004)
(green_build, red_build) = self.builder.find_failure_transition(
self.builder.build(10), look_back_limit=2)
self.assertEqual(green_build, None)
self.assertEqual(red_build.revision(), 1008)
def test_none_build(self):
self.builder._fetch_build = lambda build_number: None
(green_build, red_build) = self.builder.find_failure_transition(
self.builder.build(10))
self.assertEqual(green_build, None)
self.assertEqual(red_build, None)
def test_flaky_tests(self):
self._install_fetch_build(lambda build_number: ["test1"]
if build_number % 2 else ["test2"])
(green_build, red_build) = self.builder.find_failure_transition(
self.builder.build(10))
self.assertEqual(green_build.revision(), 1009)
self.assertEqual(red_build.revision(), 1010)
def test_failure_and_flaky(self):
self._install_fetch_build(lambda build_number: ["test1", "test2"]
if build_number % 2 else ["test2"])
(green_build, red_build) = self.builder.find_failure_transition(
self.builder.build(10))
self.assertEqual(green_build.revision(), 1003)
self.assertEqual(red_build.revision(), 1004)
def test_no_results(self):
self._install_fetch_build(lambda build_number: ["test1", "test2"]
if build_number % 2 else ["test2"])
(green_build, red_build) = self.builder.find_failure_transition(
self.builder.build(10))
self.assertEqual(green_build.revision(), 1003)
self.assertEqual(red_build.revision(), 1004)
def test_failure_after_flaky(self):
self._install_fetch_build(lambda build_number: ["test1", "test2"]
if build_number > 6 else ["test3"])
(green_build, red_build) = self.builder.find_failure_transition(
self.builder.build(10))
self.assertEqual(green_build.revision(), 1006)
self.assertEqual(red_build.revision(), 1007)
def test_blameworthy_revisions(self):
self.assertEqual(self.builder.blameworthy_revisions(10), [1004])
self.assertEqual(
self.builder.blameworthy_revisions(10, look_back_limit=2), [])
# Flakey test avoidance requires at least 2 red builds:
self.assertEqual(self.builder.blameworthy_revisions(4), [])
self.assertEqual(
self.builder.blameworthy_revisions(4, avoid_flakey_tests=False),
[1004])
# Green builder:
self.assertEqual(self.builder.blameworthy_revisions(3), [])
def test_build_caching(self):
self.assertEqual(self.builder.build(10), self.builder.build(10))
def test_build_and_revision_for_filename(self):
expectations = {
"r47483 (1)/": (47483, 1),
"r47483 (1).zip": (47483, 1),
}
for filename, revision_and_build in expectations.items():
self.assertEqual(
self.builder._revision_and_build_for_filename(filename),
revision_and_build)
def test_results(self):
builder = Builder('builder', BuildBot())
b = Build(builder, 123, 123, True)
self.assertTrue(b.results())
class BuilderTest(unittest.TestCase):
def _mock_test_result(self, testname):
return test_results.TestResult(testname,
[test_failures.FailureTextMismatch()])
def _install_fetch_build(self, failure):
def _mock_fetch_build(build_number):
build = Build(builder=self.builder,
build_number=build_number,
revision=build_number + 1000,
is_green=build_number < 4)
results = [
self._mock_test_result(testname)
for testname in failure(build_number)
]
build._layout_test_results = LayoutTestResults(results)
return build
self.builder._fetch_build = _mock_fetch_build
def setUp(self):
self.buildbot = BuildBot()
self.builder = Builder(u"Test Builder \u2661", self.buildbot)
self._install_fetch_build(lambda build_number: ["test1", "test2"])
def test_find_regression_window(self):
regression_window = self.builder.find_regression_window(
self.builder.build(10))
self.assertEqual(regression_window.build_before_failure().revision(),
1003)
self.assertEqual(regression_window.failing_build().revision(), 1004)
regression_window = self.builder.find_regression_window(
self.builder.build(10), look_back_limit=2)
self.assertEqual(regression_window.build_before_failure(), None)
self.assertEqual(regression_window.failing_build().revision(), 1008)
def test_none_build(self):
self.builder._fetch_build = lambda build_number: None
regression_window = self.builder.find_regression_window(
self.builder.build(10))
self.assertEqual(regression_window.build_before_failure(), None)
self.assertEqual(regression_window.failing_build(), None)
def test_flaky_tests(self):
self._install_fetch_build(lambda build_number: ["test1"]
if build_number % 2 else ["test2"])
regression_window = self.builder.find_regression_window(
self.builder.build(10))
self.assertEqual(regression_window.build_before_failure().revision(),
1009)
self.assertEqual(regression_window.failing_build().revision(), 1010)
def test_failure_and_flaky(self):
self._install_fetch_build(lambda build_number: ["test1", "test2"]
if build_number % 2 else ["test2"])
regression_window = self.builder.find_regression_window(
self.builder.build(10))
self.assertEqual(regression_window.build_before_failure().revision(),
1003)
self.assertEqual(regression_window.failing_build().revision(), 1004)
def test_no_results(self):
self._install_fetch_build(lambda build_number: ["test1", "test2"]
if build_number % 2 else ["test2"])
regression_window = self.builder.find_regression_window(
self.builder.build(10))
self.assertEqual(regression_window.build_before_failure().revision(),
1003)
self.assertEqual(regression_window.failing_build().revision(), 1004)
def test_failure_after_flaky(self):
self._install_fetch_build(lambda build_number: ["test1", "test2"]
if build_number > 6 else ["test3"])
regression_window = self.builder.find_regression_window(
self.builder.build(10))
self.assertEqual(regression_window.build_before_failure().revision(),
1006)
self.assertEqual(regression_window.failing_build().revision(), 1007)
def test_find_blameworthy_regression_window(self):
self.assertEqual(
self.builder.find_blameworthy_regression_window(10).revisions(),
[1004])
self.assertEqual(
self.builder.find_blameworthy_regression_window(10,
look_back_limit=2),
None)
# Flakey test avoidance requires at least 2 red builds:
self.assertEqual(self.builder.find_blameworthy_regression_window(4),
None)
self.assertEqual(
self.builder.find_blameworthy_regression_window(
4, avoid_flakey_tests=False).revisions(), [1004])
# Green builder:
self.assertEqual(self.builder.find_blameworthy_regression_window(3),
None)
def test_build_caching(self):
self.assertEqual(self.builder.build(10), self.builder.build(10))
def test_build_and_revision_for_filename(self):
expectations = {
"r47483 (1)/": (47483, 1),
"r47483 (1).zip": (47483, 1),
}
for filename, revision_and_build in expectations.items():
self.assertEqual(
self.builder._revision_and_build_for_filename(filename),
revision_and_build)
def setUp(self):
self.buildbot = BuildBot()
self.builder = Builder(u"Test Builder \u2661", self.buildbot)
self._install_fetch_build(lambda build_number: ["test1", "test2"])
def test_latest_cached_build(self):
b = Builder('builder', BuildBot())
b._fetch_build = self._fetch_build
b._fetch_revision_to_build_map = self._fetch_revision_to_build_map
self.assertEqual("correct build", b.latest_cached_build())
class BuilderTest(unittest.TestCase):
def _mock_test_result(self, testname):
return test_results.TestResult(testname, [test_failures.FailureTextMismatch()])
def _install_fetch_build(self, failure):
def _mock_fetch_build(build_number):
build = Build(
builder=self.builder,
build_number=build_number,
revision=build_number + 1000,
is_green=build_number < 4
)
return build
self.builder._fetch_build = _mock_fetch_build
def setUp(self):
self.buildbot = BuildBot()
self.builder = Builder(u"Test Builder \u2661", self.buildbot)
self._install_fetch_build(lambda build_number: ["test1", "test2"])
def test_latest_layout_test_results(self):
self.builder.fetch_layout_test_results = lambda results_url: LayoutTestResults(None)
self.builder.accumulated_results_url = lambda: "http://dummy_url.org"
self.assertTrue(self.builder.latest_layout_test_results())
def test_build_caching(self):
self.assertEqual(self.builder.build(10), self.builder.build(10))
def test_build_and_revision_for_filename(self):
expectations = {
"r47483 (1)/" : (47483, 1),
"r47483 (1).zip" : (47483, 1),
"random junk": None,
}
for filename, revision_and_build in expectations.items():
self.assertEqual(self.builder._revision_and_build_for_filename(filename), revision_and_build)
def test_file_info_list_to_revision_to_build_list(self):
file_info_list = [
{"filename": "r47483 (1)/"},
{"filename": "r47483 (1).zip"},
{"filename": "random junk"},
]
builds_and_revisions_list = [(47483, 1), (47483, 1)]
self.assertEqual(self.builder._file_info_list_to_revision_to_build_list(file_info_list), builds_and_revisions_list)
def test_fetch_build(self):
buildbot = BuildBot()
builder = Builder(u"Test Builder \u2661", buildbot)
def mock_fetch_build_dictionary(self, build_number):
build_dictionary = {
"sourceStamp": {
"revision": None, # revision=None means a trunk build started from the force-build button on the builder page.
},
"number": int(build_number),
# Intentionally missing the 'results' key, meaning it's a "pass" build.
}
return build_dictionary
buildbot._fetch_build_dictionary = mock_fetch_build_dictionary
self.assertIsNotNone(builder._fetch_build(1))
def test_results_url(self):
builder = BuildBot().builder_with_name('WebKit Mac10.8 (dbg)')
self.assertEqual(builder.results_url(),
'https://storage.googleapis.com/chromium-layout-test-archives/WebKit_Mac10_8__dbg_')
def test_accumulated_results_url(self):
builder = BuildBot().builder_with_name('WebKit Mac10.8 (dbg)')
self.assertEqual(builder.accumulated_results_url(),
'https://storage.googleapis.com/chromium-layout-test-archives/WebKit_Mac10_8__dbg_/results/layout-test-results')
class BuilderTest(unittest.TestCase):
def _install_fetch_build(self, failure):
def _mock_fetch_build(build_number):
build = Build(
builder=self.builder,
build_number=build_number,
revision=build_number + 1000,
is_green=build_number < 4
)
build._layout_test_results = LayoutTestResults(
"http://buildbot.example.com/foo", {
LayoutTestResults.fail_key: failure(build_number),
})
return build
self.builder._fetch_build = _mock_fetch_build
def setUp(self):
self.buildbot = BuildBot()
self.builder = Builder(u"Test Builder \u2661", self.buildbot)
self._install_fetch_build(lambda build_number: ["test1", "test2"])
def test_find_failure_transition(self):
(green_build, red_build) = self.builder.find_failure_transition(self.builder.build(10))
self.assertEqual(green_build.revision(), 1003)
self.assertEqual(red_build.revision(), 1004)
(green_build, red_build) = self.builder.find_failure_transition(self.builder.build(10), look_back_limit=2)
self.assertEqual(green_build, None)
self.assertEqual(red_build.revision(), 1008)
def test_none_build(self):
self.builder._fetch_build = lambda build_number: None
(green_build, red_build) = self.builder.find_failure_transition(self.builder.build(10))
self.assertEqual(green_build, None)
self.assertEqual(red_build, None)
def test_flaky_tests(self):
self._install_fetch_build(lambda build_number: ["test1"] if build_number % 2 else ["test2"])
(green_build, red_build) = self.builder.find_failure_transition(self.builder.build(10))
self.assertEqual(green_build.revision(), 1009)
self.assertEqual(red_build.revision(), 1010)
def test_failure_and_flaky(self):
self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number % 2 else ["test2"])
(green_build, red_build) = self.builder.find_failure_transition(self.builder.build(10))
self.assertEqual(green_build.revision(), 1003)
self.assertEqual(red_build.revision(), 1004)
def test_no_results(self):
self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number % 2 else ["test2"])
(green_build, red_build) = self.builder.find_failure_transition(self.builder.build(10))
self.assertEqual(green_build.revision(), 1003)
self.assertEqual(red_build.revision(), 1004)
def test_failure_after_flaky(self):
self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number > 6 else ["test3"])
(green_build, red_build) = self.builder.find_failure_transition(self.builder.build(10))
self.assertEqual(green_build.revision(), 1006)
self.assertEqual(red_build.revision(), 1007)
def test_blameworthy_revisions(self):
self.assertEqual(self.builder.blameworthy_revisions(10), [1004])
self.assertEqual(self.builder.blameworthy_revisions(10, look_back_limit=2), [])
# Flakey test avoidance requires at least 2 red builds:
self.assertEqual(self.builder.blameworthy_revisions(4), [])
self.assertEqual(self.builder.blameworthy_revisions(4, avoid_flakey_tests=False), [1004])
# Green builder:
self.assertEqual(self.builder.blameworthy_revisions(3), [])
def test_build_caching(self):
self.assertEqual(self.builder.build(10), self.builder.build(10))
def test_build_and_revision_for_filename(self):
expectations = {
"r47483 (1)/" : (47483, 1),
"r47483 (1).zip" : (47483, 1),
}
for filename, revision_and_build in expectations.items():
self.assertEqual(self.builder._revision_and_build_for_filename(filename), revision_and_build)
class BuilderTest(unittest.TestCase):
def _mock_test_result(self, testname):
return test_results.TestResult(testname, [test_failures.FailureTextMismatch()])
def _install_fetch_build(self, failure):
def _mock_fetch_build(build_number):
build = Build(
builder=self.builder,
build_number=build_number,
revision=build_number + 1000,
is_green=build_number < 4
)
results = [self._mock_test_result(testname) for testname in failure(build_number)]
build._layout_test_results = LayoutTestResults(results)
return build
self.builder._fetch_build = _mock_fetch_build
def setUp(self):
self.buildbot = BuildBot()
self.builder = Builder(u"Test Builder \u2661", self.buildbot)
self._install_fetch_build(lambda build_number: ["test1", "test2"])
def test_find_regression_window(self):
regression_window = self.builder.find_regression_window(self.builder.build(10))
self.assertEqual(regression_window.build_before_failure().revision(), 1003)
self.assertEqual(regression_window.failing_build().revision(), 1004)
regression_window = self.builder.find_regression_window(self.builder.build(10), look_back_limit=2)
self.assertEqual(regression_window.build_before_failure(), None)
self.assertEqual(regression_window.failing_build().revision(), 1008)
def test_none_build(self):
self.builder._fetch_build = lambda build_number: None
regression_window = self.builder.find_regression_window(self.builder.build(10))
self.assertEqual(regression_window.build_before_failure(), None)
self.assertEqual(regression_window.failing_build(), None)
def test_flaky_tests(self):
self._install_fetch_build(lambda build_number: ["test1"] if build_number % 2 else ["test2"])
regression_window = self.builder.find_regression_window(self.builder.build(10))
self.assertEqual(regression_window.build_before_failure().revision(), 1009)
self.assertEqual(regression_window.failing_build().revision(), 1010)
def test_failure_and_flaky(self):
self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number % 2 else ["test2"])
regression_window = self.builder.find_regression_window(self.builder.build(10))
self.assertEqual(regression_window.build_before_failure().revision(), 1003)
self.assertEqual(regression_window.failing_build().revision(), 1004)
def test_no_results(self):
self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number % 2 else ["test2"])
regression_window = self.builder.find_regression_window(self.builder.build(10))
self.assertEqual(regression_window.build_before_failure().revision(), 1003)
self.assertEqual(regression_window.failing_build().revision(), 1004)
def test_failure_after_flaky(self):
self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number > 6 else ["test3"])
regression_window = self.builder.find_regression_window(self.builder.build(10))
self.assertEqual(regression_window.build_before_failure().revision(), 1006)
self.assertEqual(regression_window.failing_build().revision(), 1007)
def test_find_blameworthy_regression_window(self):
self.assertEqual(self.builder.find_blameworthy_regression_window(10).revisions(), [1004])
self.assertEqual(self.builder.find_blameworthy_regression_window(10, look_back_limit=2), None)
# Flakey test avoidance requires at least 2 red builds:
self.assertEqual(self.builder.find_blameworthy_regression_window(4), None)
self.assertEqual(self.builder.find_blameworthy_regression_window(4, avoid_flakey_tests=False).revisions(), [1004])
# Green builder:
self.assertEqual(self.builder.find_blameworthy_regression_window(3), None)
def test_build_caching(self):
self.assertEqual(self.builder.build(10), self.builder.build(10))
def test_build_and_revision_for_filename(self):
expectations = {
"r47483 (1)/" : (47483, 1),
"r47483 (1).zip" : (47483, 1),
}
for filename, revision_and_build in expectations.items():
self.assertEqual(self.builder._revision_and_build_for_filename(filename), revision_and_build)
class BuilderTest(unittest.TestCase):
def _mock_test_result(self, testname):
return test_results.TestResult(testname, [test_failures.FailureTextMismatch()])
def _install_fetch_build(self, failure):
def _mock_fetch_build(build_number):
build = Build(
builder=self.builder,
build_number=build_number,
revision=build_number + 1000,
is_green=build_number < 4
)
results = [self._mock_test_result(testname) for testname in failure(build_number)]
layout_test_results = LayoutTestResults(results)
def mock_layout_test_results():
return layout_test_results
build.layout_test_results = mock_layout_test_results
return build
self.builder._fetch_build = _mock_fetch_build
def setUp(self):
self.buildbot = BuildBot()
self.builder = Builder(u"Test Builder \u2661", self.buildbot)
self._install_fetch_build(lambda build_number: ["test1", "test2"])
def test_latest_layout_test_results(self):
self.builder.fetch_layout_test_results = lambda results_url: LayoutTestResults([self._mock_test_result(testname) for testname in ["test1", "test2"]])
self.builder.accumulated_results_url = lambda: "http://dummy_url.org"
self.assertTrue(self.builder.latest_layout_test_results())
def test_find_regression_window(self):
regression_window = self.builder.find_regression_window(self.builder.build(10))
self.assertEqual(regression_window.build_before_failure().revision(), 1003)
self.assertEqual(regression_window.failing_build().revision(), 1004)
regression_window = self.builder.find_regression_window(self.builder.build(10), look_back_limit=2)
self.assertEqual(regression_window.build_before_failure(), None)
self.assertEqual(regression_window.failing_build().revision(), 1008)
def test_none_build(self):
self.builder._fetch_build = lambda build_number: None
regression_window = self.builder.find_regression_window(self.builder.build(10))
self.assertEqual(regression_window.build_before_failure(), None)
self.assertEqual(regression_window.failing_build(), None)
def test_flaky_tests(self):
self._install_fetch_build(lambda build_number: ["test1"] if build_number % 2 else ["test2"])
regression_window = self.builder.find_regression_window(self.builder.build(10))
self.assertEqual(regression_window.build_before_failure().revision(), 1009)
self.assertEqual(regression_window.failing_build().revision(), 1010)
def test_failure_and_flaky(self):
self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number % 2 else ["test2"])
regression_window = self.builder.find_regression_window(self.builder.build(10))
self.assertEqual(regression_window.build_before_failure().revision(), 1003)
self.assertEqual(regression_window.failing_build().revision(), 1004)
def test_no_results(self):
self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number % 2 else ["test2"])
regression_window = self.builder.find_regression_window(self.builder.build(10))
self.assertEqual(regression_window.build_before_failure().revision(), 1003)
self.assertEqual(regression_window.failing_build().revision(), 1004)
def test_failure_after_flaky(self):
self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number > 6 else ["test3"])
regression_window = self.builder.find_regression_window(self.builder.build(10))
self.assertEqual(regression_window.build_before_failure().revision(), 1006)
self.assertEqual(regression_window.failing_build().revision(), 1007)
def test_find_blameworthy_regression_window(self):
self.assertEqual(self.builder.find_blameworthy_regression_window(10).revisions(), [1004])
self.assertEqual(self.builder.find_blameworthy_regression_window(10, look_back_limit=2), None)
# Flakey test avoidance requires at least 2 red builds:
self.assertEqual(self.builder.find_blameworthy_regression_window(4), None)
self.assertEqual(self.builder.find_blameworthy_regression_window(4, avoid_flakey_tests=False).revisions(), [1004])
# Green builder:
self.assertEqual(self.builder.find_blameworthy_regression_window(3), None)
def test_build_caching(self):
self.assertEqual(self.builder.build(10), self.builder.build(10))
def test_build_and_revision_for_filename(self):
expectations = {
"r47483 (1)/" : (47483, 1),
"r47483 (1).zip" : (47483, 1),
"random junk": None,
}
for filename, revision_and_build in expectations.items():
self.assertEqual(self.builder._revision_and_build_for_filename(filename), revision_and_build)
def test_file_info_list_to_revision_to_build_list(self):
file_info_list = [
{"filename": "r47483 (1)/"},
{"filename": "r47483 (1).zip"},
{"filename": "random junk"},
]
builds_and_revisions_list = [(47483, 1), (47483, 1)]
self.assertEqual(self.builder._file_info_list_to_revision_to_build_list(file_info_list), builds_and_revisions_list)
def test_fetch_build(self):
buildbot = BuildBot()
builder = Builder(u"Test Builder \u2661", buildbot)
def mock_fetch_build_dictionary(self, build_number):
build_dictionary = {
"sourceStamp": {
"revision": None, # revision=None means a trunk build started from the force-build button on the builder page.
},
"number": int(build_number),
# Intentionally missing the 'results' key, meaning it's a "pass" build.
}
return build_dictionary
buildbot._fetch_build_dictionary = mock_fetch_build_dictionary
self.assertNotEqual(builder._fetch_build(1), None)
