def test_last_green_revision(self): buildbot = BuildBot() def mock_builds_from_builders(): return self._fake_builds_at_index(0) # Revision, is_green # Ordered from newest (highest number) to oldest. fake_builder1 = Builder("Fake Builder 1", None) fake_builder1.revisions = [(1, True), (3, False), (5, True), (10, True), (12, False)] fake_builder2 = Builder("Fake Builder 2", None) fake_builder2.revisions = [(1, True), (3, False), (7, True), (9, True), (12, False)] some_builder = Builder("Some Builder", None) some_builder.revisions = [(1, True), (3, True), (7, True), (11, False), (12, True)] buildbot.builders = lambda: [ fake_builder1, fake_builder2, some_builder ] buildbot._revisions_for_builder = lambda builder: builder.revisions buildbot._latest_builds_from_builders = mock_builds_from_builders self.assertEqual( buildbot.last_green_revision(''), "The last known green revision is 7\nFake Builder 1: 10\nFake Builder 2: 9\nSome Builder: 12\n" ) some_builder.revisions = [(1, False), (3, False)] self.assertEqual( buildbot.last_green_revision(''), "Fake Builder 1: 10\nFake Builder 2: 9\nSome Builder has had no green revision in the last 2 runs\n" )
def test_layout_test_results(self): buildbot = BuildBot() builder = Builder(u"Foo Builder (test)", buildbot) builder._fetch_file_from_results = lambda results_url, file_name: None build = Build(builder, None, None, None) # Test that layout_test_results() returns None if the fetch fails. self.assertEqual(build.layout_test_results(), None)
def test_rollout_reason(self): sheriff = Sheriff(MockTool(), MockSheriffBot()) builders = [ Builder("Foo", None), Builder("Bar", None), ] reason = "Caused builders Foo and Bar to fail." self.assertEquals(sheriff._rollout_reason(builders), reason)
def test_revisions_for_builder(self): buildbot = BuildBot() buildbot._fetch_builder_page = lambda builder: builder.page builder_with_success = Builder('Some builder', None) builder_with_success.page = self._fake_builder_page self.assertEqual(buildbot._revisions_for_builder(builder_with_success), [(104643, False), (104636, False), (104635, True), (104633, False)]) builder_without_success = Builder('Some builder', None) builder_without_success.page = self._fake_builder_page_without_success self.assertEqual(buildbot._revisions_for_builder(builder_without_success), [(104643, False), (104636, False), (104635, False), (104633, False)])
def setUp(self): self.buildbot = BuildBot() self.builder = Builder(u"Test Builder \u2661", self.buildbot) def _mock_fetch_build(build_number): build = Build(builder=self.builder, build_number=build_number, revision=build_number + 1000, is_green=build_number < 4) return build self.builder._fetch_build = _mock_fetch_build
def test_post_blame_comment_on_bug(self): sheriff = Sheriff(MockTool(), MockSheriffBot()) builders = [ Builder("Foo", None), Builder("Bar", None), ] commit_info = Mock() commit_info.bug_id = lambda: None commit_info.revision = lambda: 4321 commit_info.committer = lambda: None commit_info.committer_email = lambda: "*****@*****.**" commit_info.reviewer = lambda: None commit_info.author = lambda: None sheriff.post_automatic_rollout_patch(commit_info, builders)
def test_fetch_build(self): buildbot = BuildBot() builder = Builder(u"Test Builder \u2661", buildbot) def mock_fetch_build_dictionary(self, build_number): build_dictionary = { "sourceStamp": { "revision": None, # revision=None means a trunk build started from the force-build button on the builder page. }, "number": int(build_number), # Intentionally missing the 'results' key, meaning it's a "pass" build. } return build_dictionary buildbot._fetch_build_dictionary = mock_fetch_build_dictionary self.assertNotEqual(builder._fetch_build(1), None)
def test_fetch_build(self): buildbot = BuildBot() builder = Builder(u"Test Builder \u2661", buildbot) def mock_fetch_build_dictionary(self, build_number): build_dictionary = { "sourceStamp": { "revision": None, # revision=None means a trunk build started from the force-build button on the builder page. }, "number": int(build_number), # Intentionally missing the 'results' key, meaning it's a "pass" build. } return build_dictionary buildbot._fetch_build_dictionary = mock_fetch_build_dictionary self.assertIsNotNone(builder._fetch_build(1))
def run(): sheriff = Sheriff(MockTool(), MockSheriffBot()) builders = [ Builder("Foo", None), Builder("Bar", None), ] commit_info = Mock() commit_info.bug_id = lambda: None commit_info.revision = lambda: 4321 # Should do nothing with no bug_id sheriff.post_blame_comment_on_bug(commit_info, builders, []) sheriff.post_blame_comment_on_bug(commit_info, builders, ["mock-test-1", "mock-test-2"]) # Should try to post a comment to the bug, but MockTool.bugs does nothing. commit_info.bug_id = lambda: 1234 sheriff.post_blame_comment_on_bug(commit_info, builders, []) sheriff.post_blame_comment_on_bug(commit_info, builders, ["mock-test-1"]) sheriff.post_blame_comment_on_bug(commit_info, builders, ["mock-test-1", "mock-test-2"])
def test_last_green_revision(self): buildbot = BuildBot() def mock_builds_from_builders(): return self._fake_builds_at_index(0) # Revision, is_green # Ordered from newest (highest number) to oldest. fake_builder1 = Builder("Fake Builder 1", None) fake_builder1.revisions = [(1, True), (3, False), (5, True), (10, True), (12, False)] fake_builder2 = Builder("Fake Builder 2", None) fake_builder2.revisions = [(1, True), (3, False), (7, True), (9, True), (12, False)] some_builder = Builder("Some Builder", None) some_builder.revisions = [(1, True), (3, True), (7, True), (11, False), (12, True)] buildbot.builders = lambda: [fake_builder1, fake_builder2, some_builder] buildbot._revisions_for_builder = lambda builder: builder.revisions buildbot._latest_builds_from_builders = mock_builds_from_builders self.assertEqual(buildbot.last_green_revision(''), "The last known green revision is 7\nFake Builder 1: 10\nFake Builder 2: 9\nSome Builder: 12\n") some_builder.revisions = [(1, False), (3, False)] self.assertEqual(buildbot.last_green_revision(''), "Fake Builder 1: 10\nFake Builder 2: 9\nSome Builder has had no green revision in the last 2 runs\n")
def setUp(self): self.buildbot = BuildBot() self.builder = Builder(u"Test Builder \u2661", self.buildbot) self._install_fetch_build(lambda build_number: ["test1", "test2"])
def test_latest_cached_build(self): b = Builder('builder', BuildBot()) b._fetch_build = self._fetch_build b._fetch_revision_to_build_map = self._fetch_revision_to_build_map self.assertEqual("correct build", b.latest_cached_build())
class BuilderTest(unittest.TestCase): def _mock_test_result(self, testname): return test_results.TestResult(testname, [test_failures.FailureTextMismatch()]) def _install_fetch_build(self, failure): def _mock_fetch_build(build_number): build = Build(builder=self.builder, build_number=build_number, revision=build_number + 1000, is_green=build_number < 4) return build self.builder._fetch_build = _mock_fetch_build def setUp(self): self.buildbot = BuildBot() self.builder = Builder(u"Test Builder \u2661", self.buildbot) self._install_fetch_build(lambda build_number: ["test1", "test2"]) def test_latest_layout_test_results(self): self.builder.fetch_layout_test_results = lambda results_url: LayoutTestResults( None) self.builder.accumulated_results_url = lambda: "http://dummy_url.org" self.assertTrue(self.builder.latest_layout_test_results()) def test_build_caching(self): self.assertEqual(self.builder.build(10), self.builder.build(10)) def test_build_and_revision_for_filename(self): expectations = { "r47483 (1)/": (47483, 1), "r47483 (1).zip": (47483, 1), "random junk": None, } for filename, revision_and_build in expectations.items(): self.assertEqual( self.builder._revision_and_build_for_filename(filename), revision_and_build) def test_file_info_list_to_revision_to_build_list(self): file_info_list = [ { "filename": "r47483 (1)/" }, { "filename": "r47483 (1).zip" }, { "filename": "random junk" }, ] builds_and_revisions_list = [(47483, 1), (47483, 1)] self.assertEqual( self.builder._file_info_list_to_revision_to_build_list( file_info_list), builds_and_revisions_list) def test_fetch_build(self): buildbot = BuildBot() builder = Builder(u"Test Builder \u2661", buildbot) def mock_fetch_build_dictionary(self, build_number): build_dictionary = { "sourceStamp": { "revision": None, # revision=None means a trunk build started from the force-build button on the builder page. }, "number": int(build_number), # Intentionally missing the 'results' key, meaning it's a "pass" build. } return build_dictionary buildbot._fetch_build_dictionary = mock_fetch_build_dictionary self.assertIsNotNone(builder._fetch_build(1)) def test_results_url(self): builder = BuildBot().builder_with_name('WebKit Mac10.8 (dbg)') self.assertEqual( builder.results_url(), 'https://storage.googleapis.com/chromium-layout-test-archives/WebKit_Mac10_8__dbg_' ) def test_accumulated_results_url(self): builder = BuildBot().builder_with_name('WebKit Mac10.8 (dbg)') self.assertEqual( builder.accumulated_results_url(), 'https://storage.googleapis.com/chromium-layout-test-archives/WebKit_Mac10_8__dbg_/results/layout-test-results' )
class BuilderTest(unittest.TestCase): def _mock_test_result(self, testname): return test_results.TestResult(testname, [test_failures.FailureTextMismatch()]) def _install_fetch_build(self, failure): def _mock_fetch_build(build_number): build = Build(builder=self.builder, build_number=build_number, revision=build_number + 1000, is_green=build_number < 4) results = [ self._mock_test_result(testname) for testname in failure(build_number) ] layout_test_results = LayoutTestResults(results) def mock_layout_test_results(): return layout_test_results build.layout_test_results = mock_layout_test_results return build self.builder._fetch_build = _mock_fetch_build def setUp(self): self.buildbot = BuildBot() self.builder = Builder(u"Test Builder \u2661", self.buildbot) self._install_fetch_build(lambda build_number: ["test1", "test2"]) def test_latest_layout_test_results(self): self.builder.fetch_layout_test_results = lambda results_url: LayoutTestResults( [ self._mock_test_result(testname) for testname in ["test1", "test2"] ]) self.builder.accumulated_results_url = lambda: "http://dummy_url.org" self.assertTrue(self.builder.latest_layout_test_results()) def test_find_regression_window(self): regression_window = self.builder.find_regression_window( self.builder.build(10)) self.assertEqual(regression_window.build_before_failure().revision(), 1003) self.assertEqual(regression_window.failing_build().revision(), 1004) regression_window = self.builder.find_regression_window( self.builder.build(10), look_back_limit=2) self.assertEqual(regression_window.build_before_failure(), None) self.assertEqual(regression_window.failing_build().revision(), 1008) def test_none_build(self): self.builder._fetch_build = lambda build_number: None regression_window = self.builder.find_regression_window( self.builder.build(10)) self.assertEqual(regression_window.build_before_failure(), None) self.assertEqual(regression_window.failing_build(), None) def test_flaky_tests(self): self._install_fetch_build(lambda build_number: ["test1"] if build_number % 2 else ["test2"]) regression_window = self.builder.find_regression_window( self.builder.build(10)) self.assertEqual(regression_window.build_before_failure().revision(), 1009) self.assertEqual(regression_window.failing_build().revision(), 1010) def test_failure_and_flaky(self): self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number % 2 else ["test2"]) regression_window = self.builder.find_regression_window( self.builder.build(10)) self.assertEqual(regression_window.build_before_failure().revision(), 1003) self.assertEqual(regression_window.failing_build().revision(), 1004) def test_no_results(self): self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number % 2 else ["test2"]) regression_window = self.builder.find_regression_window( self.builder.build(10)) self.assertEqual(regression_window.build_before_failure().revision(), 1003) self.assertEqual(regression_window.failing_build().revision(), 1004) def test_failure_after_flaky(self): self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number > 6 else ["test3"]) regression_window = self.builder.find_regression_window( self.builder.build(10)) self.assertEqual(regression_window.build_before_failure().revision(), 1006) self.assertEqual(regression_window.failing_build().revision(), 1007) def test_find_blameworthy_regression_window(self): self.assertEqual( self.builder.find_blameworthy_regression_window(10).revisions(), [1004]) self.assertEqual( self.builder.find_blameworthy_regression_window(10, look_back_limit=2), None) # Flakey test avoidance requires at least 2 red builds: self.assertEqual(self.builder.find_blameworthy_regression_window(4), None) self.assertEqual( self.builder.find_blameworthy_regression_window( 4, avoid_flakey_tests=False).revisions(), [1004]) # Green builder: self.assertEqual(self.builder.find_blameworthy_regression_window(3), None) def test_build_caching(self): self.assertEqual(self.builder.build(10), self.builder.build(10)) def test_build_and_revision_for_filename(self): expectations = { "r47483 (1)/": (47483, 1), "r47483 (1).zip": (47483, 1), "random junk": None, } for filename, revision_and_build in expectations.items(): self.assertEqual( self.builder._revision_and_build_for_filename(filename), revision_and_build) def test_file_info_list_to_revision_to_build_list(self): file_info_list = [ { "filename": "r47483 (1)/" }, { "filename": "r47483 (1).zip" }, { "filename": "random junk" }, ] builds_and_revisions_list = [(47483, 1), (47483, 1)] self.assertEqual( self.builder._file_info_list_to_revision_to_build_list( file_info_list), builds_and_revisions_list) def test_fetch_build(self): buildbot = BuildBot() builder = Builder(u"Test Builder \u2661", buildbot) def mock_fetch_build_dictionary(self, build_number): build_dictionary = { "sourceStamp": { "revision": None, # revision=None means a trunk build started from the force-build button on the builder page. }, "number": int(build_number), # Intentionally missing the 'results' key, meaning it's a "pass" build. } return build_dictionary buildbot._fetch_build_dictionary = mock_fetch_build_dictionary self.assertNotEqual(builder._fetch_build(1), None)
class BuilderTest(unittest.TestCase): def _install_fetch_build(self, failure): def _mock_fetch_build(build_number): build = Build(builder=self.builder, build_number=build_number, revision=build_number + 1000, is_green=build_number < 4) build._layout_test_results = LayoutTestResults( "http://buildbot.example.com/foo", { LayoutTestResults.fail_key: failure(build_number), }) return build self.builder._fetch_build = _mock_fetch_build def setUp(self): self.buildbot = BuildBot() self.builder = Builder(u"Test Builder \u2661", self.buildbot) self._install_fetch_build(lambda build_number: ["test1", "test2"]) def test_find_failure_transition(self): (green_build, red_build) = self.builder.find_failure_transition( self.builder.build(10)) self.assertEqual(green_build.revision(), 1003) self.assertEqual(red_build.revision(), 1004) (green_build, red_build) = self.builder.find_failure_transition( self.builder.build(10), look_back_limit=2) self.assertEqual(green_build, None) self.assertEqual(red_build.revision(), 1008) def test_none_build(self): self.builder._fetch_build = lambda build_number: None (green_build, red_build) = self.builder.find_failure_transition( self.builder.build(10)) self.assertEqual(green_build, None) self.assertEqual(red_build, None) def test_flaky_tests(self): self._install_fetch_build(lambda build_number: ["test1"] if build_number % 2 else ["test2"]) (green_build, red_build) = self.builder.find_failure_transition( self.builder.build(10)) self.assertEqual(green_build.revision(), 1009) self.assertEqual(red_build.revision(), 1010) def test_failure_and_flaky(self): self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number % 2 else ["test2"]) (green_build, red_build) = self.builder.find_failure_transition( self.builder.build(10)) self.assertEqual(green_build.revision(), 1003) self.assertEqual(red_build.revision(), 1004) def test_no_results(self): self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number % 2 else ["test2"]) (green_build, red_build) = self.builder.find_failure_transition( self.builder.build(10)) self.assertEqual(green_build.revision(), 1003) self.assertEqual(red_build.revision(), 1004) def test_failure_after_flaky(self): self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number > 6 else ["test3"]) (green_build, red_build) = self.builder.find_failure_transition( self.builder.build(10)) self.assertEqual(green_build.revision(), 1006) self.assertEqual(red_build.revision(), 1007) def test_blameworthy_revisions(self): self.assertEqual(self.builder.blameworthy_revisions(10), [1004]) self.assertEqual( self.builder.blameworthy_revisions(10, look_back_limit=2), []) # Flakey test avoidance requires at least 2 red builds: self.assertEqual(self.builder.blameworthy_revisions(4), []) self.assertEqual( self.builder.blameworthy_revisions(4, avoid_flakey_tests=False), [1004]) # Green builder: self.assertEqual(self.builder.blameworthy_revisions(3), []) def test_build_caching(self): self.assertEqual(self.builder.build(10), self.builder.build(10)) def test_build_and_revision_for_filename(self): expectations = { "r47483 (1)/": (47483, 1), "r47483 (1).zip": (47483, 1), } for filename, revision_and_build in expectations.items(): self.assertEqual( self.builder._revision_and_build_for_filename(filename), revision_and_build)
def test_results(self): builder = Builder('builder', BuildBot()) b = Build(builder, 123, 123, True) self.assertTrue(b.results())
class BuilderTest(unittest.TestCase): def _mock_test_result(self, testname): return test_results.TestResult(testname, [test_failures.FailureTextMismatch()]) def _install_fetch_build(self, failure): def _mock_fetch_build(build_number): build = Build(builder=self.builder, build_number=build_number, revision=build_number + 1000, is_green=build_number < 4) results = [ self._mock_test_result(testname) for testname in failure(build_number) ] build._layout_test_results = LayoutTestResults(results) return build self.builder._fetch_build = _mock_fetch_build def setUp(self): self.buildbot = BuildBot() self.builder = Builder(u"Test Builder \u2661", self.buildbot) self._install_fetch_build(lambda build_number: ["test1", "test2"]) def test_find_regression_window(self): regression_window = self.builder.find_regression_window( self.builder.build(10)) self.assertEqual(regression_window.build_before_failure().revision(), 1003) self.assertEqual(regression_window.failing_build().revision(), 1004) regression_window = self.builder.find_regression_window( self.builder.build(10), look_back_limit=2) self.assertEqual(regression_window.build_before_failure(), None) self.assertEqual(regression_window.failing_build().revision(), 1008) def test_none_build(self): self.builder._fetch_build = lambda build_number: None regression_window = self.builder.find_regression_window( self.builder.build(10)) self.assertEqual(regression_window.build_before_failure(), None) self.assertEqual(regression_window.failing_build(), None) def test_flaky_tests(self): self._install_fetch_build(lambda build_number: ["test1"] if build_number % 2 else ["test2"]) regression_window = self.builder.find_regression_window( self.builder.build(10)) self.assertEqual(regression_window.build_before_failure().revision(), 1009) self.assertEqual(regression_window.failing_build().revision(), 1010) def test_failure_and_flaky(self): self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number % 2 else ["test2"]) regression_window = self.builder.find_regression_window( self.builder.build(10)) self.assertEqual(regression_window.build_before_failure().revision(), 1003) self.assertEqual(regression_window.failing_build().revision(), 1004) def test_no_results(self): self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number % 2 else ["test2"]) regression_window = self.builder.find_regression_window( self.builder.build(10)) self.assertEqual(regression_window.build_before_failure().revision(), 1003) self.assertEqual(regression_window.failing_build().revision(), 1004) def test_failure_after_flaky(self): self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number > 6 else ["test3"]) regression_window = self.builder.find_regression_window( self.builder.build(10)) self.assertEqual(regression_window.build_before_failure().revision(), 1006) self.assertEqual(regression_window.failing_build().revision(), 1007) def test_find_blameworthy_regression_window(self): self.assertEqual( self.builder.find_blameworthy_regression_window(10).revisions(), [1004]) self.assertEqual( self.builder.find_blameworthy_regression_window(10, look_back_limit=2), None) # Flakey test avoidance requires at least 2 red builds: self.assertEqual(self.builder.find_blameworthy_regression_window(4), None) self.assertEqual( self.builder.find_blameworthy_regression_window( 4, avoid_flakey_tests=False).revisions(), [1004]) # Green builder: self.assertEqual(self.builder.find_blameworthy_regression_window(3), None) def test_build_caching(self): self.assertEqual(self.builder.build(10), self.builder.build(10)) def test_build_and_revision_for_filename(self): expectations = { "r47483 (1)/": (47483, 1), "r47483 (1).zip": (47483, 1), } for filename, revision_and_build in expectations.items(): self.assertEqual( self.builder._revision_and_build_for_filename(filename), revision_and_build)
class BuilderTest(unittest.TestCase): def _mock_test_result(self, testname): return test_results.TestResult(testname, [test_failures.FailureTextMismatch()]) def _install_fetch_build(self, failure): def _mock_fetch_build(build_number): build = Build( builder=self.builder, build_number=build_number, revision=build_number + 1000, is_green=build_number < 4 ) return build self.builder._fetch_build = _mock_fetch_build def setUp(self): self.buildbot = BuildBot() self.builder = Builder(u"Test Builder \u2661", self.buildbot) self._install_fetch_build(lambda build_number: ["test1", "test2"]) def test_latest_layout_test_results(self): self.builder.fetch_layout_test_results = lambda results_url: LayoutTestResults(None) self.builder.accumulated_results_url = lambda: "http://dummy_url.org" self.assertTrue(self.builder.latest_layout_test_results()) def test_build_caching(self): self.assertEqual(self.builder.build(10), self.builder.build(10)) def test_build_and_revision_for_filename(self): expectations = { "r47483 (1)/" : (47483, 1), "r47483 (1).zip" : (47483, 1), "random junk": None, } for filename, revision_and_build in expectations.items(): self.assertEqual(self.builder._revision_and_build_for_filename(filename), revision_and_build) def test_file_info_list_to_revision_to_build_list(self): file_info_list = [ {"filename": "r47483 (1)/"}, {"filename": "r47483 (1).zip"}, {"filename": "random junk"}, ] builds_and_revisions_list = [(47483, 1), (47483, 1)] self.assertEqual(self.builder._file_info_list_to_revision_to_build_list(file_info_list), builds_and_revisions_list) def test_fetch_build(self): buildbot = BuildBot() builder = Builder(u"Test Builder \u2661", buildbot) def mock_fetch_build_dictionary(self, build_number): build_dictionary = { "sourceStamp": { "revision": None, # revision=None means a trunk build started from the force-build button on the builder page. }, "number": int(build_number), # Intentionally missing the 'results' key, meaning it's a "pass" build. } return build_dictionary buildbot._fetch_build_dictionary = mock_fetch_build_dictionary self.assertIsNotNone(builder._fetch_build(1)) def test_results_url(self): builder = BuildBot().builder_with_name('WebKit Mac10.8 (dbg)') self.assertEqual(builder.results_url(), 'https://storage.googleapis.com/chromium-layout-test-archives/WebKit_Mac10_8__dbg_') def test_accumulated_results_url(self): builder = BuildBot().builder_with_name('WebKit Mac10.8 (dbg)') self.assertEqual(builder.accumulated_results_url(), 'https://storage.googleapis.com/chromium-layout-test-archives/WebKit_Mac10_8__dbg_/results/layout-test-results')
class BuilderTest(unittest.TestCase): def _install_fetch_build(self, failure): def _mock_fetch_build(build_number): build = Build( builder=self.builder, build_number=build_number, revision=build_number + 1000, is_green=build_number < 4 ) build._layout_test_results = LayoutTestResults( "http://buildbot.example.com/foo", { LayoutTestResults.fail_key: failure(build_number), }) return build self.builder._fetch_build = _mock_fetch_build def setUp(self): self.buildbot = BuildBot() self.builder = Builder(u"Test Builder \u2661", self.buildbot) self._install_fetch_build(lambda build_number: ["test1", "test2"]) def test_find_failure_transition(self): (green_build, red_build) = self.builder.find_failure_transition(self.builder.build(10)) self.assertEqual(green_build.revision(), 1003) self.assertEqual(red_build.revision(), 1004) (green_build, red_build) = self.builder.find_failure_transition(self.builder.build(10), look_back_limit=2) self.assertEqual(green_build, None) self.assertEqual(red_build.revision(), 1008) def test_none_build(self): self.builder._fetch_build = lambda build_number: None (green_build, red_build) = self.builder.find_failure_transition(self.builder.build(10)) self.assertEqual(green_build, None) self.assertEqual(red_build, None) def test_flaky_tests(self): self._install_fetch_build(lambda build_number: ["test1"] if build_number % 2 else ["test2"]) (green_build, red_build) = self.builder.find_failure_transition(self.builder.build(10)) self.assertEqual(green_build.revision(), 1009) self.assertEqual(red_build.revision(), 1010) def test_failure_and_flaky(self): self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number % 2 else ["test2"]) (green_build, red_build) = self.builder.find_failure_transition(self.builder.build(10)) self.assertEqual(green_build.revision(), 1003) self.assertEqual(red_build.revision(), 1004) def test_no_results(self): self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number % 2 else ["test2"]) (green_build, red_build) = self.builder.find_failure_transition(self.builder.build(10)) self.assertEqual(green_build.revision(), 1003) self.assertEqual(red_build.revision(), 1004) def test_failure_after_flaky(self): self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number > 6 else ["test3"]) (green_build, red_build) = self.builder.find_failure_transition(self.builder.build(10)) self.assertEqual(green_build.revision(), 1006) self.assertEqual(red_build.revision(), 1007) def test_blameworthy_revisions(self): self.assertEqual(self.builder.blameworthy_revisions(10), [1004]) self.assertEqual(self.builder.blameworthy_revisions(10, look_back_limit=2), []) # Flakey test avoidance requires at least 2 red builds: self.assertEqual(self.builder.blameworthy_revisions(4), []) self.assertEqual(self.builder.blameworthy_revisions(4, avoid_flakey_tests=False), [1004]) # Green builder: self.assertEqual(self.builder.blameworthy_revisions(3), []) def test_build_caching(self): self.assertEqual(self.builder.build(10), self.builder.build(10)) def test_build_and_revision_for_filename(self): expectations = { "r47483 (1)/" : (47483, 1), "r47483 (1).zip" : (47483, 1), } for filename, revision_and_build in expectations.items(): self.assertEqual(self.builder._revision_and_build_for_filename(filename), revision_and_build)
class BuilderTest(unittest.TestCase): def _mock_test_result(self, testname): return test_results.TestResult(testname, [test_failures.FailureTextMismatch()]) def _install_fetch_build(self, failure): def _mock_fetch_build(build_number): build = Build( builder=self.builder, build_number=build_number, revision=build_number + 1000, is_green=build_number < 4 ) results = [self._mock_test_result(testname) for testname in failure(build_number)] build._layout_test_results = LayoutTestResults(results) return build self.builder._fetch_build = _mock_fetch_build def setUp(self): self.buildbot = BuildBot() self.builder = Builder(u"Test Builder \u2661", self.buildbot) self._install_fetch_build(lambda build_number: ["test1", "test2"]) def test_find_regression_window(self): regression_window = self.builder.find_regression_window(self.builder.build(10)) self.assertEqual(regression_window.build_before_failure().revision(), 1003) self.assertEqual(regression_window.failing_build().revision(), 1004) regression_window = self.builder.find_regression_window(self.builder.build(10), look_back_limit=2) self.assertEqual(regression_window.build_before_failure(), None) self.assertEqual(regression_window.failing_build().revision(), 1008) def test_none_build(self): self.builder._fetch_build = lambda build_number: None regression_window = self.builder.find_regression_window(self.builder.build(10)) self.assertEqual(regression_window.build_before_failure(), None) self.assertEqual(regression_window.failing_build(), None) def test_flaky_tests(self): self._install_fetch_build(lambda build_number: ["test1"] if build_number % 2 else ["test2"]) regression_window = self.builder.find_regression_window(self.builder.build(10)) self.assertEqual(regression_window.build_before_failure().revision(), 1009) self.assertEqual(regression_window.failing_build().revision(), 1010) def test_failure_and_flaky(self): self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number % 2 else ["test2"]) regression_window = self.builder.find_regression_window(self.builder.build(10)) self.assertEqual(regression_window.build_before_failure().revision(), 1003) self.assertEqual(regression_window.failing_build().revision(), 1004) def test_no_results(self): self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number % 2 else ["test2"]) regression_window = self.builder.find_regression_window(self.builder.build(10)) self.assertEqual(regression_window.build_before_failure().revision(), 1003) self.assertEqual(regression_window.failing_build().revision(), 1004) def test_failure_after_flaky(self): self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number > 6 else ["test3"]) regression_window = self.builder.find_regression_window(self.builder.build(10)) self.assertEqual(regression_window.build_before_failure().revision(), 1006) self.assertEqual(regression_window.failing_build().revision(), 1007) def test_find_blameworthy_regression_window(self): self.assertEqual(self.builder.find_blameworthy_regression_window(10).revisions(), [1004]) self.assertEqual(self.builder.find_blameworthy_regression_window(10, look_back_limit=2), None) # Flakey test avoidance requires at least 2 red builds: self.assertEqual(self.builder.find_blameworthy_regression_window(4), None) self.assertEqual(self.builder.find_blameworthy_regression_window(4, avoid_flakey_tests=False).revisions(), [1004]) # Green builder: self.assertEqual(self.builder.find_blameworthy_regression_window(3), None) def test_build_caching(self): self.assertEqual(self.builder.build(10), self.builder.build(10)) def test_build_and_revision_for_filename(self): expectations = { "r47483 (1)/" : (47483, 1), "r47483 (1).zip" : (47483, 1), } for filename, revision_and_build in expectations.items(): self.assertEqual(self.builder._revision_and_build_for_filename(filename), revision_and_build)
class BuilderTest(unittest.TestCase): def _mock_test_result(self, testname): return test_results.TestResult(testname, [test_failures.FailureTextMismatch()]) def _install_fetch_build(self, failure): def _mock_fetch_build(build_number): build = Build( builder=self.builder, build_number=build_number, revision=build_number + 1000, is_green=build_number < 4 ) results = [self._mock_test_result(testname) for testname in failure(build_number)] layout_test_results = LayoutTestResults(results) def mock_layout_test_results(): return layout_test_results build.layout_test_results = mock_layout_test_results return build self.builder._fetch_build = _mock_fetch_build def setUp(self): self.buildbot = BuildBot() self.builder = Builder(u"Test Builder \u2661", self.buildbot) self._install_fetch_build(lambda build_number: ["test1", "test2"]) def test_latest_layout_test_results(self): self.builder.fetch_layout_test_results = lambda results_url: LayoutTestResults([self._mock_test_result(testname) for testname in ["test1", "test2"]]) self.builder.accumulated_results_url = lambda: "http://dummy_url.org" self.assertTrue(self.builder.latest_layout_test_results()) def test_find_regression_window(self): regression_window = self.builder.find_regression_window(self.builder.build(10)) self.assertEqual(regression_window.build_before_failure().revision(), 1003) self.assertEqual(regression_window.failing_build().revision(), 1004) regression_window = self.builder.find_regression_window(self.builder.build(10), look_back_limit=2) self.assertEqual(regression_window.build_before_failure(), None) self.assertEqual(regression_window.failing_build().revision(), 1008) def test_none_build(self): self.builder._fetch_build = lambda build_number: None regression_window = self.builder.find_regression_window(self.builder.build(10)) self.assertEqual(regression_window.build_before_failure(), None) self.assertEqual(regression_window.failing_build(), None) def test_flaky_tests(self): self._install_fetch_build(lambda build_number: ["test1"] if build_number % 2 else ["test2"]) regression_window = self.builder.find_regression_window(self.builder.build(10)) self.assertEqual(regression_window.build_before_failure().revision(), 1009) self.assertEqual(regression_window.failing_build().revision(), 1010) def test_failure_and_flaky(self): self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number % 2 else ["test2"]) regression_window = self.builder.find_regression_window(self.builder.build(10)) self.assertEqual(regression_window.build_before_failure().revision(), 1003) self.assertEqual(regression_window.failing_build().revision(), 1004) def test_no_results(self): self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number % 2 else ["test2"]) regression_window = self.builder.find_regression_window(self.builder.build(10)) self.assertEqual(regression_window.build_before_failure().revision(), 1003) self.assertEqual(regression_window.failing_build().revision(), 1004) def test_failure_after_flaky(self): self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number > 6 else ["test3"]) regression_window = self.builder.find_regression_window(self.builder.build(10)) self.assertEqual(regression_window.build_before_failure().revision(), 1006) self.assertEqual(regression_window.failing_build().revision(), 1007) def test_find_blameworthy_regression_window(self): self.assertEqual(self.builder.find_blameworthy_regression_window(10).revisions(), [1004]) self.assertEqual(self.builder.find_blameworthy_regression_window(10, look_back_limit=2), None) # Flakey test avoidance requires at least 2 red builds: self.assertEqual(self.builder.find_blameworthy_regression_window(4), None) self.assertEqual(self.builder.find_blameworthy_regression_window(4, avoid_flakey_tests=False).revisions(), [1004]) # Green builder: self.assertEqual(self.builder.find_blameworthy_regression_window(3), None) def test_build_caching(self): self.assertEqual(self.builder.build(10), self.builder.build(10)) def test_build_and_revision_for_filename(self): expectations = { "r47483 (1)/" : (47483, 1), "r47483 (1).zip" : (47483, 1), "random junk": None, } for filename, revision_and_build in expectations.items(): self.assertEqual(self.builder._revision_and_build_for_filename(filename), revision_and_build) def test_file_info_list_to_revision_to_build_list(self): file_info_list = [ {"filename": "r47483 (1)/"}, {"filename": "r47483 (1).zip"}, {"filename": "random junk"}, ] builds_and_revisions_list = [(47483, 1), (47483, 1)] self.assertEqual(self.builder._file_info_list_to_revision_to_build_list(file_info_list), builds_and_revisions_list) def test_fetch_build(self): buildbot = BuildBot() builder = Builder(u"Test Builder \u2661", buildbot) def mock_fetch_build_dictionary(self, build_number): build_dictionary = { "sourceStamp": { "revision": None, # revision=None means a trunk build started from the force-build button on the builder page. }, "number": int(build_number), # Intentionally missing the 'results' key, meaning it's a "pass" build. } return build_dictionary buildbot._fetch_build_dictionary = mock_fetch_build_dictionary self.assertNotEqual(builder._fetch_build(1), None)