def __call__(self, master_url): try: master_json = buildbot.fetch_master_json(master_url) if not master_json: return (None, None, None, master_url) master_alerts, stale_master_alert = alert_builder.alerts_for_master( self._cache, master_url, master_json, self._old_alerts, self._builder_filter, self._jobs) # FIXME: The builder info doesn't really belong here. The builder # revisions tool uses this and we happen to have the builder json cached # at this point so it's cheap to compute, but it should be moved # to a different feed. data, stale_builder_alerts = ( buildbot.latest_builder_info_and_alerts_for_master( self._cache, master_url, master_json)) if stale_master_alert: stale_builder_alerts.append(stale_master_alert) return (master_alerts, data, stale_builder_alerts, master_url) except: # Put all exception text into an exception and raise that so it doesn't # get eaten by the multiprocessing code. raise Exception(''.join( traceback.format_exception(*sys.exc_info())))
def test_builder_info_for_master_no_build(self): """Test that we don't crash when the buildbot/CBE both don't have a build for a given build ID. """ master0_url = 'http://foo/bar/master0' master0 = { 'builders': { 'builder0': { 'cachedBuilds': [0], 'currentBuilds': [], 'state': 'happy', "pendingBuilds": 1, } } } def fetch(_cache, _master_url, _builder_name, _latest_build_id): return None, None old_fetch = buildbot.fetch_build_json try: buildbot.fetch_build_json = fetch latest = buildbot.latest_builder_info_and_alerts_for_master( self.cache, master0_url, master0)[0] self.assertEqual(latest, {}) finally: buildbot.fetch_build_json = old_fetch
def __call__(self, master_url): try: master_json = buildbot.fetch_master_json(master_url) if not master_json: return (None, None, None, master_url) master_alerts, stale_master_alert = alert_builder.alerts_for_master( self._cache, master_url, master_json, self._old_alerts, self._builder_filter, self._jobs) # FIXME: The builder info doesn't really belong here. The builder # revisions tool uses this and we happen to have the builder json cached # at this point so it's cheap to compute, but it should be moved # to a different feed. data, stale_builder_alerts = ( buildbot.latest_builder_info_and_alerts_for_master( self._cache, master_url, master_json)) if stale_master_alert: stale_builder_alerts.append(stale_master_alert) return (master_alerts, data, stale_builder_alerts, master_url) except: # Put all exception text into an exception and raise that so it doesn't # get eaten by the multiprocessing code. msg = '%s for master url %s' % ( ''.join(traceback.format_exception(*sys.exc_info())), master_url, ) raise Exception(msg)
def test_latest_builder_info_and_alerts_for_master(self): k_example_master_json = { "builders": { "Win Builder": { "basedir": "Win_Builder", "cachedBuilds": [0, 1770, 1771], "category": "2windows", "currentBuilds": [1772], "pendingBuilds": 7, "slaves": ["build2-m1"], "state": "building" }, "Not Running Builder": { "basedir": "Not_Running_Builder", "cachedBuilds": [], "category": "2windows", "currentBuilds": [], "pendingBuilds": 7, "slaves": ["build2-m1"], "state": "idle" } } } def mock_fetch_build_json(_cache, _master_url, _builder_name, _build_number): k_example_build_json = { "blame": [ "[email protected]@0039d316-1c4b-4281-b951-d872f2087c98", "[email protected]@0039d316-1c4b-4281-b951-d872f2087c98" ], "builderName": "Win Builder", "currentStep": None, "eta": None, "number": 1771, "pendingBuilds": 7, "properties": [ [ "build_archive_url", ("gs://chromium-win-archive/chromium.win/" "Win Builder/full-build-win32_289623.zip"), "Annotation(package build)" ], [ "buildbotURL", "http://build.chromium.org/p/chromium.win/", "master.cfg" ], [ "buildername", "Win Builder", "Builder" ], [ "buildnumber", 1771, "Build" ], [ "git_revision", "7ddb6d39574175cdd237eca54537e84fb960d3b8", "Change" ], [ "got_nacl_revision_cp", "refs/heads/master@{#13611}", "Annotation(bot_update)" ], [ "got_nacl_revision", "570e50beb76a2bdf6be4b345cbd47f225caf90af", "Annotation(bot_update)" ], [ "got_revision_cp", "refs/heads/master@{#289623}", "Annotation(bot_update)" ], [ "got_revision", "7ddb6d39574175cdd237eca54537e84fb960d3b8", "Annotation(bot_update)" ], [ "got_swarming_client_revision", "bbf1fcca7932d92cca9d7dab46ea271a7f6d61fb", "Annotation(bot_update)" ], [ "got_v8_revision_cp", "refs/heads/master@{#23117}", "Annotation(bot_update)" ], [ "got_v8_revision", "f284b29e37d97d7ee9128055862179dcbda7e398", "Annotation(bot_update)" ], [ "got_webkit_revision_cp", "refs/heads/master@{#180191}", "Annotation(bot_update)" ], [ "got_webkit_revision", "9df9a9e66fed3921ec1f620f92ea7333a9c18122", "Annotation(bot_update)" ], [ "got_webrtc_revision_cp", "refs/heads/master@{#6886}", "Annotation(bot_update)" ], [ "got_webrtc_revision", "c2ef523233552340785557abce1129a0f61537eb", "Annotation(bot_update)" ], [ "mastername", "chromium.win", "master.cfg" ] ], "steps": [ { "eta": None, "expectations": [ [ "output", 863225, 748982.2582168579 ] ], "hidden": False, "isFinished": True, "isStarted": True, "logs": [ [ "preamble", ("http://build.chromium.org/p/chromium.win/builders/"\ "Win%20Builder/builds/1771/steps/steps/logs/preamble") ], [ "stdio", ("http://build.chromium.org/p/chromium.win/builders/"\ "Win%20Builder/builds/1771/steps/steps/logs/stdio") ] ], "name": "steps", "results": [ 0, [] ], "step_number": 0, "text": [ "running steps via annotated script" ], "times": [ 3.3, 4.4 ], "urls": {} }, { "eta": None, "expectations": [], "hidden": False, "isFinished": True, "isStarted": True, "logs": [ [ "stdio", ("http://build.chromium.org/p/chromium.win/builders/"\ "Win%20Builder/builds/1771/steps/setup_build/logs/stdio") ], [ "run_recipe", ("http://build.chromium.org/p/chromium.win/builders/"\ "Win%20Builder/builds/1771/steps/setup_build/logs/run_recipe") ] ], "name": "setup_build", "results": [ 0, [] ], "step_number": 3, "text": [ "setup_build", "<br/>running recipe: \"chromium\"" ], "times": [ 5.5, 6.6 ], "urls": {} } ], "text": [ "build", "successful" ], "times": [ 1.1, 2.2 ], } return k_example_build_json, 'chrome-build-extract' old_fetch_build_json = buildbot.fetch_build_json try: buildbot.fetch_build_json = mock_fetch_build_json builder_info = buildbot.latest_builder_info_and_alerts_for_master( self.cache, 'http://build.chromium.org/p/chromium.webkit', k_example_master_json)[0] expected_builder_info = { 'chromium.webkit': { 'Win Builder': { 'state': 'building', 'monitor_url': 'https://chrome-monitor.appspot.com/view_graph' + '/chromium.webkit Win Builder Times (Last 100 Builds)', 'build_source': 'chrome-build-extract', 'lastUpdateTime': 2.2, 'revisions': { 'v8': 23117, 'chromium': 289623, 'nacl': 13611, 'blink': 180191 } } } } self.assertEqual(builder_info, expected_builder_info) finally: buildbot.fetch_build_json = old_fetch_build_json
def test_builder_info_for_master(self): """ Tests latest_builder_info_and_alerts_for_master. We have to pre-fill the build json cache to avoid this test hitting the network, which accounts for much of the complexity here. """ def cache_set(master_url, builder, build, value): key = buildbot.cache_key_for_build(master_url, builder, build) self.cache.set(key, value) def build(index): commit_position = 'refs/heads/master@{#%d}' return { 'properties': [['got_revision_cp', commit_position % (100 + index)], ['got_webkit_revision_cp', commit_position % (200 + index)], ['got_v8_revision_cp', commit_position % (300 + index)], ['got_nacl_revision_cp', commit_position % (400 + index)]], 'times': [1.0, 2.0], 'index': index } builds = map(build, range(0, 10)) master0_url = 'http://foo/bar/master0' master0 = { 'builders': { 'builder0': { 'cachedBuilds': [0, 1], 'currentBuilds': [2], 'state': 'happy', "pendingBuilds": 1, }, 'builder1': { 'cachedBuilds': [2, 3, 4, 5, 7], 'currentBuilds': [4, 5, 6, 7], 'state': 'sad', "pendingBuilds": 1, } } } # set up the cache so we don't hit the network for b in builds: cache_set(master0_url, 'builder0', b['index'], b) cache_set(master0_url, 'builder1', b['index'], b) latest = buildbot.latest_builder_info_and_alerts_for_master( self.cache, master0_url, master0)[0] self.assertIn('master0', latest) self.assertIn('builder0', latest['master0']) self.assertIn('builder1', latest['master0']) # b0's latest build is 1 b0 = latest['master0']['builder0']['revisions'] self.assertEqual(b0['chromium'], 102) self.assertEqual(b0['blink'], 202) self.assertEqual(b0['v8'], 302) self.assertEqual(b0['nacl'], 402) # b1's latest build is 7 b1 = latest['master0']['builder1']['revisions'] self.assertEqual(b1['chromium'], 107) self.assertEqual(b1['blink'], 207) self.assertEqual(b1['v8'], 307) self.assertEqual(b1['nacl'], 407)
def test_latest_builder_info_and_alerts_for_master(self): k_example_master_json = { "builders": { "Win Builder": { "basedir": "Win_Builder", "cachedBuilds": [ 0, 1770, 1771 ], "category": "2windows", "currentBuilds": [ 1772 ], "pendingBuilds": 7, "slaves": [ "build2-m1" ], "state": "building" }, "Not Running Builder": { "basedir": "Not_Running_Builder", "cachedBuilds": [ ], "category": "2windows", "currentBuilds": [ ], "pendingBuilds": 7, "slaves": [ "build2-m1" ], "state": "idle" } } } def mock_fetch_build_json(_cache, _master_url, _builder_name, _build_number): k_example_build_json = { "blame": [ "[email protected]@0039d316-1c4b-4281-b951-d872f2087c98", "[email protected]@0039d316-1c4b-4281-b951-d872f2087c98" ], "builderName": "Win Builder", "currentStep": None, "eta": None, "number": 1771, "pendingBuilds": 7, "properties": [ [ "build_archive_url", ("gs://chromium-win-archive/chromium.win/" "Win Builder/full-build-win32_289623.zip"), "Annotation(package build)" ], [ "buildbotURL", "http://build.chromium.org/p/chromium.win/", "master.cfg" ], [ "buildername", "Win Builder", "Builder" ], [ "buildnumber", 1771, "Build" ], [ "git_revision", "7ddb6d39574175cdd237eca54537e84fb960d3b8", "Change" ], [ "got_nacl_revision_cp", "refs/heads/master@{#13611}", "Annotation(bot_update)" ], [ "got_nacl_revision", "570e50beb76a2bdf6be4b345cbd47f225caf90af", "Annotation(bot_update)" ], [ "got_revision_cp", "refs/heads/master@{#289623}", "Annotation(bot_update)" ], [ "got_revision", "7ddb6d39574175cdd237eca54537e84fb960d3b8", "Annotation(bot_update)" ], [ "got_swarming_client_revision", "bbf1fcca7932d92cca9d7dab46ea271a7f6d61fb", "Annotation(bot_update)" ], [ "got_v8_revision_cp", "refs/heads/master@{#23117}", "Annotation(bot_update)" ], [ "got_v8_revision", "f284b29e37d97d7ee9128055862179dcbda7e398", "Annotation(bot_update)" ], [ "got_webkit_revision_cp", "refs/heads/master@{#180191}", "Annotation(bot_update)" ], [ "got_webkit_revision", "9df9a9e66fed3921ec1f620f92ea7333a9c18122", "Annotation(bot_update)" ], [ "got_webrtc_revision_cp", "refs/heads/master@{#6886}", "Annotation(bot_update)" ], [ "got_webrtc_revision", "c2ef523233552340785557abce1129a0f61537eb", "Annotation(bot_update)" ], [ "mastername", "chromium.win", "master.cfg" ] ], "steps": [ { "eta": None, "expectations": [ [ "output", 863225, 748982.2582168579 ] ], "hidden": False, "isFinished": True, "isStarted": True, "logs": [ [ "preamble", ("http://build.chromium.org/p/chromium.win/builders/"\ "Win%20Builder/builds/1771/steps/steps/logs/preamble") ], [ "stdio", ("http://build.chromium.org/p/chromium.win/builders/"\ "Win%20Builder/builds/1771/steps/steps/logs/stdio") ] ], "name": "steps", "results": [ 0, [] ], "step_number": 0, "text": [ "running steps via annotated script" ], "times": [ 3.3, 4.4 ], "urls": {} }, { "eta": None, "expectations": [], "hidden": False, "isFinished": True, "isStarted": True, "logs": [ [ "stdio", ("http://build.chromium.org/p/chromium.win/builders/"\ "Win%20Builder/builds/1771/steps/setup_build/logs/stdio") ], [ "run_recipe", ("http://build.chromium.org/p/chromium.win/builders/"\ "Win%20Builder/builds/1771/steps/setup_build/logs/run_recipe") ] ], "name": "setup_build", "results": [ 0, [] ], "step_number": 3, "text": [ "setup_build", "<br/>running recipe: \"chromium\"" ], "times": [ 5.5, 6.6 ], "urls": {} } ], "text": [ "build", "successful" ], "times": [ 1.1, 2.2 ], } return k_example_build_json, 'chrome-build-extract' old_fetch_build_json = buildbot.fetch_build_json try: buildbot.fetch_build_json = mock_fetch_build_json builder_info = buildbot.latest_builder_info_and_alerts_for_master( self.cache, 'http://build.chromium.org/p/chromium.webkit', k_example_master_json)[0] expected_builder_info = { 'chromium.webkit': { 'Win Builder': { 'state': 'building', 'monitor_url': 'https://chrome-monitor.appspot.com/view_graph' + '/chromium.webkit Win Builder Times (Last 100 Builds)', 'build_source': 'chrome-build-extract', 'lastUpdateTime': 2.2, 'revisions': { 'v8': 23117, 'chromium': 289623, 'nacl': 13611, 'blink': 180191 } } } } self.assertEqual(builder_info, expected_builder_info) finally: buildbot.fetch_build_json = old_fetch_build_json
def test_builder_info_for_master(self): """ Tests latest_builder_info_and_alerts_for_master. We have to pre-fill the build json cache to avoid this test hitting the network, which accounts for much of the complexity here. """ def cache_set(master_url, builder, build, value): key = buildbot.cache_key_for_build(master_url, builder, build) self.cache.set(key, value) def build(index): commit_position = 'refs/heads/master@{#%d}' return { 'properties': [ ['got_revision_cp', commit_position % (100 + index)], ['got_webkit_revision_cp', commit_position % (200 + index)], ['got_v8_revision_cp', commit_position % (300 + index)], ['got_nacl_revision_cp', commit_position % (400 + index)] ], 'times': [1.0, 2.0], 'index': index } builds = map(build, range(0, 10)) master0_url = 'http://foo/bar/master0' master0 = { 'builders': { 'builder0': { 'cachedBuilds': [0, 1], 'currentBuilds': [2], 'state': 'happy', "pendingBuilds": 1, }, 'builder1': { 'cachedBuilds': [2, 3, 4, 5, 7], 'currentBuilds': [4, 5, 6, 7], 'state': 'sad', "pendingBuilds": 1, } } } # set up the cache so we don't hit the network for b in builds: cache_set(master0_url, 'builder0', b['index'], b) cache_set(master0_url, 'builder1', b['index'], b) latest = buildbot.latest_builder_info_and_alerts_for_master( self.cache, master0_url, master0)[0] self.assertIn('master0', latest) self.assertIn('builder0', latest['master0']) self.assertIn('builder1', latest['master0']) # b0's latest build is 1 b0 = latest['master0']['builder0']['revisions'] self.assertEqual(b0['chromium'], 102) self.assertEqual(b0['blink'], 202) self.assertEqual(b0['v8'], 302) self.assertEqual(b0['nacl'], 402) # b1's latest build is 7 b1 = latest['master0']['builder1']['revisions'] self.assertEqual(b1['chromium'], 107) self.assertEqual(b1['blink'], 207) self.assertEqual(b1['v8'], 307) self.assertEqual(b1['nacl'], 407)