def testUpdateKeyDictWithDict(self): expected_dict = {str(x): x for x in range(20)} m = multiprocessing.Manager() metadata = metadata_lib.CBuildbotMetadata(multiprocess_manager=m) metadata.UpdateKeyDictWithDict('my_dict', expected_dict) self.assertEqual(expected_dict, metadata.GetDict()['my_dict'])
def testUpdateKeyDictWithDictMultiprocess(self): expected_dict = {str(x): x for x in range(20)} m = multiprocessing.Manager() metadata = metadata_lib.CBuildbotMetadata(multiprocess_manager=m) with parallel.BackgroundTaskRunner( metadata.UpdateKeyDictWithDict) as q: for k, v in expected_dict.iteritems(): q.put(['my_dict', {k: v}]) self.assertEqual(expected_dict, metadata.GetDict()['my_dict'])
def testGetDict(self): starting_dict = { 'key1': 1, 'key2': '2', 'cl_actions': [('a', 1), ('b', 2)], 'board-metadata': { 'board-1': { 'info': 432 }, }, } metadata = metadata_lib.CBuildbotMetadata(starting_dict) ending_dict = metadata.GetDict() self.assertEqual(starting_dict, ending_dict)
def __init__(self, site_config, options, multiprocess_manager): self.site_config = site_config self.options = options # Note that self.config is filled in dynamically by either of the classes # that are actually instantiated: BuilderRun and ChildBuilderRun. In other # words, self.config can be counted on anywhere except in this __init__. # The implication is that any plain attributes that are calculated from # self.config contents must be provided as properties (or methods). # See the _RealBuilderRun class and its __getattr__ method for details. self.config = None # Create the RunAttributes object for this BuilderRun and save # the id number for it in order to look it up via attrs property. attrs = RunAttributes(multiprocess_manager) self._ATTRS[id(attrs)] = attrs self._attrs_id = id(attrs) # Fill in values for all pre-computed "run configs" now, which are frozen # by this time. # TODO(mtennant): Should this use os.path.abspath like builderstage does? self.buildroot = self.options.buildroot self.buildnumber = self.options.buildnumber self.manifest_branch = self.options.branch # For remote_trybot runs, options.debug is implied, but we want true dryrun # mode only if --debug was actually specified (i.e. options.debug_forced). # TODO(mtennant): Get rid of confusing debug and debug_forced, if at all # possible. Also, eventually use "dry_run" and "verbose" options instead to # represent two distinct concepts. self.debug = self.options.debug if self.options.remote_trybot: self.debug = self.options.debug_forced # The __slots__ logic above confuses pylint. # https://bitbucket.org/logilab/pylint/issue/380/ # pylint: disable=assigning-non-slot # Certain run attributes have sensible defaults which can be set here. # This allows all code to safely assume that the run attribute exists. attrs.android_version = None attrs.chrome_version = None attrs.metadata = metadata_lib.CBuildbotMetadata( multiprocess_manager=multiprocess_manager)
def testMasterSlaveVersionMismatch(self): """Test that master/slave version mismatch causes failure.""" master_release_tag = '9999.0.0-rc1' master_build_id = self.fake_db.InsertBuild( 'master', constants.WATERFALL_INTERNAL, 2, 'master config', 'master hostname') master_metadata = metadata_lib.CBuildbotMetadata() master_metadata.UpdateKeyDictWithDict( 'version', { 'full': 'R39-9999.0.0-rc1', 'milestone': '39', 'platform': master_release_tag }) self._run.attrs.metadata.UpdateWithDict( {'master_build_id': master_build_id}) self.fake_db.UpdateMetadata(master_build_id, master_metadata) stage = self.ConstructStage() with self.assertRaises(failures_lib.StepFailure): stage.Run()
def testPerBoardDict(self): starting_per_board_dict = { 'board-1': { 'kubrick': 2001, 'bergman': 'persona', 'hitchcock': 'vertigo' }, 'board-2': { 'kubrick': ['barry lyndon', 'dr. strangelove'], 'bergman': 'the seventh seal' } } starting_dict = {'board-metadata': starting_per_board_dict} m = multiprocessing.Manager() metadata = metadata_lib.CBuildbotMetadata(metadata_dict=starting_dict, multiprocess_manager=m) extra_per_board_dict = { 'board-1': { 'kurosawa': 'rashomon', 'coen brothers': 'fargo' }, 'board-3': { 'hitchcock': 'north by northwest', 'coen brothers': 'the big lebowski' } } expected_dict = starting_per_board_dict # Write each per board key-value pair to metadata in a separate process. with parallel.BackgroundTaskRunner( metadata.UpdateBoardDictWithDict) as q: for board, board_dict in extra_per_board_dict.iteritems(): expected_dict.setdefault(board, {}).update(board_dict) for k, v in board_dict.iteritems(): q.put([board, {k: v}]) self.assertEqual(expected_dict, metadata.GetDict()['board-metadata'])
def testMultiprocessSafety(self): m = multiprocessing.Manager() metadata = metadata_lib.CBuildbotMetadata(multiprocess_manager=m) key_dict = {'key1': 1, 'key2': 2} starting_dict = { 'key1': 1, 'key2': '2', 'key3': key_dict, 'cl_actions': [('a', 1), ('b', 2)], 'board-metadata': { 'board-1': { 'info': 432 }, }, } # Test that UpdateWithDict is process-safe parallel.RunParallelSteps( [lambda: metadata.UpdateWithDict(starting_dict)]) ending_dict = metadata.GetDict() self.assertEqual(starting_dict, ending_dict) # Test that UpdateKeyDictWithDict is process-safe parallel.RunParallelSteps( [lambda: metadata.UpdateKeyDictWithDict('key3', key_dict)]) ending_dict = metadata.GetDict() self.assertEqual(starting_dict, ending_dict) # Test that RecordCLAction is process-safe fake_change = metadata_lib.GerritPatchTuple(12345, 1, False) fake_action = ('asdf,') parallel.RunParallelSteps( [lambda: metadata.RecordCLAction(fake_change, fake_action)]) ending_dict = metadata.GetDict() # Assert that an action was recorded. self.assertEqual( len(starting_dict['cl_actions']) + 1, len(ending_dict['cl_actions']))
def testUpdateBoardMetadataWithMultiprocessDict(self): starting_dict = { 'key1': 1, 'key2': '2', 'cl_actions': [('a', 1), ('b', 2)], 'board-metadata': { 'board-1': { 'info': 432 }, }, } m = multiprocessing.Manager() metadata = metadata_lib.CBuildbotMetadata(metadata_dict=starting_dict, multiprocess_manager=m) # pylint: disable=no-member update_dict = m.dict() update_dict['my_key'] = 'some value' metadata.UpdateBoardDictWithDict('board-1', update_dict) self.assertEqual( metadata.GetDict()['board-metadata']['board-1']['my_key'], 'some value')
def testUpdateBoardMetadataWithEmptyDict(self): metadata = metadata_lib.CBuildbotMetadata() metadata.UpdateBoardDictWithDict('someboard', {}) self.assertEqual(metadata.GetDict()['board-metadata']['someboard'], {})
def _PopulateFakeCidbWithTestData(self, cq): """Generate test data and insert it in the the fake cidb object. Args: cq: Whether this is a CQ run. If False, this is a Pre-CQ run. """ # Mock patches for test data. c1p1 = metadata_lib.GerritPatchTuple(1, 1, False) c2p1 = metadata_lib.GerritPatchTuple(2, 1, True) c2p2 = metadata_lib.GerritPatchTuple(2, 2, True) c3p1 = metadata_lib.GerritPatchTuple(3, 1, True) c3p2 = metadata_lib.GerritPatchTuple(3, 2, True) c4p1 = metadata_lib.GerritPatchTuple(4, 1, True) c4p2 = metadata_lib.GerritPatchTuple(4, 2, True) # Mock builder status dictionaries passed_status = {'status': constants.FINAL_STATUS_PASSED} failed_status = {'status': constants.FINAL_STATUS_FAILED} t = datetime.datetime.now() delta = datetime.timedelta(hours=1) bot_config = (constants.CQ_MASTER if cq else constants.PRE_CQ_DEFAULT_CONFIGS[0]) # pylint: disable=bad-continuation test_metadata = [ # Build 1 picks up no patches. metadata_lib.CBuildbotMetadata().UpdateWithDict({ 'build-number': 1, 'bot-config': bot_config, 'results': [], 'status': passed_status }), # Build 2 picks up c1p1 and does nothing. metadata_lib.CBuildbotMetadata().UpdateWithDict({ 'build-number': 2, 'bot-config': bot_config, 'results': [], 'status': failed_status, 'changes': [c1p1._asdict()] }).RecordCLAction(c1p1, constants.CL_ACTION_PICKED_UP, t + delta), # Build 3 picks up c1p1 and c2p1 and rejects both. # c3p1 is not included in the run because it fails to apply. metadata_lib.CBuildbotMetadata().UpdateWithDict({ 'build-number': 3, 'bot-config': bot_config, 'results': [], 'status': failed_status, 'changes': [c1p1._asdict(), c2p1._asdict()] }).RecordCLAction(c1p1, constants.CL_ACTION_PICKED_UP, t + delta).RecordCLAction( c2p1, constants.CL_ACTION_PICKED_UP, t + delta).RecordCLAction( c1p1, constants.CL_ACTION_KICKED_OUT, t + delta).RecordCLAction( c2p1, constants.CL_ACTION_KICKED_OUT, t + delta).RecordCLAction( c3p1, constants.CL_ACTION_KICKED_OUT, t + delta), # Build 4 picks up c4p1 and does nothing with it. # c4p2 isn't picked up because it fails to apply. metadata_lib.CBuildbotMetadata().UpdateWithDict({ 'build-number': 3, 'bot-config': bot_config, 'results': [], 'status': failed_status, 'changes': [c4p1._asdict()] }).RecordCLAction(c4p1, constants.CL_ACTION_PICKED_UP, t + delta).RecordCLAction( c4p2, constants.CL_ACTION_KICKED_OUT, t + delta), ] if cq: test_metadata += [ # Build 4 picks up c1p1, c2p2, c3p2, c4p1 and submits the first three. # c4p2 is submitted without being tested. # So c1p1 should be detected as a 1-time rejected good patch, # and c2p1 should be detected as a possibly bad patch. metadata_lib.CBuildbotMetadata().UpdateWithDict({ 'build-number': 4, 'bot-config': bot_config, 'results': [], 'status': passed_status, 'changes': [c1p1._asdict(), c2p2._asdict()] }).RecordCLAction( c1p1, constants.CL_ACTION_PICKED_UP, t + delta).RecordCLAction( c2p2, constants.CL_ACTION_PICKED_UP, t + delta).RecordCLAction( c3p2, constants.CL_ACTION_PICKED_UP, t + delta).RecordCLAction( c4p1, constants.CL_ACTION_PICKED_UP, t + delta).RecordCLAction( c1p1, constants.CL_ACTION_SUBMITTED, t + delta).RecordCLAction( c2p2, constants.CL_ACTION_SUBMITTED, t + delta).RecordCLAction( c3p2, constants.CL_ACTION_SUBMITTED, t + delta).RecordCLAction( c4p2, constants.CL_ACTION_SUBMITTED, t + delta), ] else: test_metadata += [ metadata_lib.CBuildbotMetadata().UpdateWithDict({ 'build-number': 5, 'bot-config': bot_config, 'results': [], 'status': failed_status, 'changes': [c4p1._asdict()] }).RecordCLAction(c4p1, constants.CL_ACTION_PICKED_UP, t + delta).RecordCLAction( c4p1, constants.CL_ACTION_KICKED_OUT, t + delta), metadata_lib.CBuildbotMetadata().UpdateWithDict({ 'build-number': 6, 'bot-config': bot_config, 'results': [], 'status': failed_status, 'changes': [c4p1._asdict()] }).RecordCLAction(c1p1, constants.CL_ACTION_PICKED_UP, t + delta).RecordCLAction( c1p1, constants.CL_ACTION_KICKED_OUT, t + delta) ] # pylint: enable=bad-continuation # test_metadata should not be guaranteed to be ordered by build number # so shuffle it, but use the same seed each time so that unit test is # deterministic. random.seed(0) random.shuffle(test_metadata) for m in test_metadata: build_id = self.fake_db.InsertBuild(m.GetValue('bot-config'), constants.WATERFALL_INTERNAL, m.GetValue('build-number'), m.GetValue('bot-config'), 'bot-hostname') m.UpdateWithDict({'build_id': build_id}) actions = [] for action_metadata in m.GetDict()['cl_actions']: actions.append( clactions.CLAction.FromMetadataEntry(action_metadata)) self.fake_db.InsertCLActions(build_id, actions)