def testUpdateKeyDictWithDict(self):
expected_dict = {str(x): x for x in range(20)}
m = multiprocessing.Manager()
metadata = metadata_lib.CBuildbotMetadata(multiprocess_manager=m)
metadata.UpdateKeyDictWithDict('my_dict', expected_dict)
self.assertEqual(expected_dict, metadata.GetDict()['my_dict'])
def testUpdateKeyDictWithDictMultiprocess(self):
expected_dict = {str(x): x for x in range(20)}
m = multiprocessing.Manager()
metadata = metadata_lib.CBuildbotMetadata(multiprocess_manager=m)
with parallel.BackgroundTaskRunner(
metadata.UpdateKeyDictWithDict) as q:
for k, v in expected_dict.iteritems():
q.put(['my_dict', {k: v}])
self.assertEqual(expected_dict, metadata.GetDict()['my_dict'])
def testGetDict(self):
starting_dict = {
'key1': 1,
'key2': '2',
'cl_actions': [('a', 1), ('b', 2)],
'board-metadata': {
'board-1': {
'info': 432
},
},
}
metadata = metadata_lib.CBuildbotMetadata(starting_dict)
ending_dict = metadata.GetDict()
self.assertEqual(starting_dict, ending_dict)
def __init__(self, site_config, options, multiprocess_manager):
self.site_config = site_config
self.options = options
# Note that self.config is filled in dynamically by either of the classes
# that are actually instantiated: BuilderRun and ChildBuilderRun. In other
# words, self.config can be counted on anywhere except in this __init__.
# The implication is that any plain attributes that are calculated from
# self.config contents must be provided as properties (or methods).
# See the _RealBuilderRun class and its __getattr__ method for details.
self.config = None
# Create the RunAttributes object for this BuilderRun and save
# the id number for it in order to look it up via attrs property.
attrs = RunAttributes(multiprocess_manager)
self._ATTRS[id(attrs)] = attrs
self._attrs_id = id(attrs)
# Fill in values for all pre-computed "run configs" now, which are frozen
# by this time.
# TODO(mtennant): Should this use os.path.abspath like builderstage does?
self.buildroot = self.options.buildroot
self.buildnumber = self.options.buildnumber
self.manifest_branch = self.options.branch
# For remote_trybot runs, options.debug is implied, but we want true dryrun
# mode only if --debug was actually specified (i.e. options.debug_forced).
# TODO(mtennant): Get rid of confusing debug and debug_forced, if at all
# possible. Also, eventually use "dry_run" and "verbose" options instead to
# represent two distinct concepts.
self.debug = self.options.debug
if self.options.remote_trybot:
self.debug = self.options.debug_forced
# The __slots__ logic above confuses pylint.
# https://bitbucket.org/logilab/pylint/issue/380/
# pylint: disable=assigning-non-slot
# Certain run attributes have sensible defaults which can be set here.
# This allows all code to safely assume that the run attribute exists.
attrs.android_version = None
attrs.chrome_version = None
attrs.metadata = metadata_lib.CBuildbotMetadata(
multiprocess_manager=multiprocess_manager)
def testMasterSlaveVersionMismatch(self):
"""Test that master/slave version mismatch causes failure."""
master_release_tag = '9999.0.0-rc1'
master_build_id = self.fake_db.InsertBuild(
'master', constants.WATERFALL_INTERNAL, 2, 'master config',
'master hostname')
master_metadata = metadata_lib.CBuildbotMetadata()
master_metadata.UpdateKeyDictWithDict(
'version', {
'full': 'R39-9999.0.0-rc1',
'milestone': '39',
'platform': master_release_tag
})
self._run.attrs.metadata.UpdateWithDict(
{'master_build_id': master_build_id})
self.fake_db.UpdateMetadata(master_build_id, master_metadata)
stage = self.ConstructStage()
with self.assertRaises(failures_lib.StepFailure):
stage.Run()
def testPerBoardDict(self):
starting_per_board_dict = {
'board-1': {
'kubrick': 2001,
'bergman': 'persona',
'hitchcock': 'vertigo'
},
'board-2': {
'kubrick': ['barry lyndon', 'dr. strangelove'],
'bergman': 'the seventh seal'
}
}
starting_dict = {'board-metadata': starting_per_board_dict}
m = multiprocessing.Manager()
metadata = metadata_lib.CBuildbotMetadata(metadata_dict=starting_dict,
multiprocess_manager=m)
extra_per_board_dict = {
'board-1': {
'kurosawa': 'rashomon',
'coen brothers': 'fargo'
},
'board-3': {
'hitchcock': 'north by northwest',
'coen brothers': 'the big lebowski'
}
}
expected_dict = starting_per_board_dict
# Write each per board key-value pair to metadata in a separate process.
with parallel.BackgroundTaskRunner(
metadata.UpdateBoardDictWithDict) as q:
for board, board_dict in extra_per_board_dict.iteritems():
expected_dict.setdefault(board, {}).update(board_dict)
for k, v in board_dict.iteritems():
q.put([board, {k: v}])
self.assertEqual(expected_dict, metadata.GetDict()['board-metadata'])
def testMultiprocessSafety(self):
m = multiprocessing.Manager()
metadata = metadata_lib.CBuildbotMetadata(multiprocess_manager=m)
key_dict = {'key1': 1, 'key2': 2}
starting_dict = {
'key1': 1,
'key2': '2',
'key3': key_dict,
'cl_actions': [('a', 1), ('b', 2)],
'board-metadata': {
'board-1': {
'info': 432
},
},
}
# Test that UpdateWithDict is process-safe
parallel.RunParallelSteps(
[lambda: metadata.UpdateWithDict(starting_dict)])
ending_dict = metadata.GetDict()
self.assertEqual(starting_dict, ending_dict)
# Test that UpdateKeyDictWithDict is process-safe
parallel.RunParallelSteps(
[lambda: metadata.UpdateKeyDictWithDict('key3', key_dict)])
ending_dict = metadata.GetDict()
self.assertEqual(starting_dict, ending_dict)
# Test that RecordCLAction is process-safe
fake_change = metadata_lib.GerritPatchTuple(12345, 1, False)
fake_action = ('asdf,')
parallel.RunParallelSteps(
[lambda: metadata.RecordCLAction(fake_change, fake_action)])
ending_dict = metadata.GetDict()
# Assert that an action was recorded.
self.assertEqual(
len(starting_dict['cl_actions']) + 1,
len(ending_dict['cl_actions']))
def testUpdateBoardMetadataWithMultiprocessDict(self):
starting_dict = {
'key1': 1,
'key2': '2',
'cl_actions': [('a', 1), ('b', 2)],
'board-metadata': {
'board-1': {
'info': 432
},
},
}
m = multiprocessing.Manager()
metadata = metadata_lib.CBuildbotMetadata(metadata_dict=starting_dict,
multiprocess_manager=m)
# pylint: disable=no-member
update_dict = m.dict()
update_dict['my_key'] = 'some value'
metadata.UpdateBoardDictWithDict('board-1', update_dict)
self.assertEqual(
metadata.GetDict()['board-metadata']['board-1']['my_key'],
'some value')
def testUpdateBoardMetadataWithEmptyDict(self):
metadata = metadata_lib.CBuildbotMetadata()
metadata.UpdateBoardDictWithDict('someboard', {})
self.assertEqual(metadata.GetDict()['board-metadata']['someboard'], {})
def _PopulateFakeCidbWithTestData(self, cq):
"""Generate test data and insert it in the the fake cidb object.
Args:
cq: Whether this is a CQ run. If False, this is a Pre-CQ run.
"""
# Mock patches for test data.
c1p1 = metadata_lib.GerritPatchTuple(1, 1, False)
c2p1 = metadata_lib.GerritPatchTuple(2, 1, True)
c2p2 = metadata_lib.GerritPatchTuple(2, 2, True)
c3p1 = metadata_lib.GerritPatchTuple(3, 1, True)
c3p2 = metadata_lib.GerritPatchTuple(3, 2, True)
c4p1 = metadata_lib.GerritPatchTuple(4, 1, True)
c4p2 = metadata_lib.GerritPatchTuple(4, 2, True)
# Mock builder status dictionaries
passed_status = {'status': constants.FINAL_STATUS_PASSED}
failed_status = {'status': constants.FINAL_STATUS_FAILED}
t = datetime.datetime.now()
delta = datetime.timedelta(hours=1)
bot_config = (constants.CQ_MASTER
if cq else constants.PRE_CQ_DEFAULT_CONFIGS[0])
# pylint: disable=bad-continuation
test_metadata = [
# Build 1 picks up no patches.
metadata_lib.CBuildbotMetadata().UpdateWithDict({
'build-number':
1,
'bot-config':
bot_config,
'results': [],
'status':
passed_status
}),
# Build 2 picks up c1p1 and does nothing.
metadata_lib.CBuildbotMetadata().UpdateWithDict({
'build-number':
2,
'bot-config':
bot_config,
'results': [],
'status':
failed_status,
'changes': [c1p1._asdict()]
}).RecordCLAction(c1p1, constants.CL_ACTION_PICKED_UP, t + delta),
# Build 3 picks up c1p1 and c2p1 and rejects both.
# c3p1 is not included in the run because it fails to apply.
metadata_lib.CBuildbotMetadata().UpdateWithDict({
'build-number':
3,
'bot-config':
bot_config,
'results': [],
'status':
failed_status,
'changes': [c1p1._asdict(), c2p1._asdict()]
}).RecordCLAction(c1p1, constants.CL_ACTION_PICKED_UP,
t + delta).RecordCLAction(
c2p1, constants.CL_ACTION_PICKED_UP,
t + delta).RecordCLAction(
c1p1, constants.CL_ACTION_KICKED_OUT,
t + delta).RecordCLAction(
c2p1, constants.CL_ACTION_KICKED_OUT,
t + delta).RecordCLAction(
c3p1,
constants.CL_ACTION_KICKED_OUT,
t + delta),
# Build 4 picks up c4p1 and does nothing with it.
# c4p2 isn't picked up because it fails to apply.
metadata_lib.CBuildbotMetadata().UpdateWithDict({
'build-number':
3,
'bot-config':
bot_config,
'results': [],
'status':
failed_status,
'changes': [c4p1._asdict()]
}).RecordCLAction(c4p1, constants.CL_ACTION_PICKED_UP,
t + delta).RecordCLAction(
c4p2, constants.CL_ACTION_KICKED_OUT,
t + delta),
]
if cq:
test_metadata += [
# Build 4 picks up c1p1, c2p2, c3p2, c4p1 and submits the first three.
# c4p2 is submitted without being tested.
# So c1p1 should be detected as a 1-time rejected good patch,
# and c2p1 should be detected as a possibly bad patch.
metadata_lib.CBuildbotMetadata().UpdateWithDict({
'build-number':
4,
'bot-config':
bot_config,
'results': [],
'status':
passed_status,
'changes': [c1p1._asdict(), c2p2._asdict()]
}).RecordCLAction(
c1p1,
constants.CL_ACTION_PICKED_UP, t + delta).RecordCLAction(
c2p2, constants.CL_ACTION_PICKED_UP,
t + delta).RecordCLAction(
c3p2, constants.CL_ACTION_PICKED_UP,
t + delta).RecordCLAction(
c4p1, constants.CL_ACTION_PICKED_UP,
t + delta).RecordCLAction(
c1p1, constants.CL_ACTION_SUBMITTED,
t + delta).RecordCLAction(
c2p2, constants.CL_ACTION_SUBMITTED,
t + delta).RecordCLAction(
c3p2,
constants.CL_ACTION_SUBMITTED,
t + delta).RecordCLAction(
c4p2,
constants.CL_ACTION_SUBMITTED,
t + delta),
]
else:
test_metadata += [
metadata_lib.CBuildbotMetadata().UpdateWithDict({
'build-number':
5,
'bot-config':
bot_config,
'results': [],
'status':
failed_status,
'changes': [c4p1._asdict()]
}).RecordCLAction(c4p1, constants.CL_ACTION_PICKED_UP,
t + delta).RecordCLAction(
c4p1, constants.CL_ACTION_KICKED_OUT,
t + delta),
metadata_lib.CBuildbotMetadata().UpdateWithDict({
'build-number':
6,
'bot-config':
bot_config,
'results': [],
'status':
failed_status,
'changes': [c4p1._asdict()]
}).RecordCLAction(c1p1, constants.CL_ACTION_PICKED_UP,
t + delta).RecordCLAction(
c1p1, constants.CL_ACTION_KICKED_OUT,
t + delta)
]
# pylint: enable=bad-continuation
# test_metadata should not be guaranteed to be ordered by build number
# so shuffle it, but use the same seed each time so that unit test is
# deterministic.
random.seed(0)
random.shuffle(test_metadata)
for m in test_metadata:
build_id = self.fake_db.InsertBuild(m.GetValue('bot-config'),
constants.WATERFALL_INTERNAL,
m.GetValue('build-number'),
m.GetValue('bot-config'),
'bot-hostname')
m.UpdateWithDict({'build_id': build_id})
actions = []
for action_metadata in m.GetDict()['cl_actions']:
actions.append(
clactions.CLAction.FromMetadataEntry(action_metadata))
self.fake_db.InsertCLActions(build_id, actions)
