def makeBuilder(self, name="bldr", patch_random=False, noReconfig=False, **config_kwargs): """Set up C{self.bldr}""" # only include the necessary required config, plus user-requested config_args = dict(name=name, workername="slv", builddir="bdir", workerbuilddir="sbdir", factory=self.factory) config_args.update(config_kwargs) self.builder_config = config.BuilderConfig(**config_args) self.bldr = builder.Builder( self.builder_config.name, _addServices=False) self.bldr.master = self.master self.bldr.botmaster = self.master.botmaster # patch into the _startBuildsFor method self.builds_started = [] def _startBuildFor(workerforbuilder, buildrequests): self.builds_started.append((workerforbuilder, buildrequests)) return defer.succeed(True) self.bldr._startBuildFor = _startBuildFor if patch_random: # patch 'random.choice' to always take the worker that sorts # last, based on its name self.patch(random, "choice", lambda lst: sorted(lst, key=lambda m: m.name)[-1]) self.bldr.startService() mastercfg = config.MasterConfig() mastercfg.builders = [self.builder_config] if not noReconfig: defer.returnValue((yield self.bldr.reconfigServiceWithBuildbotConfig(mastercfg)))
def makeBuilder(self, patch_random=False, **config_kwargs): """Set up C{self.bldr}""" self.bstatus = mock.Mock() self.factory = mock.Mock() self.master = fakemaster.make_master() # only include the necessary required config, plus user-requested config = dict(name="bldr", slavename="slv", builddir="bdir", slavebuilddir="sbdir", factory=self.factory) config.update(config_kwargs) self.bldr = builder.Builder(config, self.bstatus) self.master.db = self.db = fakedb.FakeDBConnector(self) self.bldr.master = self.master self.bldr.botmaster = self.master.botmaster # patch into the _startBuildsFor method self.builds_started = [] def _startBuildFor(slavebuilder, buildrequests): self.builds_started.append((slavebuilder, buildrequests)) return defer.succeed(True) self.bldr._startBuildFor = _startBuildFor if patch_random: # patch 'random.choice' to always take the slave that sorts # last, based on its name self.patch(random, "choice", lambda lst : sorted(lst, key=lambda m : m.name)[-1]) # we don't want the reclaim service running during tests.. self.bldr.reclaim_svc.disownServiceParent() self.bldr.startService()
def makeBuilder(self, name="bldr", patch_random=False, **config_kwargs): """Set up C{self.bldr}""" self.factory = factory.BuildFactory() self.master = fakemaster.make_master() # only include the necessary required config, plus user-requested config_args = dict(name=name, slavename="slv", builddir="bdir", slavebuilddir="sbdir", factory=self.factory) config_args.update(config_kwargs) self.builder_config = config.BuilderConfig(**config_args) self.bldr = builder.Builder(self.builder_config.name, _addServices=False) self.master.db = self.db = fakedb.FakeDBConnector(self) self.bldr.master = self.master self.bldr.botmaster = self.master.botmaster # patch into the _startBuildsFor method self.builds_started = [] def _startBuildFor(slavebuilder, buildrequests): self.builds_started.append((slavebuilder, buildrequests)) return defer.succeed(True) self.bldr._startBuildFor = _startBuildFor if patch_random: # patch 'random.choice' to always take the slave that sorts # last, based on its name self.patch(random, "choice", lambda lst: sorted(lst, key=lambda m: m.name)[-1]) self.bldr.startService() mastercfg = config.MasterConfig() mastercfg.builders = [self.builder_config] return self.bldr.reconfigService(mastercfg)
def setUp(self): self.setUpTestReactor() self.master = fakemaster.make_master(self, wantData=True, wantMq=True, wantDb=True) self.master.db.insertTestData([ fakedb.Builder(id=80, name='test'), ]) self.builder = builder.Builder('test') self.builder._builderid = 80 self.builder.config_version = 0 self.builder.master = self.master self.builder.botmaster = mock.Mock() self.builder.botmaster.getLockFromLockAccesses = lambda l, c: [] yield self.builder.startService() self.factory = factory.BuildFactory() # will have steps added later new_config = config.MasterConfig() new_config.builders.append( config.BuilderConfig(name='test', workername='testworker', factory=self.factory)) yield self.builder.reconfigServiceWithBuildbotConfig(new_config) self.worker = Worker('worker', 'pass') self.worker.sendBuilderList = lambda: defer.succeed(None) self.worker.parent = mock.Mock() self.worker.master.botmaster = mock.Mock() self.worker.botmaster.maybeStartBuildsForWorker = lambda w: None self.worker.botmaster.getBuildersForWorker = lambda w: [] self.worker.parent = self.master self.worker.startService() self.conn = fakeprotocol.FakeConnection(self.master, self.worker) yield self.worker.attached(self.conn) wfb = self.workerforbuilder = workerforbuilder.WorkerForBuilder() wfb.setBuilder(self.builder) yield wfb.attached(self.worker, {}) # add the buildset/request self.bsid, brids = yield self.master.db.buildsets.addBuildset( sourcestamps=[{}], reason='x', properties={}, builderids=[80], waited_for=False) self.brdict = \ yield self.master.db.buildrequests.getBuildRequest(brids[80]) self.buildrequest = \ yield buildrequest.BuildRequest.fromBrdict(self.master, self.brdict)
def test_workers_old_api(self): bldr = builder.Builder('bldr') with assertNotProducesWarnings(DeprecatedWorkerAPIWarning): new = bldr.workers with assertProducesWarning( DeprecatedWorkerNameWarning, message_pattern="'slaves' attribute is deprecated"): old = bldr.slaves self.assertIdentical(new, old)
def test_addLatentWorker_old_api(self): bldr = builder.Builder('bldr') with assertProducesWarning( DeprecatedWorkerNameWarning, message_pattern="'addLatentSlave' method is deprecated"): method = mock.Mock(return_value='dummy') with mock.patch('buildbot.process.builder.Builder.addLatentWorker', method): dummy = bldr.addLatentSlave(mock.Mock()) self.assertEqual(dummy, 'dummy') self.assertTrue(method.called)
def makeBuilder(self, name="bldr", patch_random=False, noReconfig=False, builderid=None, **config_kwargs): """Set up C{self.bldr}""" self.factory = factory.BuildFactory() self.master = fakemaster.make_master(testcase=self, wantData=True) self.mq = self.master.mq self.db = self.master.db if builderid is None: b = fakedb.Builder(name=name) yield self.master.db.insertTestData([b]) builderid = b.id # only include the necessary required config, plus user-requested config_args = dict(name=name, slavename="slv", builddir="bdir", slavebuilddir="sbdir", factory=self.factory) config_args.update(config_kwargs) self.builder_config = config.BuilderConfig(**config_args) self.bldr = builder.Builder(self.builder_config.name, _addServices=False) self.bldr.master = self.master self.bldr.botmaster = self.master.botmaster fbi = self.master.data.updates.findBuilderId = mock.Mock(name='fbi') fbi.side_effect = lambda name: defer.succeed(builderid) # patch into the _startBuildsFor method self.builds_started = [] def _startBuildFor(slavebuilder, buildrequests): self.builds_started.append((slavebuilder, buildrequests)) return defer.succeed(True) self.bldr._startBuildFor = _startBuildFor if patch_random: # patch 'random.choice' to always take the slave that sorts # last, based on its name self.patch(random, "choice", lambda lst: sorted(lst, key=lambda m: m.name)[-1]) self.bldr.startService() mastercfg = config.MasterConfig() mastercfg.builders = [self.builder_config] if not noReconfig: defer.returnValue((yield self.bldr.reconfigService(mastercfg)))
def test_canStartWithWorkerForBuilder_old_api(self): bldr = builder.Builder('bldr', _addServices=False) bldr.config = mock.Mock() bldr.config.locks = [] with assertProducesWarning( DeprecatedWorkerNameWarning, message_pattern="'canStartWithSlavebuilder' method is deprecated"): with mock.patch( 'buildbot.process.build.Build.canStartWithWorkerForBuilder', mock.Mock(return_value='dummy')): dummy = bldr.canStartWithSlavebuilder(mock.Mock()) self.assertEqual(dummy, 'dummy')
def test_getAvailableWorkers_old_api(self): bldr = builder.Builder('bldr', _addServices=False) with assertProducesWarning( DeprecatedWorkerNameWarning, message_pattern="'getAvailableSlaves' method is deprecated"): method = mock.Mock(return_value='dummy') with mock.patch( 'buildbot.process.builder.Builder.getAvailableWorkers', method): dummy = bldr.getAvailableSlaves(mock.Mock()) self.assertEqual(dummy, 'dummy') self.assertTrue(method.called)
def setUp(self): self.master = fakemaster.make_master(testcase=self, wantData=True, wantMq=True, wantDb=True) self.master.db.insertTestData([ fakedb.Builder(id=80, name='test'), ]) self.builder = builder.Builder('test', _addServices=False) self.builder._builderid = 80 self.builder.master = self.master yield self.builder.startService() self.factory = factory.BuildFactory() # will have steps added later new_config = config.MasterConfig() new_config.builders.append( config.BuilderConfig(name='test', slavename='testsl', factory=self.factory)) yield self.builder.reconfigServiceWithBuildbotConfig(new_config) self.slave = BuildSlave('bsl', 'pass') self.slave.sendBuilderList = lambda: defer.succeed(None) self.slave.parent = mock.Mock() self.slave.master.botmaster = mock.Mock() self.slave.botmaster.maybeStartBuildsForSlave = lambda sl: None self.slave.botmaster.getBuildersForSlave = lambda sl: [] self.slave.parent = self.master self.slave.startService() self.conn = fakeprotocol.FakeConnection(self.master, self.slave) yield self.slave.attached(self.conn) sb = self.slavebuilder = slavebuilder.SlaveBuilder() sb.setBuilder(self.builder) yield sb.attached(self.slave, {}) # add the buildset/request self.bsid, brids = yield self.master.db.buildsets.addBuildset( sourcestamps=[{}], reason=u'x', properties={}, builderids=[80], waited_for=False) self.brdict = \ yield self.master.db.buildrequests.getBuildRequest(brids[80]) self.buildrequest = \ yield buildrequest.BuildRequest.fromBrdict(self.master, self.brdict)
def test_canStartWithWorkerForBuilder_no_buildrequests(self): bldr = builder.Builder('bldr') bldr.config = mock.Mock() bldr.config.locks = [] with assertProducesWarning( Warning, message_pattern=( "Not passing corresponding buildrequests to " "Builder.canStartWithWorkerForBuilder is deprecated")): with mock.patch( 'buildbot.process.build.Build.canStartWithWorkerForBuilder', mock.Mock(return_value='dummy')): dummy = yield bldr.canStartWithWorkerForBuilder(mock.Mock()) self.assertEqual(dummy, 'dummy')
def makeBuilder(self, name): self.bstatus = mock.Mock() self.factory = mock.Mock() self.master = fakemaster.make_master() # only include the necessary required config config = dict(name=name, slavename="slv", builddir="bdir", slavebuilddir="sbdir", factory=self.factory) self.bldr = builder.Builder(config, self.bstatus) self.master.db = self.db = fakedb.FakeDBConnector(self) self.bldr.master = self.master # we don't want the reclaim service running during tests.. self.bldr.reclaim_svc.disownServiceParent() self.bldr.startService()
def makeBuilder(self, name, sourcestamps): self.bstatus = mock.Mock() bstatus_properties = mock.Mock() bstatus_properties.properties = {} self.bstatus.getProperties.return_value = bstatus_properties self.bstatus.getSourceStamps.return_value = sourcestamps self.factory = mock.Mock() self.master = fakemaster.make_master() # only include the necessary required config builder_config = config.BuilderConfig( name=name, slavename="slv", builddir="bdir", slavebuilddir="sbdir", factory=self.factory) self.bldr = builder.Builder(builder_config.name) self.master.db = self.db = fakedb.FakeDBConnector(self) self.bldr.master = self.master self.bldr.master.master.addBuildset.return_value = (1, [100])
def test_getBuilderId(self): self.factory = factory.BuildFactory() self.master = fakemaster.make_master(testcase=self, wantData=True) # only include the necessary required config, plus user-requested self.bldr = builder.Builder('bldr', _addServices=False) self.bldr.master = self.master self.master.data.updates.findBuilderId = fbi = mock.Mock() fbi.return_value = defer.succeed(13) builderid = yield self.bldr.getBuilderId() self.assertEqual(builderid, 13) fbi.assert_called_with('bldr') fbi.reset_mock() builderid = yield self.bldr.getBuilderId() self.assertEqual(builderid, 13) fbi.assert_not_called()
def makeBuilder(self, name): self.bstatus = mock.Mock() self.factory = mock.Mock() self.master = fakemaster.make_master() # only include the necessary required config builder_config = config.BuilderConfig( name=name, slavename="slv", builddir="bdir", slavebuilddir="sbdir", factory=self.factory) self.bldr = builder.Builder(builder_config.name, _addServices=False) self.master.db = self.db = fakedb.FakeDBConnector(self) self.bldr.master = self.master self.bldr.startService() mastercfg = config.MasterConfig() mastercfg.builders = [ builder_config ] return self.bldr.reconfigService(mastercfg)
def setUp(self): self.master = fakemaster.make_master(testcase=self, wantDb=True) self.builder = builder.Builder('test', _addServices=False) self.builder.master = self.master yield self.builder.startService() self.factory = factory.BuildFactory() # will have steps added later new_config = config.MasterConfig() new_config.builders.append( config.BuilderConfig(name='test', slavename='testsl', factory=self.factory)) yield self.builder.reconfigService(new_config) self.slave = BuildSlave('bsl', 'pass') self.slave.sendBuilderList = lambda: defer.succeed(None) self.slave.botmaster = mock.Mock() self.slave.botmaster.maybeStartBuildsForSlave = lambda sl: None self.slave.master = self.master self.slave.startService() self.remote = FakeBot() yield self.slave.attached(self.remote) sb = self.slavebuilder = slavebuilder.SlaveBuilder() sb.setBuilder(self.builder) yield sb.attached(self.slave, self.remote, {}) # add the buildset/request sssid = yield self.master.db.sourcestampsets.addSourceStampSet() yield self.master.db.sourcestamps.addSourceStamp( branch='br', revision='1', repository='r://', project='', sourcestampsetid=sssid) self.bsid, brids = yield self.master.db.buildsets.addBuildset( sourcestampsetid=sssid, reason=u'x', properties={}, builderNames=['test']) self.brdict = \ yield self.master.db.buildrequests.getBuildRequest(brids['test']) self.buildrequest = \ yield buildrequest.BuildRequest.fromBrdict(self.master, self.brdict)
def test_canStartWithWorkerForBuilder_renderableLocks_no_buildrequests(self): bldr = builder.Builder('bldr') bldr.config = mock.Mock() @renderer def rendered_locks(props): return [] bldr.config.locks = rendered_locks with self.assertRaisesRegex( RuntimeError, 'buildrequests parameter must be specified .+'): with mock.patch( 'buildbot.process.build.Build.canStartWithWorkerForBuilder', mock.Mock(return_value='dummy')): yield bldr.canStartWithWorkerForBuilder(mock.Mock())
def test_canStartWithWorkerForBuilder_renderableLocks(self): @implementer(interfaces.IProperties) class FakeProperties(mock.Mock): def __iter__(self): return iter([]) bldr = builder.Builder('bldr') bldr.config = mock.Mock() bldr.botmaster = mock.Mock() bldr.botmaster.getLockFromLockAccess = mock.Mock(return_value='dummy') lock1 = mock.Mock(spec=locks.MasterLock) lock1.name = "masterlock" lock2 = mock.Mock(spec=locks.WorkerLock) lock2.name = "workerlock" renderedLocks = [False] @renderer def rendered_locks(props): renderedLocks[0] = True access1 = locks.LockAccess(lock1, 'counting') access2 = locks.LockAccess(lock2, 'exclusive') return [access1, access2] bldr.config.locks = rendered_locks with mock.patch( 'buildbot.process.build.Build.canStartWithWorkerForBuilder', mock.Mock(return_value='dummy')): with mock.patch( 'buildbot.process.build.Build.setupPropertiesKnownBeforeBuildStarts', mock.Mock()): dummy = yield bldr.canStartWithWorkerForBuilder(mock.Mock(), [mock.Mock()]) self.assertEqual(dummy, 'dummy') self.assertTrue(renderedLocks[0])
def createBuilder(self, name, slavenames=None, startSlavenames=None, maybeStartBuild=None, maybeResumeBuild=None, addRunningBuilds=False): bldr = builder.Builder(name, _addServices=False) build_factory = factory.BuildFactory() self.addRunningBuilds = addRunningBuilds def getSlaves(param): return param.keys() if list and isinstance(param, dict) else [] config_args = dict(name=name, builddir="bdir", slavebuilddir="sbdir", project='default', factory=build_factory) if slavenames: config_args['slavenames'] = getSlaves(slavenames) if startSlavenames: config_args['startSlavenames'] = getSlaves(startSlavenames) bldr.config = config.BuilderConfig(**config_args) bldr.maybeStartBuild = maybeStartBuild if maybeStartBuild else self.addProcessedBuilds bldr.maybeResumeBuild = maybeResumeBuild if maybeResumeBuild \ else lambda slavebuilder, buildnumber, breqs: self.addProcessedBuilds(slavebuilder, breqs) bldr.maybeUpdateMergedBuilds = self.addMergedBuilds self.addSlavesToList(bldr.slaves, slavenames) self.addSlavesToList(bldr.startSlaves, startSlavenames) return bldr
def main(args): if args.list_masters: masterpairs = get_masters() pp_masters(masterpairs) return 0 if args.master_dir: config = read_config(args.master_dir, args.master_cfg) else: path = choose_master(args.mastername) if not path: return 2 config = read_config(path, args.master_cfg) if not config: return 2 mastername = config['BuildmasterConfig']['properties']['mastername'] builders = dup_slaves(config['BuildmasterConfig']['builders']) if args.list_builders: pp_builders(builders, mastername) return 0 my_builder = choose(builders, args.spec) if args.spec and 'hostname' in args.spec: slavename = args.spec['hostname'] elif (args.spec and 'either' in args.spec) and (args.spec['either'] != my_builder['name']): slavename = args.spec['either'] else: slavename = my_builder['slavename'] if not my_builder: return 2 my_factory = my_builder['factory'] steplist = generate_steplist(my_factory) if args.list_steps: print print 'listing steps in %s/%s:' % (mastername, my_builder['name']) print for step in steplist: if hasattr(args, 'step_regex') and not args.step_regex.search(step.name): print '-', step.name, '[skipped]' elif hasattr(args, 'stepreject_regex') and (args.stepreject_regex.search( step.name)): print '-', step.name, '[skipped]' else: print '*', step.name return 0 if not args.annotate: print >> sys.stderr, 'using %s builder \'%s\'' % (mastername, my_builder['name']) if args.build_properties: buildsetup = args.build_properties else: buildsetup = {} buildsetup['revision'] = '%d' % args.revision buildsetup['branch'] = 'src' build = base.Build([FakeRequest(buildsetup)]) safename = buildbot.util.safeTranslate(my_builder['name']) if hasattr(args, 'builderpath'): basepath = args.builderpath else: basepath = safename basedir = os.path.join('..', '..', '..', 'slave', basepath) build.basedir = basedir builderstatus = builder.BuilderStatus('test') builderstatus.nextBuildNumber = 2 builderstatus.basedir = basedir my_builder['builddir'] = safename my_builder['slavebuilddir'] = safename mybuilder = real_builder.Builder(my_builder, builderstatus) build.setBuilder(mybuilder) build_status = build_module.BuildStatus(builderstatus, 1) build_status.setProperty('blamelist', [], 'Build') build_status.setProperty('mastername', mastername, 'Build') build_status.setProperty('slavename', slavename, 'Build') build_status.setProperty('gtest_filter', [], 'Build') # if build_properties are set on the CLI, overwrite the defaults # set above when build.setupProperties is called buildprops = Properties() if args.build_properties: buildprops.update(args.build_properties, 'Botmaster') mybuilder.setBotmaster(FakeBotmaster(mastername, buildprops)) mylogger = LogClass(args.log) buildslave = FakeSlave(safename, slavename) buildslave.addUpdateAction(mylogger.log_to_file) build.build_status = build_status build.setupSlaveBuilder(buildslave) build.setupProperties() if args.output_build_properties: print print 'build properties:' print propertiesToJSON(build.getProperties()) if args.output_factory_properties: print print 'factory properties:' print propertiesToJSON(my_factory.properties) if args.output_build_properties or args.output_factory_properties: return 0 process_steps(steplist, build, buildslave, build_status, basedir) commands = get_commands(steplist) run_status = ReturnStatus() start_time = time.clock() commands_executed = 0 for command in commands: if hasattr(args, 'step_regex'): if not args.step_regex.search(command['name']): if not args.annotate: print >> sys.stderr, 'skipping step: ' + command['name'] continue if hasattr(args, 'stepreject_regex'): if args.stepreject_regex.search(command['name']): if not args.annotate: print >> sys.stderr, 'skipping step: ' + command['name'] continue if not args.annotate: print >> sys.stderr, 'running step: %s' % command['name'] else: print '@@@BUILD_STEP %s@@@' % command['name'] print >> args.log, '(in %s): %s' % (command['workdir'], shell_quote(command['command'])) mydir = os.getcwd() myenv = os.environ os.chdir(command['workdir']) # python docs says this might cause leaks on FreeBSD/OSX for envar in command['env']: os.environ[envar] = command['env'][envar] ret = chromium_utils.RunCommand(command['command'], filter_obj=mylogger, print_cmd=False) os.chdir(mydir) os.environ = myenv commands_executed += 1 if ret != 0: return 2 end_time = time.clock() if not args.annotate: print >> sys.stderr, '%d commands completed (%0.2fs).' % ( commands_executed, end_time - start_time) else: if commands_executed < 1: print '0 commands executed.' return run_status.code
def MockBuild(my_builder, buildsetup, mastername, slavename, basepath=None, build_properties=None, slavedir=None): """Given a builder object and configuration, mock a Buildbot setup around it. This sets up a mock BuildMaster, BuildSlave, Build, BuildStatus, and all other superstructure required for BuildSteps inside the provided builder to render properly. These BuildSteps are returned to the user in an array. It additionally returns the build object (in order to get its properties if desired). buildsetup is passed straight into the FakeSource's init method and contains sourcestamp information (revision, branch, etc). basepath is the directory of the build (what goes under build/slave/, for example 'Chromium_Linux_Builder'. It is nominally inferred from the builder name, but it can be overridden. This is useful when pointing the buildrunner at a different builder than what it's running under. build_properties will update and override build_properties after all builder-derived defaults have been set. """ my_factory = my_builder['factory'] steplist = ListSteps(my_factory) build = base.Build([FakeRequest(buildsetup)]) safename = buildbot.util.safeTranslate(my_builder['name']) my_builder['builddir'] = safename my_builder.setdefault('slavebuilddir', safename) workdir_root = None if not slavedir: workdir_root = os.path.join(SCRIPT_DIR, '..', '..', 'slave', my_builder['slavebuilddir']) if not basepath: basepath = safename if not slavedir: slavedir = os.path.join(SCRIPT_DIR, '..', '..', 'slave') basedir = os.path.join(slavedir, basepath) build.basedir = basedir if not workdir_root: workdir_root = basedir builderstatus = builder.BuilderStatus('test') builderstatus.basedir = basedir buildnumber = build_properties.get('buildnumber', 1) builderstatus.nextBuildNumber = buildnumber + 1 mybuilder = real_builder.Builder(my_builder, builderstatus) build.setBuilder(mybuilder) build_status = build_module.BuildStatus(builderstatus, buildnumber) build_status.setProperty('blamelist', [], 'Build') build_status.setProperty('mastername', mastername, 'Build') build_status.setProperty('slavename', slavename, 'Build') build_status.setProperty('gtest_filter', [], 'Build') build_status.setProperty('extra_args', [], 'Build') build_status.setProperty('build_id', buildnumber, 'Build') # if build_properties are passed in, overwrite the defaults above: buildprops = Properties() if build_properties: buildprops.update(build_properties, 'Botmaster') mybuilder.setBotmaster(FakeBotmaster(mastername, buildprops)) buildslave = FakeSlave(safename, my_builder.get('slavebuilddir'), slavename) build.build_status = build_status build.setupSlaveBuilder(buildslave) build.setupProperties() process_steps(steplist, build, buildslave, build_status, workdir_root) return steplist, build