def testGet(self): # the master.cfg file contains "lock ids", which are instances of # MasterLock and SlaveLock but which are not actually Locks per se. # When the build starts, these markers are turned into RealMasterLock # and RealSlaveLock instances. This insures that any builds running # on slaves that were unaffected by the config change are still # referring to the same Lock instance as new builds by builders that # *were* affected by the change. There have been bugs in the past in # which this didn't happen, and the Locks were bypassed because half # the builders were using one incarnation of the lock while the other # half were using a separate (but equal) incarnation. # # Changing the lock id in any way should cause it to be replaced in # the BotMaster. This will result in a couple of funky artifacts: # builds in progress might pay attention to a different lock, so we # might bypass the locking for the duration of a couple builds. # There's also the problem of old Locks lingering around in # BotMaster.locks, but they're small and shouldn't really cause a # problem. b = master.BotMaster() l1 = locks.MasterLock("one") l1a = locks.MasterLock("one") l2 = locks.MasterLock("one", maxCount=4) rl1 = b.getLockByID(l1) rl2 = b.getLockByID(l1a) self.failUnlessIdentical(rl1, rl2) rl3 = b.getLockByID(l2) self.failIfIdentical(rl1, rl3) s1 = locks.SlaveLock("one") s1a = locks.SlaveLock("one") s2 = locks.SlaveLock("one", maxCount=4) s3 = locks.SlaveLock("one", maxCount=4, maxCountForSlave={"a":1, "b":2}) s3a = locks.SlaveLock("one", maxCount=4, maxCountForSlave={"a":1, "b":2}) s4 = locks.SlaveLock("one", maxCount=4, maxCountForSlave={"a":4, "b":4}) rl1 = b.getLockByID(s1) rl2 = b.getLockByID(s1a) self.failUnlessIdentical(rl1, rl2) rl3 = b.getLockByID(s2) self.failIfIdentical(rl1, rl3) rl4 = b.getLockByID(s3) self.failIfIdentical(rl1, rl4) self.failIfIdentical(rl3, rl4) rl5 = b.getLockByID(s3a) self.failUnlessIdentical(rl4, rl5) rl6 = b.getLockByID(s4) self.failIfIdentical(rl5, rl6)
def test_renderableLocks(self): master_lock = locks.MasterLock("masterlock") worker_lock = locks.WorkerLock("workerlock") lock_accesses = [] @renderer def rendered_locks(props): master_access = locks.LockAccess(master_lock, 'counting') worker_access = locks.LockAccess(worker_lock, 'exclusive') lock_accesses.append(master_access) lock_accesses.append(worker_access) return [master_access, worker_access] self.setupStep(self.LockBuildStep(testcase=self, lock_accesses=lock_accesses, locks=rendered_locks)) self.expectOutcome(result=SUCCESS) yield self.runStep() self.assertEqual(len(lock_accesses), 2) botmaster = self.step.build.builder.botmaster real_master_lock = yield botmaster.getLockFromLockAccess(lock_accesses[0], self.build.config_version) real_worker_lock = yield botmaster.getLockFromLockAccess(lock_accesses[1], self.build.config_version) self.assertTrue(real_master_lock.isAvailable(self, lock_accesses[0])) self.assertIn('workername', real_worker_lock.locks) self.assertTrue(real_worker_lock.locks['workername'].isAvailable(self, lock_accesses[1]))
def test_regularLocks(self): lock1 = locks.MasterLock("masterlock") lock2 = locks.WorkerLock("workerlock") self.setupStep(self.FakeBuildStep( locks=[locks.LockAccess(lock1, 'counting'), locks.LockAccess(lock2, 'exclusive')])) self.expectOutcome(result=SUCCESS) yield self.runStep()
def test_constructor_full(self): lock1, lock2 = locks.MasterLock('lock1'), locks.MasterLock('lock2') access1, access2 = lock1.access('counting'), lock2.access('counting') bs = yield self.createWorker('bot', 'pass', max_builds=2, notify_on_missing=['*****@*****.**'], missing_timeout=120, properties={'a': 'b'}, locks=[access1, access2]) yield bs.startService() self.assertEqual(bs.max_builds, 2) self.assertEqual(bs.notify_on_missing, ['*****@*****.**']) self.assertEqual(bs.missing_timeout, 120) self.assertEqual(bs.properties.getProperty('a'), 'b') self.assertEqual(bs.access, [access1, access2])
def testMaster(self): mid1 = locks.MasterLock("name1") mid2 = locks.MasterLock("name1") mid3 = locks.MasterLock("name3") mid4 = locks.MasterLock("name1", 3) self.failUnlessEqual(mid1, mid2) self.failIfEqual(mid1, mid3) # they should all be hashable d = {mid1: 1, mid2: 2, mid3: 3, mid4: 4} l1 = self.make(mid1) self.failUnlessEqual(l1.name, "name1") self.failUnlessEqual(l1.maxCount, 1) self.failUnlessIdentical(l1.getLock(slave("slave1")), l1) l4 = self.make(mid4) self.failUnlessEqual(l4.name, "name1") self.failUnlessEqual(l4.maxCount, 3) self.failUnlessIdentical(l4.getLock(slave("slave1")), l4)
def test_setServiceParent_masterLocks(self): """ http://trac.buildbot.net/ticket/2278 """ master = self.master = fakemaster.make_master() botmaster = FakeBotMaster(master) botmaster.startService() lock = locks.MasterLock('masterlock') bs = self.ConcreteBuildSlave('bot', 'pass', locks=[lock]) bs.setServiceParent(botmaster)
def test_setServiceParent_masterLocks(self): """ http://trac.buildbot.net/ticket/2278 """ master = self.master bm = botmaster.FakeBotMaster(master) bm.startService() lock = locks.MasterLock('masterlock') bs = ConcreteBuildSlave('bot', 'pass', locks=[lock.access("counting")]) bs.setServiceParent(bm)
def test_setServiceParent_masterLocks(self): """ http://trac.buildbot.net/ticket/2278 """ master = self.master bsmanager = master.workers yield master.startService() lock = locks.MasterLock('masterlock') bs = ConcreteWorker('bot', 'pass', locks=[lock.access("counting")]) bs.setServiceParent(bsmanager)
def testNowMixed2(self): """ Test counting is not possible when an exclsuive has the lock """ lid = locks.MasterLock('dummy') lac = locks.LockAccess(lid, 'counting') lae = locks.LockAccess(lid, 'exclusive') l = locks.BaseLock("name", maxCount=2) self.failUnless(l.isAvailable(lae)) l.claim("count-owner", lae) self.failIf(l.isAvailable(lac)) l.release("count-owner", lae) self.failUnless(l.isAvailable(lae))
def testLaterCounting(self): lid = locks.MasterLock('dummy') la = locks.LockAccess(lid, 'counting') lock = locks.BaseLock("name", 2) lock.claim("owner1", la) lock.claim("owner2", la) d = claimHarder(lock, "owner3", la) d.addCallback(lambda lock: lock.release("owner3", la)) lock.release("owner2", la) lock.release("owner1", la) return d
def testNowCounting(self): lid = locks.MasterLock('dummy') la = locks.LockAccess(lid, 'counting') lock = locks.BaseLock("name", 2) self.failUnless(lock.isAvailable(la)) lock.claim("owner1", la) self.failUnless(lock.isAvailable(la)) lock.claim("owner2", la) self.failIf(lock.isAvailable(la)) lock.release("owner1", la) self.failUnless(lock.isAvailable(la)) lock.release("owner2", la) self.failUnless(lock.isAvailable(la))
def test_renderableLocks(self): lock1 = locks.MasterLock("masterlock") lock2 = locks.WorkerLock("workerlock") renderedLocks = [False] @renderer def rendered_locks(props): renderedLocks[0] = True access1 = locks.LockAccess(lock1, 'counting') access2 = locks.LockAccess(lock2, 'exclusive') return [access1, access2] self.setupStep(self.FakeBuildStep(locks=rendered_locks)) self.expectOutcome(result=SUCCESS) yield self.runStep() self.assertTrue(renderedLocks[0])
def test_regularLocks(self): master_lock = locks.MasterLock("masterlock") worker_lock = locks.WorkerLock("workerlock") lock_accesses = [locks.LockAccess(master_lock, 'counting'), locks.LockAccess(worker_lock, 'exclusive')] self.setupStep(self.LockBuildStep(testcase=self, lock_accesses=lock_accesses, locks=lock_accesses)) self.expectOutcome(result=SUCCESS) yield self.runStep() botmaster = self.step.build.builder.botmaster real_master_lock = yield botmaster.getLockFromLockAccess(lock_accesses[0], self.build.config_version) real_worker_lock = yield botmaster.getLockFromLockAccess(lock_accesses[1], self.build.config_version) self.assertTrue(real_master_lock.isAvailable(self, lock_accesses[0])) self.assertIn('workername', real_worker_lock.locks) self.assertTrue(real_worker_lock.locks['workername'].isAvailable(self, lock_accesses[1]))
def testRandomCounting(self): lid = locks.MasterLock('dummy') la = locks.LockAccess(lid, 'counting') COUNT = 5 lock = locks.BaseLock("name", COUNT) dl = [] for i in range(100): owner = "owner%d" % i mode = random.choice(["now", "very soon", "soon"]) d = claimHarder(lock, owner, la) def _check(lock): self.failIf(len(lock.owners) > COUNT) return lock d.addCallback(_check) d.addCallback(hold, owner, la, mode) dl.append(d) d = defer.DeferredList(dl) d.addCallback(self._cleanup, lock, COUNT, la) return d
def __init__(self): self.workers = [] self.builders = [] self.auto_builder_names = [] self.local_builder_lock = locks.MasterLock("local", maxCount=2) self.deps_lock = locks.WorkerLock("deps", maxCount=1) # Add Linux workers and builders. for linux_distro, versions in CONFIG['linux'].items(): factory = LINUX_FACTORIES[linux_distro] for version in versions: self._AddBuilderAndWorker(linux_distro, version, factory) # Add Ubuntu PPA. for linux_distro in CONFIG['linux']['ubuntu']: self._AddBuilder( name='Ubuntu Stable PPA %s' % linux_distro.title(), worker='ubuntu-%s' % linux_distro, build_factory=builders.MakePPABuilder(linux_distro, "stable", PPA_STABLE)) self._AddBuilder( name='Ubuntu Unstable PPA %s' % linux_distro.title(), worker='ubuntu-%s' % linux_distro, build_factory=builders.MakePPABuilder(linux_distro, "unstable", PPA_UNSTABLE)) # Add special workers. for name in CONFIG['special_workers']: self._AddWorker(name) # Source. self._AddBuilder(name='Source', worker='opensuse-lp152', build_factory=builders.MakeSourceBuilder()) # AppImage. self._AddBuilder( name='AppImage Qt 5', worker='appimage-qt5', build_factory=builders.MakeAppImageBuilder(name="Qt5")) self._AddBuilder( name='AppImage Qt 6', worker='appimage-qt6', build_factory=builders.MakeAppImageBuilder(name="Qt6")) # MXE. self._AddBuilder(name='MXE', worker='mingw', build_factory=builders.MakeMXEBuilder(), auto=False, deps_lock='exclusive') # Windows. self._AddBuilder(name='Windows Release x86 Qt 5', worker='mingw', build_factory=builders.MakeWindowsBuilder( is_debug=False, is_64=False, with_qt6=False), deps_lock='counting') self._AddBuilder(name='Windows Release x64 Qt 5', worker='mingw', build_factory=builders.MakeWindowsBuilder( is_debug=False, is_64=True, with_qt6=False), deps_lock='counting') self._AddBuilder(name='Windows Debug x86 Qt 5', worker='mingw', build_factory=builders.MakeWindowsBuilder( is_debug=True, is_64=False, with_qt6=False), deps_lock='counting') self._AddBuilder(name='Windows Debug x64 Qt 5', worker='mingw', build_factory=builders.MakeWindowsBuilder( is_debug=True, is_64=True, with_qt6=False), deps_lock='counting') self._AddBuilder(name='Windows Release x86 Qt 6', worker='mingw', build_factory=builders.MakeWindowsBuilder( is_debug=False, is_64=False, with_qt6=True), deps_lock='counting') self._AddBuilder(name='Windows Release x64 Qt 6', worker='mingw', build_factory=builders.MakeWindowsBuilder( is_debug=False, is_64=True, with_qt6=True), deps_lock='counting') self._AddBuilder(name='Windows Debug x86 Qt 6', worker='mingw', build_factory=builders.MakeWindowsBuilder( is_debug=True, is_64=False, with_qt6=True), deps_lock='counting') self._AddBuilder(name='Windows Debug x64 Qt 6', worker='mingw', build_factory=builders.MakeWindowsBuilder( is_debug=True, is_64=True, with_qt6=True), deps_lock='counting')
from buildbot import locks slave_cpu_lock = locks.SlaveLock('slave_cpu_lock', maxCount=1) master_upload_lock = locks.MasterLock('master_upload_lock') def skipped_or_success(results, step): from buildbot.status.results import SKIPPED from buildbot.status.results import SUCCESS return ((results == SKIPPED) or (results == SUCCESS)) def skipped(results, step): from buildbot.status.results import SKIPPED return (results == SKIPPED) def get_cmake_step(link, type, options=[]): from buildbot.process.properties import Interpolate from buildbot.steps.shell import ShellCommand build_type = '' if type == 'debug': build_type += '-DCMAKE_BUILD_TYPE=Debug' else: build_type += '-DCMAKE_BUILD_TYPE=Release'
def testCompetitionCounting(self): lid = locks.MasterLock('dummy') la = locks.LockAccess(lid, 'counting') return self._testCompetition(la)
def testLaterCounting(self): lid = locks.MasterLock('dummy') la = locks.LockAccess(lid, 'counting') return self._testLater(la)
def test_cancelWhileLocksAvailable(self): def _owns_lock(step, lock): access = [ step_access for step_lock, step_access in step.locks if step_lock == lock ][0] return lock.isOwner(step, access) def _lock_available(step, lock): access = [ step_access for step_lock, step_access in step.locks if step_lock == lock ][0] return lock.isAvailable(step, access) lock1 = locks.MasterLock("masterlock1") real_lock1 = locks.RealMasterLock(lock1) lock2 = locks.MasterLock("masterlock2") real_lock2 = locks.RealMasterLock(lock2) stepa = self.setupStep( self.FakeBuildStep(locks=[(real_lock1, locks.LockAccess(lock1, 'exclusive'))])) stepb = self.setupStep( self.FakeBuildStep(locks=[(real_lock2, locks.LockAccess(lock2, 'exclusive'))])) stepc = self.setupStep( self.FakeBuildStep( locks=[(real_lock1, locks.LockAccess(lock1, 'exclusive') ), (real_lock2, locks.LockAccess(lock2, 'exclusive'))])) stepd = self.setupStep( self.FakeBuildStep( locks=[(real_lock1, locks.LockAccess(lock1, 'exclusive') ), (real_lock2, locks.LockAccess(lock2, 'exclusive'))])) # Start all the steps yield stepa.acquireLocks() yield stepb.acquireLocks() c_d = stepc.acquireLocks() d_d = stepd.acquireLocks() # Check that step a and step b have the locks self.assertTrue(_owns_lock(stepa, real_lock1)) self.assertTrue(_owns_lock(stepb, real_lock2)) # Check that step c does not have a lock self.assertFalse(_owns_lock(stepc, real_lock1)) self.assertFalse(_owns_lock(stepc, real_lock2)) # Check that step d does not have a lock self.assertFalse(_owns_lock(stepd, real_lock1)) self.assertFalse(_owns_lock(stepd, real_lock2)) # Release lock 1 stepa.releaseLocks() yield deferLater(reactor, 0, lambda: None) # lock1 should be available for step c self.assertTrue(_lock_available(stepc, real_lock1)) self.assertFalse(_lock_available(stepc, real_lock2)) self.assertFalse(_lock_available(stepd, real_lock1)) self.assertFalse(_lock_available(stepd, real_lock2)) # Cancel step c stepc.interrupt("cancelling") yield c_d # Check that step c does not have a lock self.assertFalse(_owns_lock(stepc, real_lock1)) self.assertFalse(_owns_lock(stepc, real_lock2)) # No lock should be available for step c self.assertFalse(_lock_available(stepc, real_lock1)) self.assertFalse(_lock_available(stepc, real_lock2)) # lock 1 should be available for step d self.assertTrue(_lock_available(stepd, real_lock1)) self.assertFalse(_lock_available(stepd, real_lock2)) # Release lock 2 stepb.releaseLocks() # Both locks should be available for step d self.assertTrue(_lock_available(stepd, real_lock1)) self.assertTrue(_lock_available(stepd, real_lock2)) # So it should run yield d_d # Check that step d owns the locks self.assertTrue(_owns_lock(stepd, real_lock1)) self.assertTrue(_owns_lock(stepd, real_lock2))
from buildbot import locks build_lock_dict = {} db_lock_143=locks.MasterLock('database_143') build_lock_dict["db_lock_143"] = db_lock_143 db_lock_11=locks.MasterLock('database_11') build_lock_dict['db_lock_11'] = db_lock_11 db_lock_141=locks.MasterLock('database_141') build_lock_dict['db_lock_141'] = db_lock_141 db_lock_30=locks.MasterLock('database_30') build_lock_dict['db_lock_30'] = db_lock_30 db_lock_98=locks.MasterLock('database_98') build_lock_dict['db_lock_98'] = db_lock_98 db_lock_97=locks.MasterLock('database_97') build_lock_dict['db_lock_97'] = db_lock_97 db_lock_82=locks.MasterLock('database_82') build_lock_dict['db_lock_82'] = db_lock_82 db_lock_73=locks.MasterLock('database_73') build_lock_dict['db_lock_73'] = db_lock_73 db_lock_202=locks.MasterLock('database_202') build_lock_dict['db_lock_202'] = db_lock_202 db_lock_208=locks.MasterLock('database_208') build_lock_dict['db_lock_208'] = db_lock_208
def testLaterExclusive(self): lid = locks.MasterLock('dummy') la = locks.LockAccess(lid, 'exclusive') return self._testLater(la)
# vim: set ts=8 sw=4 sts=4 tw=99 et ft=python : from am.factory import PathBuilder from am.factory import ShellCommand from buildbot import locks from buildbot.config import BuilderConfig from buildbot.process import factory from buildbot.steps.source.git import Git from am.sourcemod.slaves import Slaves pdb_lock = locks.MasterLock("symbolstore") def CreateBuilder(slave, version, branch): name = slave if 'display' in Slaves[slave]: name = Slaves[slave]['display'] return BuilderConfig(name='{0}-{1}'.format(name, version), builddir='{0}-{1}'.format(name, version), slavename=slave, factory=SMFactory(slave, branch)) def SMFactory(slave, branch): os = Slaves[slave]['os'] paths = PathBuilder(os) bootstrap_argv = ['perl', paths.join('tools', 'buildbot', 'bootstrap.pl')] build_argv = ['perl', paths.join('tools', 'buildbot', 'startbuild.pl')] upload_argv = [ 'perl',
def testCompetitionExclusive(self): lid = locks.MasterLock('dummy') la = locks.LockAccess(lid, 'exclusive') return self._testCompetition(la)
def __init__(self): self.slaves = [] self.builders = [] self.auto_builder_names = [] self.local_builder_lock = locks.MasterLock("local", maxCount=2) self.deps_lock = locks.SlaveLock("deps", maxCount=1) # Add linux slaves and builders. for linux_distro, versions in CONFIG['linux'].iteritems(): factory = LINUX_FACTORIES[linux_distro] for version in versions: self._AddBuilderAndSlave(linux_distro, version, False, factory) self._AddBuilderAndSlave(linux_distro, version, True, factory) # Add Ubuntu PPAs. for version in CONFIG['linux']['ubuntu']: self._AddBuilder(name='Ubuntu dev PPA %s' % version.title(), slave='ubuntu-%s-32' % version, build_factory=builders.MakePPABuilder( version, DEV_PPA)) self._AddBuilder(name='Ubuntu official PPA %s' % version.title(), slave='ubuntu-%s-32' % version, build_factory=builders.MakePPABuilder( version, OFFICIAL_PPA), auto=False) # Add special slaves. for name in CONFIG['special_slaves']: self._AddSlave(name) # Windows. self._AddBuilder(name='Windows Dependencies', slave='mingw', build_factory=builders.MakeWindowsDepsBuilder(), auto=False, deps_lock='exclusive') self._AddBuilder(name='Windows Release', slave='mingw', build_factory=builders.MakeWindowsBuilder( is_debug=False, is_portable=False), deps_lock='counting') self._AddBuilder(name='Windows Debug', slave='mingw', build_factory=builders.MakeWindowsBuilder( is_debug=True, is_portable=False), deps_lock='counting') self._AddBuilder(name='Windows Portable', slave='mingw', build_factory=builders.MakeWindowsBuilder( is_debug=False, is_portable=True), deps_lock='counting') # Mac. self._AddSlave('mac') self._AddBuilder(name='Mac Dependencies', slave='mac', build_factory=builders.MakeMacDepsBuilder(), auto=False, local_lock=False, deps_lock='exclusive') self._AddBuilder(name='Mac Release', slave='mac', build_factory=builders.MakeMacBuilder(), local_lock=False, deps_lock='counting') self._AddBuilder(name='Mac Cross', slave='mac-cross', build_factory=builders.MakeMacCrossBuilder()) # Spotify. self._AddBuilder(name='Spofify blob 32-bit', slave='spotify-blob-32', build_factory=builders.MakeSpotifyBlobBuilder()) self._AddBuilder(name='Spofify blob 64-bit', slave='spotify-blob-64', build_factory=builders.MakeSpotifyBlobBuilder()) # Transifex. self._AddBuilder(name='Transifex POT push', slave='transifex', build_factory=builders.MakeTransifexPotPushBuilder()) self._AddBuilder(name='Transifex PO pull', slave='transifex', build_factory=builders.MakeTransifexPoPullBuilder(), auto=False) self._AddBuilder( name='Transifex website POT push', slave='transifex', build_factory=builders.MakeWebsiteTransifexPotPushBuilder(), auto=False) self._AddBuilder( name='Transifex website PO pull', slave='transifex', build_factory=builders.MakeWebsiteTransifexPoPullBuilder(), auto=False) self._AddBuilder( name='Transifex Android PO pull', slave='transifex', build_factory=builders.MakeAndroidTransifexPoPullBuilder(), auto=False) # Android. self._AddBuilder(name='Android Remote', slave='android', build_factory=builders.MakeAndroidRemoteBuilder(), auto=False) # Source. self._AddBuilder(name='Source', slave='ubuntu-xenial-64', build_factory=builders.MakeSourceBuilder())