def testGet(self):
# the master.cfg file contains "lock ids", which are instances of
# MasterLock and SlaveLock but which are not actually Locks per se.
# When the build starts, these markers are turned into RealMasterLock
# and RealSlaveLock instances. This insures that any builds running
# on slaves that were unaffected by the config change are still
# referring to the same Lock instance as new builds by builders that
# *were* affected by the change. There have been bugs in the past in
# which this didn't happen, and the Locks were bypassed because half
# the builders were using one incarnation of the lock while the other
# half were using a separate (but equal) incarnation.
#
# Changing the lock id in any way should cause it to be replaced in
# the BotMaster. This will result in a couple of funky artifacts:
# builds in progress might pay attention to a different lock, so we
# might bypass the locking for the duration of a couple builds.
# There's also the problem of old Locks lingering around in
# BotMaster.locks, but they're small and shouldn't really cause a
# problem.
b = master.BotMaster()
l1 = locks.MasterLock("one")
l1a = locks.MasterLock("one")
l2 = locks.MasterLock("one", maxCount=4)
rl1 = b.getLockByID(l1)
rl2 = b.getLockByID(l1a)
self.failUnlessIdentical(rl1, rl2)
rl3 = b.getLockByID(l2)
self.failIfIdentical(rl1, rl3)
s1 = locks.SlaveLock("one")
s1a = locks.SlaveLock("one")
s2 = locks.SlaveLock("one", maxCount=4)
s3 = locks.SlaveLock("one", maxCount=4,
maxCountForSlave={"a":1, "b":2})
s3a = locks.SlaveLock("one", maxCount=4,
maxCountForSlave={"a":1, "b":2})
s4 = locks.SlaveLock("one", maxCount=4,
maxCountForSlave={"a":4, "b":4})
rl1 = b.getLockByID(s1)
rl2 = b.getLockByID(s1a)
self.failUnlessIdentical(rl1, rl2)
rl3 = b.getLockByID(s2)
self.failIfIdentical(rl1, rl3)
rl4 = b.getLockByID(s3)
self.failIfIdentical(rl1, rl4)
self.failIfIdentical(rl3, rl4)
rl5 = b.getLockByID(s3a)
self.failUnlessIdentical(rl4, rl5)
rl6 = b.getLockByID(s4)
self.failIfIdentical(rl5, rl6)
def test_renderableLocks(self):
master_lock = locks.MasterLock("masterlock")
worker_lock = locks.WorkerLock("workerlock")
lock_accesses = []
@renderer
def rendered_locks(props):
master_access = locks.LockAccess(master_lock, 'counting')
worker_access = locks.LockAccess(worker_lock, 'exclusive')
lock_accesses.append(master_access)
lock_accesses.append(worker_access)
return [master_access, worker_access]
self.setupStep(self.LockBuildStep(testcase=self, lock_accesses=lock_accesses,
locks=rendered_locks))
self.expectOutcome(result=SUCCESS)
yield self.runStep()
self.assertEqual(len(lock_accesses), 2)
botmaster = self.step.build.builder.botmaster
real_master_lock = yield botmaster.getLockFromLockAccess(lock_accesses[0],
self.build.config_version)
real_worker_lock = yield botmaster.getLockFromLockAccess(lock_accesses[1],
self.build.config_version)
self.assertTrue(real_master_lock.isAvailable(self, lock_accesses[0]))
self.assertIn('workername', real_worker_lock.locks)
self.assertTrue(real_worker_lock.locks['workername'].isAvailable(self, lock_accesses[1]))
def test_regularLocks(self):
lock1 = locks.MasterLock("masterlock")
lock2 = locks.WorkerLock("workerlock")
self.setupStep(self.FakeBuildStep(
locks=[locks.LockAccess(lock1, 'counting'), locks.LockAccess(lock2, 'exclusive')]))
self.expectOutcome(result=SUCCESS)
yield self.runStep()
def test_constructor_full(self):
lock1, lock2 = locks.MasterLock('lock1'), locks.MasterLock('lock2')
access1, access2 = lock1.access('counting'), lock2.access('counting')
bs = yield self.createWorker('bot', 'pass',
max_builds=2,
notify_on_missing=['*****@*****.**'],
missing_timeout=120,
properties={'a': 'b'},
locks=[access1, access2])
yield bs.startService()
self.assertEqual(bs.max_builds, 2)
self.assertEqual(bs.notify_on_missing, ['*****@*****.**'])
self.assertEqual(bs.missing_timeout, 120)
self.assertEqual(bs.properties.getProperty('a'), 'b')
self.assertEqual(bs.access, [access1, access2])
def testMaster(self):
mid1 = locks.MasterLock("name1")
mid2 = locks.MasterLock("name1")
mid3 = locks.MasterLock("name3")
mid4 = locks.MasterLock("name1", 3)
self.failUnlessEqual(mid1, mid2)
self.failIfEqual(mid1, mid3)
# they should all be hashable
d = {mid1: 1, mid2: 2, mid3: 3, mid4: 4}
l1 = self.make(mid1)
self.failUnlessEqual(l1.name, "name1")
self.failUnlessEqual(l1.maxCount, 1)
self.failUnlessIdentical(l1.getLock(slave("slave1")), l1)
l4 = self.make(mid4)
self.failUnlessEqual(l4.name, "name1")
self.failUnlessEqual(l4.maxCount, 3)
self.failUnlessIdentical(l4.getLock(slave("slave1")), l4)
def test_setServiceParent_masterLocks(self):
"""
http://trac.buildbot.net/ticket/2278
"""
master = self.master = fakemaster.make_master()
botmaster = FakeBotMaster(master)
botmaster.startService()
lock = locks.MasterLock('masterlock')
bs = self.ConcreteBuildSlave('bot', 'pass', locks=[lock])
bs.setServiceParent(botmaster)
def test_setServiceParent_masterLocks(self):
"""
http://trac.buildbot.net/ticket/2278
"""
master = self.master
bm = botmaster.FakeBotMaster(master)
bm.startService()
lock = locks.MasterLock('masterlock')
bs = ConcreteBuildSlave('bot', 'pass', locks=[lock.access("counting")])
bs.setServiceParent(bm)
def test_setServiceParent_masterLocks(self):
"""
http://trac.buildbot.net/ticket/2278
"""
master = self.master
bsmanager = master.workers
yield master.startService()
lock = locks.MasterLock('masterlock')
bs = ConcreteWorker('bot', 'pass', locks=[lock.access("counting")])
bs.setServiceParent(bsmanager)
def testNowMixed2(self):
""" Test counting is not possible when an exclsuive has the lock """
lid = locks.MasterLock('dummy')
lac = locks.LockAccess(lid, 'counting')
lae = locks.LockAccess(lid, 'exclusive')
l = locks.BaseLock("name", maxCount=2)
self.failUnless(l.isAvailable(lae))
l.claim("count-owner", lae)
self.failIf(l.isAvailable(lac))
l.release("count-owner", lae)
self.failUnless(l.isAvailable(lae))
def testLaterCounting(self):
lid = locks.MasterLock('dummy')
la = locks.LockAccess(lid, 'counting')
lock = locks.BaseLock("name", 2)
lock.claim("owner1", la)
lock.claim("owner2", la)
d = claimHarder(lock, "owner3", la)
d.addCallback(lambda lock: lock.release("owner3", la))
lock.release("owner2", la)
lock.release("owner1", la)
return d
def testNowCounting(self):
lid = locks.MasterLock('dummy')
la = locks.LockAccess(lid, 'counting')
lock = locks.BaseLock("name", 2)
self.failUnless(lock.isAvailable(la))
lock.claim("owner1", la)
self.failUnless(lock.isAvailable(la))
lock.claim("owner2", la)
self.failIf(lock.isAvailable(la))
lock.release("owner1", la)
self.failUnless(lock.isAvailable(la))
lock.release("owner2", la)
self.failUnless(lock.isAvailable(la))
def test_renderableLocks(self):
lock1 = locks.MasterLock("masterlock")
lock2 = locks.WorkerLock("workerlock")
renderedLocks = [False]
@renderer
def rendered_locks(props):
renderedLocks[0] = True
access1 = locks.LockAccess(lock1, 'counting')
access2 = locks.LockAccess(lock2, 'exclusive')
return [access1, access2]
self.setupStep(self.FakeBuildStep(locks=rendered_locks))
self.expectOutcome(result=SUCCESS)
yield self.runStep()
self.assertTrue(renderedLocks[0])
def test_regularLocks(self):
master_lock = locks.MasterLock("masterlock")
worker_lock = locks.WorkerLock("workerlock")
lock_accesses = [locks.LockAccess(master_lock, 'counting'),
locks.LockAccess(worker_lock, 'exclusive')]
self.setupStep(self.LockBuildStep(testcase=self, lock_accesses=lock_accesses,
locks=lock_accesses))
self.expectOutcome(result=SUCCESS)
yield self.runStep()
botmaster = self.step.build.builder.botmaster
real_master_lock = yield botmaster.getLockFromLockAccess(lock_accesses[0],
self.build.config_version)
real_worker_lock = yield botmaster.getLockFromLockAccess(lock_accesses[1],
self.build.config_version)
self.assertTrue(real_master_lock.isAvailable(self, lock_accesses[0]))
self.assertIn('workername', real_worker_lock.locks)
self.assertTrue(real_worker_lock.locks['workername'].isAvailable(self, lock_accesses[1]))
def testRandomCounting(self):
lid = locks.MasterLock('dummy')
la = locks.LockAccess(lid, 'counting')
COUNT = 5
lock = locks.BaseLock("name", COUNT)
dl = []
for i in range(100):
owner = "owner%d" % i
mode = random.choice(["now", "very soon", "soon"])
d = claimHarder(lock, owner, la)
def _check(lock):
self.failIf(len(lock.owners) > COUNT)
return lock
d.addCallback(_check)
d.addCallback(hold, owner, la, mode)
dl.append(d)
d = defer.DeferredList(dl)
d.addCallback(self._cleanup, lock, COUNT, la)
return d
def __init__(self):
self.workers = []
self.builders = []
self.auto_builder_names = []
self.local_builder_lock = locks.MasterLock("local", maxCount=2)
self.deps_lock = locks.WorkerLock("deps", maxCount=1)
# Add Linux workers and builders.
for linux_distro, versions in CONFIG['linux'].items():
factory = LINUX_FACTORIES[linux_distro]
for version in versions:
self._AddBuilderAndWorker(linux_distro, version, factory)
# Add Ubuntu PPA.
for linux_distro in CONFIG['linux']['ubuntu']:
self._AddBuilder(
name='Ubuntu Stable PPA %s' % linux_distro.title(),
worker='ubuntu-%s' % linux_distro,
build_factory=builders.MakePPABuilder(linux_distro, "stable",
PPA_STABLE))
self._AddBuilder(
name='Ubuntu Unstable PPA %s' % linux_distro.title(),
worker='ubuntu-%s' % linux_distro,
build_factory=builders.MakePPABuilder(linux_distro, "unstable",
PPA_UNSTABLE))
# Add special workers.
for name in CONFIG['special_workers']:
self._AddWorker(name)
# Source.
self._AddBuilder(name='Source',
worker='opensuse-lp152',
build_factory=builders.MakeSourceBuilder())
# AppImage.
self._AddBuilder(
name='AppImage Qt 5',
worker='appimage-qt5',
build_factory=builders.MakeAppImageBuilder(name="Qt5"))
self._AddBuilder(
name='AppImage Qt 6',
worker='appimage-qt6',
build_factory=builders.MakeAppImageBuilder(name="Qt6"))
# MXE.
self._AddBuilder(name='MXE',
worker='mingw',
build_factory=builders.MakeMXEBuilder(),
auto=False,
deps_lock='exclusive')
# Windows.
self._AddBuilder(name='Windows Release x86 Qt 5',
worker='mingw',
build_factory=builders.MakeWindowsBuilder(
is_debug=False, is_64=False, with_qt6=False),
deps_lock='counting')
self._AddBuilder(name='Windows Release x64 Qt 5',
worker='mingw',
build_factory=builders.MakeWindowsBuilder(
is_debug=False, is_64=True, with_qt6=False),
deps_lock='counting')
self._AddBuilder(name='Windows Debug x86 Qt 5',
worker='mingw',
build_factory=builders.MakeWindowsBuilder(
is_debug=True, is_64=False, with_qt6=False),
deps_lock='counting')
self._AddBuilder(name='Windows Debug x64 Qt 5',
worker='mingw',
build_factory=builders.MakeWindowsBuilder(
is_debug=True, is_64=True, with_qt6=False),
deps_lock='counting')
self._AddBuilder(name='Windows Release x86 Qt 6',
worker='mingw',
build_factory=builders.MakeWindowsBuilder(
is_debug=False, is_64=False, with_qt6=True),
deps_lock='counting')
self._AddBuilder(name='Windows Release x64 Qt 6',
worker='mingw',
build_factory=builders.MakeWindowsBuilder(
is_debug=False, is_64=True, with_qt6=True),
deps_lock='counting')
self._AddBuilder(name='Windows Debug x86 Qt 6',
worker='mingw',
build_factory=builders.MakeWindowsBuilder(
is_debug=True, is_64=False, with_qt6=True),
deps_lock='counting')
self._AddBuilder(name='Windows Debug x64 Qt 6',
worker='mingw',
build_factory=builders.MakeWindowsBuilder(
is_debug=True, is_64=True, with_qt6=True),
deps_lock='counting')
from buildbot import locks
slave_cpu_lock = locks.SlaveLock('slave_cpu_lock', maxCount=1)
master_upload_lock = locks.MasterLock('master_upload_lock')
def skipped_or_success(results, step):
from buildbot.status.results import SKIPPED
from buildbot.status.results import SUCCESS
return ((results == SKIPPED) or (results == SUCCESS))
def skipped(results, step):
from buildbot.status.results import SKIPPED
return (results == SKIPPED)
def get_cmake_step(link, type, options=[]):
from buildbot.process.properties import Interpolate
from buildbot.steps.shell import ShellCommand
build_type = ''
if type == 'debug':
build_type += '-DCMAKE_BUILD_TYPE=Debug'
else:
build_type += '-DCMAKE_BUILD_TYPE=Release'
def testCompetitionCounting(self):
lid = locks.MasterLock('dummy')
la = locks.LockAccess(lid, 'counting')
return self._testCompetition(la)
def testLaterCounting(self):
lid = locks.MasterLock('dummy')
la = locks.LockAccess(lid, 'counting')
return self._testLater(la)
def test_cancelWhileLocksAvailable(self):
def _owns_lock(step, lock):
access = [
step_access for step_lock, step_access in step.locks
if step_lock == lock
][0]
return lock.isOwner(step, access)
def _lock_available(step, lock):
access = [
step_access for step_lock, step_access in step.locks
if step_lock == lock
][0]
return lock.isAvailable(step, access)
lock1 = locks.MasterLock("masterlock1")
real_lock1 = locks.RealMasterLock(lock1)
lock2 = locks.MasterLock("masterlock2")
real_lock2 = locks.RealMasterLock(lock2)
stepa = self.setupStep(
self.FakeBuildStep(locks=[(real_lock1,
locks.LockAccess(lock1, 'exclusive'))]))
stepb = self.setupStep(
self.FakeBuildStep(locks=[(real_lock2,
locks.LockAccess(lock2, 'exclusive'))]))
stepc = self.setupStep(
self.FakeBuildStep(
locks=[(real_lock1, locks.LockAccess(lock1, 'exclusive')
), (real_lock2,
locks.LockAccess(lock2, 'exclusive'))]))
stepd = self.setupStep(
self.FakeBuildStep(
locks=[(real_lock1, locks.LockAccess(lock1, 'exclusive')
), (real_lock2,
locks.LockAccess(lock2, 'exclusive'))]))
# Start all the steps
yield stepa.acquireLocks()
yield stepb.acquireLocks()
c_d = stepc.acquireLocks()
d_d = stepd.acquireLocks()
# Check that step a and step b have the locks
self.assertTrue(_owns_lock(stepa, real_lock1))
self.assertTrue(_owns_lock(stepb, real_lock2))
# Check that step c does not have a lock
self.assertFalse(_owns_lock(stepc, real_lock1))
self.assertFalse(_owns_lock(stepc, real_lock2))
# Check that step d does not have a lock
self.assertFalse(_owns_lock(stepd, real_lock1))
self.assertFalse(_owns_lock(stepd, real_lock2))
# Release lock 1
stepa.releaseLocks()
yield deferLater(reactor, 0, lambda: None)
# lock1 should be available for step c
self.assertTrue(_lock_available(stepc, real_lock1))
self.assertFalse(_lock_available(stepc, real_lock2))
self.assertFalse(_lock_available(stepd, real_lock1))
self.assertFalse(_lock_available(stepd, real_lock2))
# Cancel step c
stepc.interrupt("cancelling")
yield c_d
# Check that step c does not have a lock
self.assertFalse(_owns_lock(stepc, real_lock1))
self.assertFalse(_owns_lock(stepc, real_lock2))
# No lock should be available for step c
self.assertFalse(_lock_available(stepc, real_lock1))
self.assertFalse(_lock_available(stepc, real_lock2))
# lock 1 should be available for step d
self.assertTrue(_lock_available(stepd, real_lock1))
self.assertFalse(_lock_available(stepd, real_lock2))
# Release lock 2
stepb.releaseLocks()
# Both locks should be available for step d
self.assertTrue(_lock_available(stepd, real_lock1))
self.assertTrue(_lock_available(stepd, real_lock2))
# So it should run
yield d_d
# Check that step d owns the locks
self.assertTrue(_owns_lock(stepd, real_lock1))
self.assertTrue(_owns_lock(stepd, real_lock2))
from buildbot import locks
build_lock_dict = {}
db_lock_143=locks.MasterLock('database_143')
build_lock_dict["db_lock_143"] = db_lock_143
db_lock_11=locks.MasterLock('database_11')
build_lock_dict['db_lock_11'] = db_lock_11
db_lock_141=locks.MasterLock('database_141')
build_lock_dict['db_lock_141'] = db_lock_141
db_lock_30=locks.MasterLock('database_30')
build_lock_dict['db_lock_30'] = db_lock_30
db_lock_98=locks.MasterLock('database_98')
build_lock_dict['db_lock_98'] = db_lock_98
db_lock_97=locks.MasterLock('database_97')
build_lock_dict['db_lock_97'] = db_lock_97
db_lock_82=locks.MasterLock('database_82')
build_lock_dict['db_lock_82'] = db_lock_82
db_lock_73=locks.MasterLock('database_73')
build_lock_dict['db_lock_73'] = db_lock_73
db_lock_202=locks.MasterLock('database_202')
build_lock_dict['db_lock_202'] = db_lock_202
db_lock_208=locks.MasterLock('database_208')
build_lock_dict['db_lock_208'] = db_lock_208
def testLaterExclusive(self):
lid = locks.MasterLock('dummy')
la = locks.LockAccess(lid, 'exclusive')
return self._testLater(la)
# vim: set ts=8 sw=4 sts=4 tw=99 et ft=python :
from am.factory import PathBuilder
from am.factory import ShellCommand
from buildbot import locks
from buildbot.config import BuilderConfig
from buildbot.process import factory
from buildbot.steps.source.git import Git
from am.sourcemod.slaves import Slaves
pdb_lock = locks.MasterLock("symbolstore")
def CreateBuilder(slave, version, branch):
name = slave
if 'display' in Slaves[slave]:
name = Slaves[slave]['display']
return BuilderConfig(name='{0}-{1}'.format(name, version),
builddir='{0}-{1}'.format(name, version),
slavename=slave,
factory=SMFactory(slave, branch))
def SMFactory(slave, branch):
os = Slaves[slave]['os']
paths = PathBuilder(os)
bootstrap_argv = ['perl', paths.join('tools', 'buildbot', 'bootstrap.pl')]
build_argv = ['perl', paths.join('tools', 'buildbot', 'startbuild.pl')]
upload_argv = [
'perl',
def testCompetitionExclusive(self):
lid = locks.MasterLock('dummy')
la = locks.LockAccess(lid, 'exclusive')
return self._testCompetition(la)
def __init__(self):
self.slaves = []
self.builders = []
self.auto_builder_names = []
self.local_builder_lock = locks.MasterLock("local", maxCount=2)
self.deps_lock = locks.SlaveLock("deps", maxCount=1)
# Add linux slaves and builders.
for linux_distro, versions in CONFIG['linux'].iteritems():
factory = LINUX_FACTORIES[linux_distro]
for version in versions:
self._AddBuilderAndSlave(linux_distro, version, False, factory)
self._AddBuilderAndSlave(linux_distro, version, True, factory)
# Add Ubuntu PPAs.
for version in CONFIG['linux']['ubuntu']:
self._AddBuilder(name='Ubuntu dev PPA %s' % version.title(),
slave='ubuntu-%s-32' % version,
build_factory=builders.MakePPABuilder(
version, DEV_PPA))
self._AddBuilder(name='Ubuntu official PPA %s' % version.title(),
slave='ubuntu-%s-32' % version,
build_factory=builders.MakePPABuilder(
version, OFFICIAL_PPA),
auto=False)
# Add special slaves.
for name in CONFIG['special_slaves']:
self._AddSlave(name)
# Windows.
self._AddBuilder(name='Windows Dependencies',
slave='mingw',
build_factory=builders.MakeWindowsDepsBuilder(),
auto=False,
deps_lock='exclusive')
self._AddBuilder(name='Windows Release',
slave='mingw',
build_factory=builders.MakeWindowsBuilder(
is_debug=False, is_portable=False),
deps_lock='counting')
self._AddBuilder(name='Windows Debug',
slave='mingw',
build_factory=builders.MakeWindowsBuilder(
is_debug=True, is_portable=False),
deps_lock='counting')
self._AddBuilder(name='Windows Portable',
slave='mingw',
build_factory=builders.MakeWindowsBuilder(
is_debug=False, is_portable=True),
deps_lock='counting')
# Mac.
self._AddSlave('mac')
self._AddBuilder(name='Mac Dependencies',
slave='mac',
build_factory=builders.MakeMacDepsBuilder(),
auto=False,
local_lock=False,
deps_lock='exclusive')
self._AddBuilder(name='Mac Release',
slave='mac',
build_factory=builders.MakeMacBuilder(),
local_lock=False,
deps_lock='counting')
self._AddBuilder(name='Mac Cross',
slave='mac-cross',
build_factory=builders.MakeMacCrossBuilder())
# Spotify.
self._AddBuilder(name='Spofify blob 32-bit',
slave='spotify-blob-32',
build_factory=builders.MakeSpotifyBlobBuilder())
self._AddBuilder(name='Spofify blob 64-bit',
slave='spotify-blob-64',
build_factory=builders.MakeSpotifyBlobBuilder())
# Transifex.
self._AddBuilder(name='Transifex POT push',
slave='transifex',
build_factory=builders.MakeTransifexPotPushBuilder())
self._AddBuilder(name='Transifex PO pull',
slave='transifex',
build_factory=builders.MakeTransifexPoPullBuilder(),
auto=False)
self._AddBuilder(
name='Transifex website POT push',
slave='transifex',
build_factory=builders.MakeWebsiteTransifexPotPushBuilder(),
auto=False)
self._AddBuilder(
name='Transifex website PO pull',
slave='transifex',
build_factory=builders.MakeWebsiteTransifexPoPullBuilder(),
auto=False)
self._AddBuilder(
name='Transifex Android PO pull',
slave='transifex',
build_factory=builders.MakeAndroidTransifexPoPullBuilder(),
auto=False)
# Android.
self._AddBuilder(name='Android Remote',
slave='android',
build_factory=builders.MakeAndroidRemoteBuilder(),
auto=False)
# Source.
self._AddBuilder(name='Source',
slave='ubuntu-xenial-64',
build_factory=builders.MakeSourceBuilder())