def testStopBuildWaitingForLocks(self):
b = self.build
l = WorkerLock('lock')
lock_access = l.access('counting')
l.access = lambda mode: lock_access
real_lock = b.builder.botmaster.getLockByID(l) \
.getLock(self.workerforbuilder.worker)
b.setLocks([lock_access])
step = Mock()
step.return_value = step
step.startStep.return_value = SUCCESS
step.alwaysRun = False
b.setStepFactories([FakeStepFactory(step)])
real_lock.claim(Mock(), l.access('counting'))
def acquireLocks(res=None):
retval = Build.acquireLocks(b, res)
b.stopBuild('stop it')
return retval
b.acquireLocks = acquireLocks
b.startBuild(FakeBuildStatus(), self.workerforbuilder)
self.assertNotIn(('startStep', (self.workerforbuilder.worker.conn,), {}),
step.method_calls)
self.assertTrue(b.currentStep is None)
self.assertEqual(b.results, CANCELLED)
self.assertNotIn(('interrupt', ('stop it',), {}), step.method_calls)
def testBuildcanStartWithWorkerForBuilder(self):
b = self.build
workerforbuilder1 = Mock()
workerforbuilder2 = Mock()
l = WorkerLock('lock')
counting_access = l.access('counting')
real_lock = b.builder.botmaster.getLockByID(l)
# no locks, so both these pass (call twice to verify there's no state/memory)
lock_list = [(real_lock, counting_access)]
self.assertTrue(Build.canStartWithWorkerForBuilder(lock_list, workerforbuilder1))
self.assertTrue(Build.canStartWithWorkerForBuilder(lock_list, workerforbuilder1))
self.assertTrue(Build.canStartWithWorkerForBuilder(lock_list, workerforbuilder2))
self.assertTrue(Build.canStartWithWorkerForBuilder(lock_list, workerforbuilder2))
worker_lock_1 = real_lock.getLock(workerforbuilder1.worker)
worker_lock_2 = real_lock.getLock(workerforbuilder2.worker)
# then have workerforbuilder2 claim its lock:
worker_lock_2.claim(workerforbuilder2, counting_access)
self.assertTrue(Build.canStartWithWorkerForBuilder(lock_list, workerforbuilder1))
self.assertTrue(Build.canStartWithWorkerForBuilder(lock_list, workerforbuilder1))
self.assertFalse(Build.canStartWithWorkerForBuilder(lock_list, workerforbuilder2))
self.assertFalse(Build.canStartWithWorkerForBuilder(lock_list, workerforbuilder2))
worker_lock_2.release(workerforbuilder2, counting_access)
# then have workerforbuilder1 claim its lock:
worker_lock_1.claim(workerforbuilder1, counting_access)
self.assertFalse(Build.canStartWithWorkerForBuilder(lock_list, workerforbuilder1))
self.assertFalse(Build.canStartWithWorkerForBuilder(lock_list, workerforbuilder1))
self.assertTrue(Build.canStartWithWorkerForBuilder(lock_list, workerforbuilder2))
self.assertTrue(Build.canStartWithWorkerForBuilder(lock_list, workerforbuilder2))
worker_lock_1.release(workerforbuilder1, counting_access)
def testStopBuildWaitingForLocks_lostRemote(self):
b = self.build
lock = WorkerLock('lock')
lock_access = lock.access('counting')
lock.access = lambda mode: lock_access
real_lock = b.builder.botmaster.getLockByID(lock) \
.getLock(self.workerforbuilder.worker)
b.setLocks([lock_access])
step = FakeBuildStep()
step.alwaysRun = False
b.setStepFactories([FakeStepFactory(step)])
real_lock.claim(Mock(), lock.access('counting'))
def acquireLocks(res=None):
retval = Build.acquireLocks(b, res)
b.lostRemote()
return retval
b.acquireLocks = acquireLocks
b.startBuild(FakeBuildStatus(), self.workerforbuilder)
self.assertTrue(b.currentStep is None)
self.assertEqual(b.results, RETRY)
self.build.build_status.setText.assert_called_with(
["retry", "lost", "connection"])
self.build.build_status.setResults.assert_called_with(RETRY)
def testStopBuildWaitingForLocks(self):
b = self.build
lock = WorkerLock('lock')
lock_access = lock.access('counting')
lock.access = lambda mode: lock_access
real_lock = b.builder.botmaster.getLockByID(lock) \
.getLock(self.workerforbuilder.worker)
b.setLocks([lock_access])
step = FakeBuildStep()
step.alwaysRun = False
b.setStepFactories([FakeStepFactory(step)])
real_lock.claim(Mock(), lock.access('counting'))
def acquireLocks(res=None):
retval = Build.acquireLocks(b, res)
b.stopBuild('stop it')
return retval
b.acquireLocks = acquireLocks
b.startBuild(FakeBuildStatus(), self.workerforbuilder)
self.assertTrue(b.currentStep is None)
self.assertEqual(b.results, CANCELLED)
def testStopBuildWaitingForStepLocks(self):
b = self.build
lock = WorkerLock('lock')
lock_access = lock.access('counting')
lock.access = lambda mode: lock_access
real_lock = b.builder.botmaster.getLockByID(lock) \
.getLock(self.workerforbuilder.worker)
step = LoggingBuildStep(locks=[lock_access])
b.setStepFactories([FakeStepFactory(step)])
real_lock.claim(Mock(), lock.access('counting'))
gotLocks = [False]
def acquireLocks(res=None):
gotLocks[0] = True
retval = LoggingBuildStep.acquireLocks(step, res)
self.assertTrue(b.currentStep is step)
b.stopBuild('stop it')
return retval
step.acquireLocks = acquireLocks
b.startBuild(FakeBuildStatus(), self.workerforbuilder)
self.assertEqual(gotLocks, [True])
self.assertEqual(b.results, CANCELLED)
def testBuildLocksAcquired(self):
b = self.build
lock = WorkerLock('lock')
claimCount = [0]
lock_access = lock.access('counting')
lock.access = lambda mode: lock_access
real_lock = b.builder.botmaster.getLockByID(lock) \
.getLock(self.workerforbuilder.worker)
def claim(owner, access):
claimCount[0] += 1
return real_lock.old_claim(owner, access)
real_lock.old_claim = real_lock.claim
real_lock.claim = claim
b.setLocks([lock_access])
step = Mock()
step.return_value = step
step.startStep.return_value = SUCCESS
b.setStepFactories([FakeStepFactory(step)])
b.startBuild(FakeBuildStatus(), self.workerforbuilder)
self.assertEqual(b.results, SUCCESS)
self.assertIn(('startStep', (self.workerforbuilder.worker.conn, ), {}),
step.method_calls)
self.assertEqual(claimCount[0], 1)
def testBuildLocksAcquired(self):
b = self.build
l = WorkerLock('lock')
claimCount = [0]
lock_access = l.access('counting')
l.access = lambda mode: lock_access
real_lock = b.builder.botmaster.getLockByID(l) \
.getLock(self.workerforbuilder.worker)
def claim(owner, access):
claimCount[0] += 1
return real_lock.old_claim(owner, access)
real_lock.old_claim = real_lock.claim
real_lock.claim = claim
b.setLocks([lock_access])
step = Mock()
step.return_value = step
step.startStep.return_value = SUCCESS
b.setStepFactories([FakeStepFactory(step)])
b.startBuild(FakeBuildStatus(), self.workerforbuilder)
self.assertEqual(b.results, SUCCESS)
self.assertIn(('startStep', (self.workerforbuilder.worker.conn,), {}),
step.method_calls)
self.assertEquals(claimCount[0], 1)
def testBuildWaitingForLocks(self):
b = self.build
l = WorkerLock('lock')
claimCount = [0]
lock_access = l.access('counting')
l.access = lambda mode: lock_access
real_lock = b.builder.botmaster.getLockByID(l) \
.getLock(self.workerforbuilder.worker)
def claim(owner, access):
claimCount[0] += 1
return real_lock.old_claim(owner, access)
real_lock.old_claim = real_lock.claim
real_lock.claim = claim
b.setLocks([lock_access])
step = Mock()
step.return_value = step
step.startStep.return_value = SUCCESS
b.setStepFactories([FakeStepFactory(step)])
real_lock.claim(Mock(), l.access('counting'))
b.startBuild(FakeBuildStatus(), self.workerforbuilder)
self.assertNotIn(('startStep', (self.workerforbuilder.worker.conn,), {}),
step.method_calls)
self.assertEquals(claimCount[0], 1)
self.assertTrue(b.currentStep is None)
self.assertTrue(b._acquiringLock is not None)
def testStopBuildWaitingForLocks_lostRemote(self):
b = self.build
lock = WorkerLock('lock')
lock_access = lock.access('counting')
lock.access = lambda mode: lock_access
real_workerlock = yield b.builder.botmaster.getLockByID(lock, 0)
real_lock = real_workerlock.getLockForWorker(self.workerforbuilder.worker.workername)
yield b.setLocks([lock_access])
step = FakeBuildStep()
step.alwaysRun = False
b.setStepFactories([FakeStepFactory(step)])
real_lock.claim(Mock(), lock.access('counting'))
def acquireLocks(res=None):
retval = Build.acquireLocks(b, res)
b.lostRemote()
return retval
b.acquireLocks = acquireLocks
b.startBuild(self.workerforbuilder)
self.assertTrue(b.currentStep is None)
self.assertEqual(b.results, RETRY)
def testBuildWaitingForLocks(self):
b = self.build
lock = WorkerLock('lock')
claimCount = [0]
lock_access = lock.access('counting')
lock.access = lambda mode: lock_access
real_workerlock = yield b.builder.botmaster.getLockByID(lock, 0)
real_lock = real_workerlock.getLockForWorker(self.workerforbuilder.worker.workername)
def claim(owner, access):
claimCount[0] += 1
return real_lock.old_claim(owner, access)
real_lock.old_claim = real_lock.claim
real_lock.claim = claim
yield b.setLocks([lock_access])
step = FakeBuildStep()
b.setStepFactories([FakeStepFactory(step)])
real_lock.claim(Mock(), lock.access('counting'))
b.startBuild(self.workerforbuilder)
self.assertEqual(claimCount[0], 1)
self.assertTrue(b.currentStep is None)
self.assertTrue(b._acquiringLock is not None)
def testStopBuildWaitingForLocks_lostRemote(self):
b = self.build
l = WorkerLock('lock')
lock_access = l.access('counting')
l.access = lambda mode: lock_access
real_lock = b.builder.botmaster.getLockByID(l) \
.getLock(self.workerforbuilder.worker)
b.setLocks([lock_access])
step = Mock()
step.return_value = step
step.startStep.return_value = SUCCESS
step.alwaysRun = False
b.setStepFactories([FakeStepFactory(step)])
real_lock.claim(Mock(), l.access('counting'))
def acquireLocks(res=None):
retval = Build.acquireLocks(b, res)
b.lostRemote()
return retval
b.acquireLocks = acquireLocks
b.startBuild(FakeBuildStatus(), None, self.workerforbuilder)
self.assert_(('startStep', (self.workerforbuilder.worker.conn,), {})
not in step.method_calls)
self.assert_(b.currentStep is None)
self.assertEqual(b.results, RETRY)
self.assert_(('interrupt', ('stop it',), {}) not in step.method_calls)
self.build.build_status.setText.assert_called_with(["retry", "lost", "connection"])
self.build.build_status.setResults.assert_called_with(RETRY)
def testBuildWaitingForLocks(self):
b = self.build
l = WorkerLock('lock')
claimCount = [0]
lock_access = l.access('counting')
l.access = lambda mode: lock_access
real_lock = b.builder.botmaster.getLockByID(l) \
.getLock(self.workerforbuilder.worker)
def claim(owner, access):
claimCount[0] += 1
return real_lock.old_claim(owner, access)
real_lock.old_claim = real_lock.claim
real_lock.claim = claim
b.setLocks([lock_access])
step = Mock()
step.return_value = step
step.startStep.return_value = SUCCESS
b.setStepFactories([FakeStepFactory(step)])
real_lock.claim(Mock(), l.access('counting'))
b.startBuild(FakeBuildStatus(), None, self.workerforbuilder)
self.assert_(('startStep', (self.workerforbuilder.worker.conn,), {})
not in step.method_calls)
self.assertEquals(claimCount[0], 1)
self.assert_(b.currentStep is None)
self.assert_(b._acquiringLock is not None)
def testBuildWaitingForLocks(self):
b = self.build
lock = WorkerLock('lock')
claimCount = [0]
lock_access = lock.access('counting')
lock.access = lambda mode: lock_access
real_lock = b.builder.botmaster.getLockByID(lock) \
.getLock(self.workerforbuilder.worker)
def claim(owner, access):
claimCount[0] += 1
return real_lock.old_claim(owner, access)
real_lock.old_claim = real_lock.claim
real_lock.claim = claim
b.setLocks([lock_access])
step = FakeBuildStep()
b.setStepFactories([FakeStepFactory(step)])
real_lock.claim(Mock(), lock.access('counting'))
b.startBuild(FakeBuildStatus(), self.workerforbuilder)
self.assertEqual(claimCount[0], 1)
self.assertTrue(b.currentStep is None)
self.assertTrue(b._acquiringLock is not None)
def test_worker_lock_update_from_lockid_count_for_worker(
self, acquire_before, worker_count_before, worker_count_after):
max_count_before = {}
if worker_count_before:
max_count_before = {'worker1': 5}
max_count_after = {}
if worker_count_after:
max_count_after = {'worker1': 7}
lockid = WorkerLock('lock1', maxCount=3,
maxCountForWorker=max_count_before)
lock = RealWorkerLock(lockid)
if acquire_before:
worker_lock = lock.getLockForWorker('worker1')
self.assertEqual(worker_lock.maxCount,
5 if worker_count_before else 3)
lockid = WorkerLock('lock1', maxCount=4,
maxCountForWorker=max_count_after)
lock.updateFromLockId(lockid)
if not acquire_before:
worker_lock = lock.getLockForWorker('worker1')
self.assertEqual(worker_lock.maxCount,
7 if worker_count_after else 4)
def testStopBuildWaitingForLocks(self):
b = self.build
lock = WorkerLock('lock')
lock_access = lock.access('counting')
lock.access = lambda mode: lock_access
real_lock = b.builder.botmaster.getLockByID(lock) \
.getLock(self.workerforbuilder.worker)
b.setLocks([lock_access])
step = FakeBuildStep()
step.alwaysRun = False
b.setStepFactories([FakeStepFactory(step)])
real_lock.claim(Mock(), lock.access('counting'))
def acquireLocks(res=None):
retval = Build.acquireLocks(b, res)
b.stopBuild('stop it')
return retval
b.acquireLocks = acquireLocks
b.startBuild(FakeBuildStatus(), self.workerforbuilder)
self.assertTrue(b.currentStep is None)
self.assertEqual(b.results, CANCELLED)
def testStopBuildWaitingForLocks_lostRemote(self):
b = self.build
l = WorkerLock('lock')
lock_access = l.access('counting')
l.access = lambda mode: lock_access
real_lock = b.builder.botmaster.getLockByID(l) \
.getLock(self.workerforbuilder.worker)
b.setLocks([lock_access])
step = Mock()
step.return_value = step
step.startStep.return_value = SUCCESS
step.alwaysRun = False
b.setStepFactories([FakeStepFactory(step)])
real_lock.claim(Mock(), l.access('counting'))
def acquireLocks(res=None):
retval = Build.acquireLocks(b, res)
b.lostRemote()
return retval
b.acquireLocks = acquireLocks
b.startBuild(FakeBuildStatus(), None, self.workerforbuilder)
self.assert_(('startStep', (self.workerforbuilder.worker.conn, ),
{}) not in step.method_calls)
self.assert_(b.currentStep is None)
self.assertEqual(b.results, RETRY)
self.assert_(('interrupt', ('stop it', ), {}) not in step.method_calls)
self.build.build_status.setText.assert_called_with(
["retry", "lost", "connection"])
self.build.build_status.setResults.assert_called_with(RETRY)
def testStopBuildWaitingForLocks(self):
b = self.build
l = WorkerLock("lock")
lock_access = l.access("counting")
l.access = lambda mode: lock_access
real_lock = b.builder.botmaster.getLockByID(l).getLock(self.workerbuilder.worker)
b.setLocks([lock_access])
step = Mock()
step.return_value = step
step.startStep.return_value = SUCCESS
step.alwaysRun = False
b.setStepFactories([FakeStepFactory(step)])
real_lock.claim(Mock(), l.access("counting"))
def acquireLocks(res=None):
retval = Build.acquireLocks(b, res)
b.stopBuild("stop it")
return retval
b.acquireLocks = acquireLocks
b.startBuild(FakeBuildStatus(), None, self.workerbuilder)
self.assert_(("startStep", (self.workerbuilder.worker.conn,), {}) not in step.method_calls)
self.assert_(b.currentStep is None)
self.assertEqual(b.results, CANCELLED)
self.assert_(("interrupt", ("stop it",), {}) not in step.method_calls)
def testStopBuildWaitingForLocks_lostRemote(self):
b = self.build
lock = WorkerLock('lock')
lock_access = lock.access('counting')
lock.access = lambda mode: lock_access
real_lock = b.builder.botmaster.getLockByID(lock) \
.getLock(self.workerforbuilder.worker)
b.setLocks([lock_access])
step = FakeBuildStep()
step.alwaysRun = False
b.setStepFactories([FakeStepFactory(step)])
real_lock.claim(Mock(), lock.access('counting'))
def acquireLocks(res=None):
retval = Build.acquireLocks(b, res)
b.lostRemote()
return retval
b.acquireLocks = acquireLocks
b.startBuild(FakeBuildStatus(), self.workerforbuilder)
self.assertTrue(b.currentStep is None)
self.assertEqual(b.results, RETRY)
self.build.build_status.setText.assert_called_with(
["retry", "lost", "connection"])
self.build.build_status.setResults.assert_called_with(RETRY)
def testStopBuildWaitingForStepLocks(self):
b = self.build
lock = WorkerLock('lock')
lock_access = lock.access('counting')
lock.access = lambda mode: lock_access
real_workerlock = yield b.builder.botmaster.getLockByID(lock, 0)
real_lock = real_workerlock.getLockForWorker(self.workerforbuilder.worker.workername)
step = BuildStep(locks=[lock_access])
b.setStepFactories([FakeStepFactory(step)])
real_lock.claim(Mock(), lock.access('counting'))
gotLocks = [False]
def acquireLocks(res=None):
gotLocks[0] = True
retval = BuildStep.acquireLocks(step, res)
self.assertTrue(b.currentStep is step)
b.stopBuild('stop it')
return retval
step.acquireLocks = acquireLocks
b.startBuild(self.workerforbuilder)
self.assertEqual(gotLocks, [True])
self.assertEqual(b.results, CANCELLED)
def testBuildcanStartWithWorkerForBuilder(self):
b = self.build
workerforbuilder1 = Mock()
workerforbuilder2 = Mock()
l = WorkerLock('lock')
counting_access = l.access('counting')
real_lock = b.builder.botmaster.getLockByID(l)
# no locks, so both these pass (call twice to verify there's no state/memory)
lock_list = [(real_lock, counting_access)]
self.assertTrue(Build.canStartWithWorkerForBuilder(lock_list, workerforbuilder1))
self.assertTrue(Build.canStartWithWorkerForBuilder(lock_list, workerforbuilder1))
self.assertTrue(Build.canStartWithWorkerForBuilder(lock_list, workerforbuilder2))
self.assertTrue(Build.canStartWithWorkerForBuilder(lock_list, workerforbuilder2))
worker_lock_1 = real_lock.getLock(workerforbuilder1.worker)
worker_lock_2 = real_lock.getLock(workerforbuilder2.worker)
# then have workerforbuilder2 claim its lock:
worker_lock_2.claim(workerforbuilder2, counting_access)
self.assertTrue(Build.canStartWithWorkerForBuilder(lock_list, workerforbuilder1))
self.assertTrue(Build.canStartWithWorkerForBuilder(lock_list, workerforbuilder1))
self.assertFalse(Build.canStartWithWorkerForBuilder(lock_list, workerforbuilder2))
self.assertFalse(Build.canStartWithWorkerForBuilder(lock_list, workerforbuilder2))
worker_lock_2.release(workerforbuilder2, counting_access)
# then have workerforbuilder1 claim its lock:
worker_lock_1.claim(workerforbuilder1, counting_access)
self.assertFalse(Build.canStartWithWorkerForBuilder(lock_list, workerforbuilder1))
self.assertFalse(Build.canStartWithWorkerForBuilder(lock_list, workerforbuilder1))
self.assertTrue(Build.canStartWithWorkerForBuilder(lock_list, workerforbuilder2))
self.assertTrue(Build.canStartWithWorkerForBuilder(lock_list, workerforbuilder2))
worker_lock_1.release(workerforbuilder1, counting_access)
def testBuildLocksAcquired(self):
b = self.build
lock = WorkerLock('lock')
claimCount = [0]
lock_access = lock.access('counting')
lock.access = lambda mode: lock_access
real_workerlock = yield b.builder.botmaster.getLockByID(lock, 0)
real_lock = real_workerlock.getLockForWorker(self.workerforbuilder.worker.workername)
def claim(owner, access):
claimCount[0] += 1
return real_lock.old_claim(owner, access)
real_lock.old_claim = real_lock.claim
real_lock.claim = claim
yield b.setLocks([lock_access])
step = FakeBuildStep()
b.setStepFactories([FakeStepFactory(step)])
b.startBuild(self.workerforbuilder)
self.assertEqual(b.results, SUCCESS)
self.assertEqual(claimCount[0], 1)
def testBuildLocksOrder(self):
"""Test that locks are acquired in FIFO order; specifically that
counting locks cannot jump ahead of exclusive locks"""
eBuild = self.build
cBuilder = FakeBuilder(self.master)
cBuild = Build([self.request])
cBuild.setBuilder(cBuilder)
eWorker = Mock()
cWorker = Mock()
eWorker.worker = self.worker
cWorker.worker = self.worker
eWorker.prepare = cWorker.prepare = lambda _: True
eWorker.ping = cWorker.ping = lambda: True
l = WorkerLock('lock', 2)
claimLog = []
realLock = self.master.botmaster.getLockByID(l).getLock(self.worker)
def claim(owner, access):
claimLog.append(owner)
return realLock.oldClaim(owner, access)
realLock.oldClaim = realLock.claim
realLock.claim = claim
eBuild.setLocks([l.access('exclusive')])
cBuild.setLocks([l.access('counting')])
fakeBuild = Mock()
fakeBuildAccess = l.access('counting')
realLock.claim(fakeBuild, fakeBuildAccess)
step = Mock()
step.return_value = step
step.startStep.return_value = SUCCESS
eBuild.setStepFactories([FakeStepFactory(step)])
cBuild.setStepFactories([FakeStepFactory(step)])
e = eBuild.startBuild(FakeBuildStatus(), eWorker)
c = cBuild.startBuild(FakeBuildStatus(), cWorker)
d = defer.DeferredList([e, c])
realLock.release(fakeBuild, fakeBuildAccess)
def check(ign):
self.assertEqual(eBuild.results, SUCCESS)
self.assertEqual(cBuild.results, SUCCESS)
self.assertEqual(claimLog, [fakeBuild, eBuild, cBuild])
d.addCallback(check)
return d
def testBuildLocksOrder(self):
"""Test that locks are acquired in FIFO order; specifically that
counting locks cannot jump ahead of exclusive locks"""
eBuild = self.build
cBuilder = FakeBuilder(self.master)
cBuild = Build([self.request])
cBuild.setBuilder(cBuilder)
eWorker = Mock()
cWorker = Mock()
eWorker.worker = self.worker
cWorker.worker = self.worker
eWorker.prepare = cWorker.prepare = lambda _: True
eWorker.ping = cWorker.ping = lambda: True
l = WorkerLock("lock", 2)
claimLog = []
realLock = self.master.botmaster.getLockByID(l).getLock(self.worker)
def claim(owner, access):
claimLog.append(owner)
return realLock.oldClaim(owner, access)
realLock.oldClaim = realLock.claim
realLock.claim = claim
eBuild.setLocks([l.access("exclusive")])
cBuild.setLocks([l.access("counting")])
fakeBuild = Mock()
fakeBuildAccess = l.access("counting")
realLock.claim(fakeBuild, fakeBuildAccess)
step = Mock()
step.return_value = step
step.startStep.return_value = SUCCESS
eBuild.setStepFactories([FakeStepFactory(step)])
cBuild.setStepFactories([FakeStepFactory(step)])
e = eBuild.startBuild(FakeBuildStatus(), eWorker)
c = cBuild.startBuild(FakeBuildStatus(), cWorker)
d = defer.DeferredList([e, c])
realLock.release(fakeBuild, fakeBuildAccess)
def check(ign):
self.assertEqual(eBuild.results, SUCCESS)
self.assertEqual(cBuild.results, SUCCESS)
self.assertEqual(claimLog, [fakeBuild, eBuild, cBuild])
d.addCallback(check)
return d
def testBuild_canAcquireLocks(self):
b = self.build
workerforbuilder1 = Mock()
workerforbuilder2 = Mock()
lock = WorkerLock('lock')
counting_access = lock.access('counting')
real_lock = yield b.builder.botmaster.getLockByID(lock, 0)
# no locks, so both these pass (call twice to verify there's no
# state/memory)
lock_list = [(real_lock, counting_access)]
self.assertTrue(
Build._canAcquireLocks(lock_list, workerforbuilder1))
self.assertTrue(
Build._canAcquireLocks(lock_list, workerforbuilder1))
self.assertTrue(
Build._canAcquireLocks(lock_list, workerforbuilder2))
self.assertTrue(
Build._canAcquireLocks(lock_list, workerforbuilder2))
worker_lock_1 = real_lock.getLockForWorker(
workerforbuilder1.worker.workername)
worker_lock_2 = real_lock.getLockForWorker(
workerforbuilder2.worker.workername)
# then have workerforbuilder2 claim its lock:
worker_lock_2.claim(workerforbuilder2, counting_access)
self.assertTrue(
Build._canAcquireLocks(lock_list, workerforbuilder1))
self.assertTrue(
Build._canAcquireLocks(lock_list, workerforbuilder1))
self.assertFalse(
Build._canAcquireLocks(lock_list, workerforbuilder2))
self.assertFalse(
Build._canAcquireLocks(lock_list, workerforbuilder2))
worker_lock_2.release(workerforbuilder2, counting_access)
# then have workerforbuilder1 claim its lock:
worker_lock_1.claim(workerforbuilder1, counting_access)
self.assertFalse(
Build._canAcquireLocks(lock_list, workerforbuilder1))
self.assertFalse(
Build._canAcquireLocks(lock_list, workerforbuilder1))
self.assertTrue(
Build._canAcquireLocks(lock_list, workerforbuilder2))
self.assertTrue(
Build._canAcquireLocks(lock_list, workerforbuilder2))
worker_lock_1.release(workerforbuilder1, counting_access)
def testBuildLocksOrder(self):
"""Test that locks are acquired in FIFO order; specifically that
counting locks cannot jump ahead of exclusive locks"""
eBuild = self.build
cBuilder = FakeBuilder(self.master)
cBuild = Build([self.request])
cBuild.setBuilder(cBuilder)
eWorker = Mock()
cWorker = Mock()
eWorker.worker = self.worker
cWorker.worker = self.worker
eWorker.substantiate_if_needed = cWorker.substantiate_if_needed = lambda _: True
eWorker.ping = cWorker.ping = lambda: True
lock = WorkerLock('lock', 2)
claimLog = []
real_workerlock = yield self.master.botmaster.getLockByID(lock, 0)
realLock = real_workerlock.getLockForWorker(self.worker.workername)
def claim(owner, access):
claimLog.append(owner)
return realLock.oldClaim(owner, access)
realLock.oldClaim = realLock.claim
realLock.claim = claim
yield eBuild.setLocks([lock.access('exclusive')])
yield cBuild.setLocks([lock.access('counting')])
fakeBuild = Mock()
fakeBuildAccess = lock.access('counting')
realLock.claim(fakeBuild, fakeBuildAccess)
step = FakeBuildStep()
eBuild.setStepFactories([FakeStepFactory(step)])
cBuild.setStepFactories([FakeStepFactory(step)])
e = eBuild.startBuild(eWorker)
c = cBuild.startBuild(cWorker)
d = defer.DeferredList([e, c])
realLock.release(fakeBuild, fakeBuildAccess)
yield d
self.assertEqual(eBuild.results, SUCCESS)
self.assertEqual(cBuild.results, SUCCESS)
self.assertEqual(claimLog, [fakeBuild, eBuild, cBuild])
def test_init_maxCountForWorker_old_api_warns(self):
counts = {'w1': 10, 'w2': 20}
with assertProducesWarning(
DeprecatedWorkerNameWarning,
message_pattern="'maxCountForSlave' keyword argument is deprecated"):
lock = WorkerLock("name", maxCount=1, maxCountForSlave=counts)
self.assertEqual(lock.maxCountForWorker, counts)
def test_worker_lock_update_from_lockid(self):
lock = RealWorkerLock('lock1')
lock.updateFromLockId(WorkerLock('lock1', maxCount=3), 0)
worker_lock = lock.getLockForWorker('worker1')
self.assertEqual(worker_lock.maxCount, 3)
lock.updateFromLockId(WorkerLock('lock1', maxCount=5), 0)
self.assertEqual(lock.lockName, 'lock1')
self.assertEqual(lock.maxCount, 5)
self.assertEqual(lock.description, '<WorkerLock(lock1, 5, {})>')
self.assertEqual(worker_lock.lockName, 'lock1')
self.assertEqual(worker_lock.maxCount, 5)
self.assertTrue(worker_lock.description.startswith(
'<WorkerLock(lock1, 5)[worker1]'))
with self.assertRaises(AssertionError):
lock.updateFromLockId(WorkerLock('lock2', maxCount=4), 0)
def test_worker_lock_init_from_lockid_count_for_worker(self):
lockid = WorkerLock('lock1', maxCount=3,
maxCountForWorker={'worker2': 5})
lock = RealWorkerLock(lockid)
self.assertEqual(lock.name, 'lock1')
self.assertEqual(lock.maxCount, 3)
worker_lock = lock.getLockForWorker('worker1')
self.assertEqual(worker_lock.maxCount, 3)
worker_lock = lock.getLockForWorker('worker2')
self.assertEqual(worker_lock.maxCount, 5)
def test_maxCountForWorker_old_api(self):
lock = WorkerLock("lock")
with assertNotProducesWarnings(DeprecatedWorkerAPIWarning):
new = lock.maxCountForWorker
with assertProducesWarning(
DeprecatedWorkerNameWarning,
message_pattern="'maxCountForSlave' attribute is deprecated"):
old = lock.maxCountForSlave
self.assertIdentical(new, old)
def test_worker_lock_init_from_lockid(self):
lockid = WorkerLock('lock1', maxCount=3)
lock = RealWorkerLock(lockid)
self.assertEqual(lock.name, 'lock1')
self.assertEqual(lock.maxCount, 3)
self.assertEqual(lock.description, '<WorkerLock(lock1, 3, {})>')
worker_lock = lock.getLockForWorker('worker1')
self.assertEqual(worker_lock.name, 'lock1')
self.assertEqual(worker_lock.maxCount, 3)
self.assertTrue(worker_lock.description.startswith(
'<WorkerLock(lock1, 3)[worker1]'))
def testBuildLocksAcquired(self):
b = self.build
lock = WorkerLock('lock')
claimCount = [0]
lock_access = lock.access('counting')
lock.access = lambda mode: lock_access
real_lock = b.builder.botmaster.getLockByID(lock) \
.getLock(self.workerforbuilder.worker)
def claim(owner, access):
claimCount[0] += 1
return real_lock.old_claim(owner, access)
real_lock.old_claim = real_lock.claim
real_lock.claim = claim
b.setLocks([lock_access])
step = FakeBuildStep()
b.setStepFactories([FakeStepFactory(step)])
b.startBuild(FakeBuildStatus(), self.workerforbuilder)
self.assertEqual(b.results, SUCCESS)
self.assertEqual(claimCount[0], 1)
def test_init_maxCountForWorker_old_api_no_warns(self):
counts = {'w1': 10, 'w2': 20}
with assertNotProducesWarnings(DeprecatedWorkerAPIWarning):
lock = WorkerLock("name", maxCount=1, maxCountForWorker=counts)
self.assertEqual(lock.maxCountForWorker, counts)
def __init__(self, Environ):
# Max number of running builds
build_lock = WorkerLock('build',
maxCount=2,
maxCountForWorker={
'farmer-worker1': 2,
})
# All repo
all_repos = {
'quantum_espresso': {
'repository': 'https://gitlab.com/QEF/q-e.git',
'branch': 'develop',
},
'quantum_espresso_GPU': {
'repository': 'https://gitlab.com/QEF/q-e-gpu.git',
'branch': 'gpu-develop',
},
'wannier90': {
'repository':
'https://github.com/wannier-developers/wannier90.git',
# 'repository': 'https://github.com/sponce24/wannier90.git',
'branch': 'develop',
},
}
############################################################################
# QE code
############################################################################
self.checkout_qe = [
steps.Git(
name="checkout_qe",
method="copy",
repourl=all_repos["quantum_espresso"]["repository"],
branch=all_repos["quantum_espresso"]["branch"],
haltOnFailure=True,
alwaysUseLatest=True,
)
]
self.checkout_qe_GPU = [
steps.Git(
name="checkout_qe",
method="copy",
repourl=all_repos["quantum_espresso_GPU"]["repository"],
branch=all_repos["quantum_espresso_GPU"]["branch"],
haltOnFailure=True,
alwaysUseLatest=True,
)
]
self.configure_qe = [
ShellCommand(name="configure_qe",
command=["./configure"],
env=Environ,
workdir="build",
locks=[build_lock.access('counting')],
haltOnFailure=True,
descriptionDone=["configure_qe"])
]
self.configure_qe_hdf5 = [
ShellCommand(name="configure_qe_hdf5",
command=[
"./configure",
"--with-hdf5=/home/buildbot/local/hdf5-104-gcc102"
],
env=Environ,
workdir="build",
locks=[build_lock.access('counting')],
haltOnFailure=True,
descriptionDone=["configure_qe_hdf5"])
]
self.configure_qe_serial = [
ShellCommand(name="configure_qe_serial",
command=["./configure", "--enable-parallel=no"],
env=Environ,
workdir="build",
locks=[build_lock.access('counting')],
haltOnFailure=True,
descriptionDone=["configure_qe_serial"])
]
self.configure_qe_mp = [
ShellCommand(name="configure_qe_mp",
command=[
"./configure", "--enable-openmp",
"--enable-parallel"
],
env=Environ,
workdir="build",
locks=[build_lock.access('counting')],
haltOnFailure=True,
descriptionDone=["configure_qe_mp"])
]
self.configure_qe_GPU = [
ShellCommand(name="configure_qe_GPU",
command=[
"./configure",
"--with-cuda=/opt/pgi/linux86-64/2019/cuda/10.1/",
"--with-cuda-runtime=10.1", "--with-cuda-cc=60",
"--with-scalapack=no", "--enable-openmp"
],
env=Environ,
workdir="build",
locks=[build_lock.access('counting')],
haltOnFailure=True,
descriptionDone=["configure_qe_GPU"])
]
self.debug_flags = [
ShellCommand(
name="debug_flags",
command=Interpolate(
'sed -i "s/FFLAGS = -O3 -g/FFLAGS = -g -Wall -fbounds-check -frange-check -finit-integer=987654321 -finit-real=nan -finit-logical=true -finit-character=64/g" make.inc'
),
env=Environ,
workdir="build",
locks=[build_lock.access('counting')],
haltOnFailure=True,
descriptionDone=["debug_flags"])
]
self.env_qe1 = [
ShellCommand(
name="env_qe1",
command=Interpolate(
'sed -i "s/TESTCODE_NPROCS=4/TESTCODE_NPROCS=4/g" ENVIRONMENT'
),
env=Environ,
workdir="build/test-suite/",
locks=[build_lock.access('counting')],
haltOnFailure=True,
descriptionDone=["env_qe1"])
]
self.env_qe2 = [
ShellCommand(
name="env_qe2",
command=Interpolate(
'echo "export OMP_NUM_THREADS=1" >> ENVIRONMENT'),
#command=["cat","'export OMP_NUM_THREADS=2'",">>", "ENVIRONMENT"],
env=Environ,
workdir="build/test-suite/",
locks=[build_lock.access('counting')],
haltOnFailure=True,
descriptionDone=["env_qe2"])
]
self.make_pw = [
ShellCommand(
name="make_pw",
command=["make", "-j", "4", "pwall", "cp", "ld1", "hp"],
env=Environ,
workdir="build",
haltOnFailure=True,
descriptionDone=["make_pw"],
locks=[build_lock.access('counting')])
]
self.make_pw_GPU = [
ShellCommand(name="make_pw_GPU",
command=["make", "-j", "4", "pw"],
env=Environ,
workdir="build",
haltOnFailure=True,
descriptionDone=["make_pw"],
locks=[build_lock.access('counting')])
]
self.make_ph = [
ShellCommand(name="make_ph",
command=["make", "ph"],
env=Environ,
workdir="build",
haltOnFailure=True,
descriptionDone=["make_ph"],
locks=[build_lock.access('counting')])
]
self.make_epw0 = [
ShellCommand(name="make_epw0",
command=["make"],
env=Environ,
workdir="build/EPW/src/",
haltOnFailure=True,
descriptionDone=["make_epw"],
locks=[build_lock.access('counting')])
]
self.make_epw = [
ShellCommand(name="make_epw",
command=["make", "epw"],
env=Environ,
workdir="build",
haltOnFailure=True,
descriptionDone=["make_epw"],
locks=[build_lock.access('counting')])
]
self.make_lr = [
ShellCommand(
name="make_lr",
command=["make", "-j", "8", "lrmods"],
env=Environ,
workdir="build",
haltOnFailure=True,
descriptionDone=["make_lr"],
locks=[build_lock.access('counting')],
)
]
self.test_clean = [
ShellCommand(
name="test_clean",
command=["make", "clean"],
env=Environ,
workdir="build/test-suite",
descriptionDone=["test_clean"],
locks=[build_lock.access('counting')],
)
]
self.clean = [
ShellCommand(command=["make", "veryclean"],
alwaysRun=True,
flunkOnFailure=False,
workdir="build")
]
self.test0 = [
ShellCommand(name="test_prolog",
command=["make", "prolog"],
env=Environ,
workdir="build/test-suite",
haltOnFailure=False,
descriptionDone=["make prolog"],
locks=[build_lock.access('counting')])
]
self.test_para_PW = [
ShellCommand(name="PW_para",
command=["make", "run-tests-pw-parallel"],
env=Environ,
workdir="build/test-suite",
haltOnFailure=False,
descriptionDone=["PW para tests"],
locks=[build_lock.access('counting')])
]
self.test_serial_PW = [
ShellCommand(name="PW_serial",
command=["make", "run-tests-pw-serial"],
env=Environ,
workdir="build/test-suite",
haltOnFailure=False,
descriptionDone=["PW serial tests"],
locks=[build_lock.access('counting')])
]
self.test_para_CP = [
ShellCommand(name="CP_para",
command=["make", "run-tests-cp-parallel"],
env=Environ,
workdir="build/test-suite",
haltOnFailure=False,
descriptionDone=["CP para tests"],
locks=[build_lock.access('counting')])
]
self.test_serial_CP = [
ShellCommand(name="CP_serial",
command=["make", "run-tests-cp-serial"],
env=Environ,
workdir="build/test-suite",
haltOnFailure=False,
descriptionDone=["CP serial tests"],
locks=[build_lock.access('counting')])
]
self.test_para_PH = [
ShellCommand(name="PH_para",
command=["make", "run-tests-ph-parallel"],
env=Environ,
workdir="build/test-suite",
haltOnFailure=False,
descriptionDone=["PH para tests"],
locks=[build_lock.access('counting')])
]
self.test_serial_PH = [
ShellCommand(name="PH_serial",
command=["make", "run-tests-ph-serial"],
env=Environ,
workdir="build/test-suite",
haltOnFailure=False,
descriptionDone=["PH serial tests"],
locks=[build_lock.access('counting')])
]
self.test_para_EPW = [
ShellCommand(name="EPW_para",
command=["make", "run-tests-epw-parallel"],
env=Environ,
workdir="build/test-suite",
haltOnFailure=False,
descriptionDone=["EPW para tests"],
locks=[build_lock.access('counting')])
]
self.test_serial_EPW = [
ShellCommand(name="EPW_serial",
command=["make", "run-tests-epw-serial"],
env=Environ,
workdir="build/test-suite",
haltOnFailure=False,
descriptionDone=["EPW serial tests"],
locks=[build_lock.access('counting')])
]
self.test_para_HP = [
ShellCommand(name="HP_para",
command=["make", "run-tests-hp-parallel"],
env=Environ,
workdir="build/test-suite",
haltOnFailure=False,
descriptionDone=["HP para tests"],
locks=[build_lock.access('counting')])
]
self.test_serial_HP = [
ShellCommand(name="HP_serial",
command=["make", "run-tests-hp-serial"],
env=Environ,
workdir="build/test-suite",
haltOnFailure=False,
descriptionDone=["HP serial tests"],
locks=[build_lock.access('counting')])
]
############################################################################
# SGW code
############################################################################
# self.configure_qe2 = [ShellCommand(
# name="configure_qe",
# command=["./configure"],
# env=Environ,
# workdir="build",
# locks=[build_lock.access('counting')],
# haltOnFailure = True,descriptionDone=["configure_qe"]
# )]
#
# self.make_pw2 = [ShellCommand(
# name="make_pw",
# command=["make","pw","lrmods"],
# env=Environ,
# workdir="build",
# haltOnFailure=True, descriptionDone=["make_pw"],
# locks=[build_lock.access('counting')]
# )]
#
# self.checkout_sgw = [steps.Git(
# name="checkout_sgw",
# repourl=all_repos["sternheimer_gw"]["repository"],
# branch=all_repos["sternheimer_gw"]["branch"],
# workdir="build/SGW",
# haltOnFailure = True,
# alwaysUseLatest = True,
# )]
#
# self.make_clean = [ShellCommand(
# name="make_clean",
# command=["make", "clean"],
# env=Environ,
# workdir="build/SGW",
# haltOnFailure = True,
# descriptionDone = ["make_clean"],
# locks=[build_lock.access('counting')],
# )]
#
# self.make_sgw = [ShellCommand(
# name="make_sgw",
# command=["make"],
# env=Environ,
# workdir="build/SGW",
# haltOnFailure = True,
# descriptionDone = ["make_sgw"],
# locks=[build_lock.access('counting')],
# )]
#
# self.test_sgw = [ShellCommand(
# name="test_sgw",
# command=["make", "run-tests"],
# env=Environ,
# workdir="build/SGW/test-suite",
# haltOnFailure = True,
# descriptionDone = ["test_sgw"],
# locks=[build_lock.access('counting')],
# )]
#
# self.test_clean_sgw = [ShellCommand(
# name="test_clean",
# command=["make", "clean"],
# env=Environ,
# workdir="build/SGW/test-suite",
# descriptionDone = ["test_clean"],
# locks=[build_lock.access('counting')],
# )]
############################################################################
# Wannier code
############################################################################
self.checkout_wannier = [
steps.Git(
name="checkout_wannier",
method="copy",
workdir="build/WAN",
repourl=all_repos["wannier90"]["repository"],
branch=all_repos["wannier90"]["branch"],
haltOnFailure=True,
alwaysUseLatest=True,
)
]
self.cpconfig = [
ShellCommand(
name="cp_config",
command=[
"cp",
"test-suite/config/TestFarm/farmer_gcc640_serial.inc",
"make.inc"
],
env=Environ,
workdir="build/WAN",
haltOnFailure=True,
descriptionDone=["cp_config"],
locks=[build_lock.access('counting')])
]
self.cpgcc730 = [
ShellCommand(
name="cp_config",
command=[
"cp",
"test-suite/config/TestFarm/farmer_gcc730_openmpi1107.inc",
"make.inc"
],
env=Environ,
workdir="build/WAN",
haltOnFailure=True,
descriptionDone=["cp_config"],
locks=[build_lock.access('counting')])
]
self.cpintel17 = [
ShellCommand(
name="cp_config",
command=[
"cp",
"test-suite/config/TestFarm/farmer_intel17_openmpi313.inc",
"make.inc"
],
env=Environ,
workdir="build/WAN",
haltOnFailure=True,
descriptionDone=["cp_config"],
locks=[build_lock.access('counting')])
]
self.cpintel17i = [
ShellCommand(
name="cp_config",
command=[
"cp", "test-suite/config/TestFarm/farmer_intel17_impi.inc",
"make.inc"
],
env=Environ,
workdir="build/WAN",
haltOnFailure=True,
descriptionDone=["cp_config"],
locks=[build_lock.access('counting')])
]
self.cpintel18 = [
ShellCommand(
name="cp_config",
command=[
"cp",
"test-suite/config/TestFarm/farmer_intel18_openmpi313.inc",
"make.inc"
],
env=Environ,
workdir="build/WAN",
haltOnFailure=True,
descriptionDone=["cp_config"],
locks=[build_lock.access('counting')])
]
self.cppgi18 = [
ShellCommand(
name="cp_config",
command=[
"cp",
"test-suite/config/TestFarm/farmer_pgi18_mvapich23b.inc",
"make.inc"
],
env=Environ,
workdir="build/WAN",
haltOnFailure=True,
descriptionDone=["cp_config"],
locks=[build_lock.access('counting')])
]
self.clean_wannier = [
ShellCommand(
name="clean_wannier",
command=["make", "clean"],
env=Environ,
workdir="build/WAN",
haltOnFailure=True,
descriptionDone=["clean_wannier"],
locks=[build_lock.access('counting')],
)
]
self.clean_tests = [
ShellCommand(
name="clean_tests",
command=["python", "clean_tests"],
env=Environ,
workdir="build/WAN/test-suite",
haltOnFailure=True,
descriptionDone=["clean_tests"],
locks=[build_lock.access('counting')],
)
]
self.make_wannier = [
ShellCommand(
name="make_wannier",
command=["make"],
env=Environ,
workdir="build/WAN",
haltOnFailure=True,
descriptionDone=["make_wannier"],
locks=[build_lock.access('counting')],
)
]
self.make_wannier2 = [
ShellCommand(
name="make_wannier2",
command=["make", "default", "w90chk2chk"],
env=Environ,
workdir="build/WAN",
haltOnFailure=True,
descriptionDone=["make_wannier2"],
locks=[build_lock.access('counting')],
)
]
self.test_wannier_serial = [
ShellCommand(
name="test_wannier_seq",
command=["./run_tests", "--category=default"],
env=Environ,
workdir="build/WAN/test-suite",
haltOnFailure=True,
descriptionDone=["test_wannier_seq"],
locks=[build_lock.access('counting')],
)
]
self.test_wannier_para = [
ShellCommand(
name="test_wannier_para",
command=["./run_tests", "--category=default", "--numprocs=4"],
env=Environ,
workdir="build/WAN/test-suite",
haltOnFailure=True,
descriptionDone=["test_wannier_para"],
locks=[build_lock.access('counting')],
)
]
from buildbot.locks import MasterLock, WorkerLock
from buildbot.process import factory, results
from buildbot.process.properties import Property
from buildbot.scheduler import Triggerable
from buildbot.schedulers.timed import NightlyTriggerable
from buildbot.schedulers.basic import SingleBranchScheduler
from buildbot.schedulers.forcesched import ForceScheduler, CodebaseParameter, StringParameter, BooleanParameter
from buildbot.steps.source.git import Git
from buildbot.steps.trigger import Trigger
from buildbot.steps.shell import Configure, Compile, Test
import config
import steps
# Lock to avoid running more than 1 build at the same time on a worker
lock_build = WorkerLock("worker", maxCount=1)
# builds contains all build trees
# ccache is the cache for compiled objects used by ccache
# packages contains all final packages generated
# src contains the source trees
# triggers is some working directory needed by triggers
# bshomes is used for various build systems (like Gradle) to avoid downloading things at each run
for data_dir in ["builds", "ccache", "packages", "src", "triggers", "bshomes"]:
os.makedirs(os.path.join(config.buildbot_data_dir, data_dir),
exist_ok=True)
class Build:
__slots__ = ['name']