def testStopBuildWaitingForLocks_lostRemote(self): b = self.build l = SlaveLock('lock') lock_access = l.access('counting') l.access = lambda mode: lock_access real_lock = b.builder.botmaster.getLockByID(l) \ .getLock(self.slavebuilder.slave) b.setLocks([lock_access]) step = Mock() step.return_value = step step.startStep.return_value = SUCCESS step.alwaysRun = False b.setStepFactories([FakeStepFactory(step)]) real_lock.claim(Mock(), l.access('counting')) def acquireLocks(res=None): retval = Build.acquireLocks(b, res) b.lostRemote() return retval b.acquireLocks = acquireLocks b.startBuild(FakeBuildStatus(), None, self.slavebuilder) self.assert_(('startStep', (self.slavebuilder.slave.conn,), {}) not in step.method_calls) self.assert_(b.currentStep is None) self.assertEqual(b.results, RETRY) self.assert_(('interrupt', ('stop it',), {}) not in step.method_calls) self.build.build_status.setText.assert_called_with(["retry", "lost", "connection"]) self.build.build_status.setResults.assert_called_with(RETRY)
def testStopBuildWaitingForStepLocks(self): b = self.build slavebuilder = Mock() l = SlaveLock('lock') lock_access = l.access('counting') l.access = lambda mode: lock_access real_lock = b.builder.botmaster.getLockByID(l).getLock(slavebuilder) step = LoggingBuildStep(locks=[lock_access]) b.setStepFactories([FakeStepFactory(step)]) real_lock.claim(Mock(), l.access('counting')) gotLocks = [False] def acquireLocks(res=None): gotLocks[0] = True retval = LoggingBuildStep.acquireLocks(step, res) self.assert_(b.currentStep is step) b.stopBuild('stop it') return retval step.acquireLocks = acquireLocks step.setStepStatus = Mock() step._step_status = Mock() step.step_status.addLog().chunkSize = 10 step.step_status.getLogs.return_value = [] b.startBuild(FakeBuildStatus(), None, slavebuilder) self.assertEqual(gotLocks, [True]) self.assert_(('stepStarted', (), {}) in step.step_status.method_calls) self.assertEqual(b.result, EXCEPTION)
def test_locks_released_after_interrupt(self): l = SlaveLock('lock') lock_access = l.access('exclusive') l.access = lambda mode: lock_access class InterruptBuildStep(buildstep.BuildStep): def __init__(self,unit_test_obj,*args,**kw): self.unit_test_obj = unit_test_obj buildstep.BuildStep.__init__(self,*args,**kw) def start(self): self.interrupt("stop") def interrupt(self,arg): slavebuilder = self.build.slavebuilder.slave getLockByID = self.build.builder.botmaster.getLockByID real_lock = getLockByID(lock_access.lockid) l = real_lock.getLock(slavebuilder) self.unit_test_obj.assert_(not l.isAvailable(self,lock_access)) buildstep.BuildStep.interrupt(self,arg) self.unit_test_obj.assert_(l.isAvailable(self,lock_access)) buildstep.BuildStep.finished(self,EXCEPTION) step=InterruptBuildStep(self,locks=[l]) self.setupStep(step) self.setupLockEnv() self.expectOutcome(result=EXCEPTION, status_text=['generic','interrupted']) return self.runStep()
def test_locks_released_after_success(self): l = SlaveLock('lock') lock_access = l.access('exclusive') class FakeBuildStepCheckLock(buildstep.BuildStep): def __init__(self,unit_test_obj,*args,**kw): self.unit_test_obj = unit_test_obj buildstep.BuildStep.__init__(self,*args,**kw) def start(self): return self.finished(SUCCESS) def finished(self,res): slavebuilder = self.build.slavebuilder.slave getLockByID=self.build.builder.botmaster.getLockByID real_lock = getLockByID(lock_access.lockid) l = real_lock.getLock(slavebuilder) self.unit_test_obj.assert_(not l.isAvailable(self,lock_access)) buildstep.BuildStep.finished(self,res) self.unit_test_obj.assert_(l.isAvailable(self,lock_access)) step=FakeBuildStepCheckLock(self,locks=[lock_access]) self.setupStep(step) self.setupLockEnv() self.expectOutcome(result=SUCCESS, status_text=['generic']) return self.runStep()
def testStopBuildWaitingForStepLocks(self): b = self.build l = SlaveLock('lock') lock_access = l.access('counting') l.access = lambda mode: lock_access real_lock = b.builder.botmaster.getLockByID(l) \ .getLock(self.slavebuilder.slave) step = LoggingBuildStep(locks=[lock_access]) b.setStepFactories([FakeStepFactory(step)]) real_lock.claim(Mock(), l.access('counting')) gotLocks = [False] def acquireLocks(res=None): gotLocks[0] = True retval = LoggingBuildStep.acquireLocks(step, res) self.assert_(b.currentStep is step) b.stopBuild('stop it') return retval step.acquireLocks = acquireLocks b.startBuild(FakeBuildStatus(), None, self.slavebuilder) self.assertEqual(gotLocks, [True]) self.assertEqual(b.results, CANCELLED)
def testStopBuildWaitingForLocks(self): r = FakeRequest() b = Build([r]) b.setBuilder(Mock()) b.builder.botmaster = FakeMaster() slavebuilder = Mock() status = Mock() l = SlaveLock('lock') lock_access = l.access('counting') l.access = lambda mode: lock_access real_lock = b.builder.botmaster.getLockByID(l).getLock(slavebuilder) b.setLocks([l]) step = Mock() step.return_value = step step.startStep.return_value = SUCCESS b.setStepFactories([(step, {})]) real_lock.claim(Mock(), l.access('counting')) def acquireLocks(res=None): retval = Build.acquireLocks(b, res) b.stopBuild('stop it') return retval b.acquireLocks = acquireLocks b.startBuild(status, None, slavebuilder) self.assert_( ('startStep', (b.remote,), {}) not in step.method_calls) self.assert_(b.currentStep is None) self.assert_("Interrupted" in b.text, b.text) self.assertEqual(b.result, FAILURE) self.assert_( ('interrupt', ('stop it',), {}) not in step.method_calls)
def testBuildLocksAcquired(self): r = FakeRequest() b = Build([r]) b.setBuilder(Mock()) b.builder.botmaster = FakeMaster() slavebuilder = Mock() status = Mock() l = SlaveLock("lock") claimCount = [0] lock_access = l.access("counting") l.access = lambda mode: lock_access real_lock = b.builder.botmaster.getLockByID(l).getLock(slavebuilder) def claim(owner, access): claimCount[0] += 1 return real_lock.old_claim(owner, access) real_lock.old_claim = real_lock.claim real_lock.claim = claim b.setLocks([l]) step = Mock() step.return_value = step step.startStep.return_value = SUCCESS b.setStepFactories([(step, {})]) b.startBuild(status, None, slavebuilder) self.assertEqual(b.result, SUCCESS) self.assert_(("startStep", (b.remote,), {}) in step.method_calls) self.assertEquals(claimCount[0], 1)
def testStopBuildWaitingForLocks(self): b = self.build l = SlaveLock("lock") lock_access = l.access("counting") l.access = lambda mode: lock_access real_lock = b.builder.botmaster.getLockByID(l).getLock(self.slavebuilder.slave) b.setLocks([lock_access]) step = Mock() step.return_value = step step.startStep.return_value = SUCCESS step.alwaysRun = False b.setStepFactories([FakeStepFactory(step)]) real_lock.claim(Mock(), l.access("counting")) def acquireLocks(res=None): retval = Build.acquireLocks(b, res) b.stopBuild("stop it") return retval b.acquireLocks = acquireLocks b.startBuild(FakeBuildStatus(), None, self.slavebuilder) self.assert_(("startStep", (self.slavebuilder.slave.conn,), {}) not in step.method_calls) self.assert_(b.currentStep is None) self.assertEqual(b.result, CANCELLED) self.assert_(("interrupt", ("stop it",), {}) not in step.method_calls)
def testStopBuildWaitingForLocks(self): b = self.build slavebuilder = Mock() l = SlaveLock('lock') lock_access = l.access('counting') l.access = lambda mode: lock_access real_lock = b.builder.botmaster.getLockByID(l).getLock(slavebuilder) b.setLocks([lock_access]) step = Mock() step.return_value = step step.startStep.return_value = SUCCESS step.alwaysRun = False b.setStepFactories([FakeStepFactory(step)]) real_lock.claim(Mock(), l.access('counting')) def acquireLocks(res=None): retval = Build.acquireLocks(b, res) b.stopBuild('stop it') return retval b.acquireLocks = acquireLocks b.startBuild(FakeBuildStatus(), None, slavebuilder) self.assert_(('startStep', (slavebuilder.remote, ), {}) not in step.method_calls) self.assert_(b.currentStep is None) self.assertEqual(b.result, EXCEPTION) self.assert_(('interrupt', ('stop it', ), {}) not in step.method_calls)
def testStopBuildWaitingForLocks(self): b = self.build slavebuilder = Mock() l = SlaveLock('lock') lock_access = l.access('counting') l.access = lambda mode: lock_access real_lock = b.builder.botmaster.getLockByID(l).getLock(slavebuilder) b.setLocks([l]) step = Mock() step.return_value = step step.startStep.return_value = SUCCESS step.alwaysRun = False b.setStepFactories([(step, {})]) real_lock.claim(Mock(), l.access('counting')) def acquireLocks(res=None): retval = Build.acquireLocks(b, res) b.stopBuild('stop it') return retval b.acquireLocks = acquireLocks b.startBuild(FakeBuildStatus(), None, slavebuilder) self.assert_( ('startStep', (b.remote,), {}) not in step.method_calls) self.assert_(b.currentStep is None) self.assertEqual(b.result, EXCEPTION) self.assert_( ('interrupt', ('stop it',), {}) not in step.method_calls)
def testBuildcanStartWithSlavebuilder(self): b = self.build slavebuilder1 = Mock() slavebuilder2 = Mock() l = SlaveLock('lock') counting_access = l.access('counting') real_lock = b.builder.botmaster.getLockByID(l) # no locks, so both these pass (call twice to verify there's no state/memory) lock_list = [(real_lock, counting_access)] self.assertTrue(Build.canStartWithSlavebuilder(lock_list, slavebuilder1)) self.assertTrue(Build.canStartWithSlavebuilder(lock_list, slavebuilder1)) self.assertTrue(Build.canStartWithSlavebuilder(lock_list, slavebuilder2)) self.assertTrue(Build.canStartWithSlavebuilder(lock_list, slavebuilder2)) slave_lock_1 = real_lock.getLock(slavebuilder1.slave) slave_lock_2 = real_lock.getLock(slavebuilder2.slave) # then have slavebuilder2 claim its lock: slave_lock_2.claim(slavebuilder2, counting_access) self.assertTrue(Build.canStartWithSlavebuilder(lock_list, slavebuilder1)) self.assertTrue(Build.canStartWithSlavebuilder(lock_list, slavebuilder1)) self.assertFalse(Build.canStartWithSlavebuilder(lock_list, slavebuilder2)) self.assertFalse(Build.canStartWithSlavebuilder(lock_list, slavebuilder2)) slave_lock_2.release(slavebuilder2, counting_access) # then have slavebuilder1 claim its lock: slave_lock_1.claim(slavebuilder1, counting_access) self.assertFalse(Build.canStartWithSlavebuilder(lock_list, slavebuilder1)) self.assertFalse(Build.canStartWithSlavebuilder(lock_list, slavebuilder1)) self.assertTrue(Build.canStartWithSlavebuilder(lock_list, slavebuilder2)) self.assertTrue(Build.canStartWithSlavebuilder(lock_list, slavebuilder2)) slave_lock_1.release(slavebuilder1, counting_access)
def testStopBuildWaitingForStepLocks(self): b = self.build l = SlaveLock("lock") lock_access = l.access("counting") l.access = lambda mode: lock_access real_lock = b.builder.botmaster.getLockByID(l).getLock(self.slavebuilder.slave) step = LoggingBuildStep(locks=[lock_access]) b.setStepFactories([FakeStepFactory(step)]) real_lock.claim(Mock(), l.access("counting")) gotLocks = [False] def acquireLocks(res=None): gotLocks[0] = True retval = LoggingBuildStep.acquireLocks(step, res) self.assert_(b.currentStep is step) b.stopBuild("stop it") return retval step.acquireLocks = acquireLocks b.startBuild(FakeBuildStatus(), None, self.slavebuilder) self.assertEqual(gotLocks, [True]) self.assertEqual(b.result, CANCELLED)
def testStopBuildWaitingForLocks_lostRemote(self): b = self.build slavebuilder = Mock() l = SlaveLock('lock') lock_access = l.access('counting') l.access = lambda mode: lock_access real_lock = b.builder.botmaster.getLockByID(l).getLock(slavebuilder) b.setLocks([lock_access]) step = Mock() step.return_value = step step.startStep.return_value = SUCCESS step.alwaysRun = False b.setStepFactories([FakeStepFactory(step)]) real_lock.claim(Mock(), l.access('counting')) def acquireLocks(res=None): retval = Build.acquireLocks(b, res) b.lostRemote() return retval b.acquireLocks = acquireLocks b.startBuild(FakeBuildStatus(), None, slavebuilder) self.assert_(('startStep', (slavebuilder.remote,), {}) not in step.method_calls) self.assert_(b.currentStep is None) self.assertEqual(b.result, RETRY) self.assert_(('interrupt', ('stop it',), {}) not in step.method_calls) self.build.build_status.setText.assert_called_with(["retry", "lost", "remote"]) self.build.build_status.setResults.assert_called_with(RETRY)
def testBuildLocksAcquired(self): b = self.build l = SlaveLock('lock') claimCount = [0] lock_access = l.access('counting') l.access = lambda mode: lock_access real_lock = b.builder.botmaster.getLockByID(l) \ .getLock(self.slavebuilder.slave) def claim(owner, access): claimCount[0] += 1 return real_lock.old_claim(owner, access) real_lock.old_claim = real_lock.claim real_lock.claim = claim b.setLocks([lock_access]) step = Mock() step.return_value = step step.startStep.return_value = SUCCESS b.setStepFactories([FakeStepFactory(step)]) b.startBuild(FakeBuildStatus(), None, self.slavebuilder) self.assertEqual(b.results, SUCCESS) self.assertIn(('startStep', (self.slavebuilder.slave.conn,), {}), step.method_calls) self.assertEquals(claimCount[0], 1)
def testBuildLocksAcquired(self): b = self.build l = SlaveLock("lock") claimCount = [0] lock_access = l.access("counting") l.access = lambda mode: lock_access real_lock = b.builder.botmaster.getLockByID(l).getLock(self.slavebuilder.slave) def claim(owner, access): claimCount[0] += 1 return real_lock.old_claim(owner, access) real_lock.old_claim = real_lock.claim real_lock.claim = claim b.setLocks([lock_access]) step = Mock() step.return_value = step step.startStep.return_value = SUCCESS b.setStepFactories([FakeStepFactory(step)]) b.startBuild(FakeBuildStatus(), None, self.slavebuilder) self.assertEqual(b.result, SUCCESS) self.assertIn(("startStep", (self.slavebuilder.slave.conn,), {}), step.method_calls) self.assertEquals(claimCount[0], 1)
def testBuildLocksAcquired(self): r = FakeRequest() b = Build([r]) b.setBuilder(Mock()) b.builder.botmaster = FakeMaster() slavebuilder = Mock() status = Mock() l = SlaveLock('lock') claimCount = [0] lock_access = l.access('counting') l.access = lambda mode: lock_access real_lock = b.builder.botmaster.getLockByID(l).getLock(slavebuilder) def claim(owner, access): claimCount[0] += 1 return real_lock.old_claim(owner, access) real_lock.old_claim = real_lock.claim real_lock.claim = claim b.setLocks([l]) step = Mock() step.return_value = step step.startStep.return_value = SUCCESS b.setStepFactories([(step, {})]) b.startBuild(status, None, slavebuilder) self.assertEqual(b.result, SUCCESS) self.assert_( ('startStep', (b.remote,), {}) in step.method_calls) self.assertEquals(claimCount[0], 1)
def testBuildWaitingForLocks(self): b = self.build slavebuilder = Mock() l = SlaveLock('lock') claimCount = [0] lock_access = l.access('counting') l.access = lambda mode: lock_access real_lock = b.builder.botmaster.getLockByID(l).getLock(slavebuilder.slave) def claim(owner, access): claimCount[0] += 1 return real_lock.old_claim(owner, access) real_lock.old_claim = real_lock.claim real_lock.claim = claim b.setLocks([lock_access]) step = Mock() step.return_value = step step.startStep.return_value = SUCCESS b.setStepFactories([FakeStepFactory(step)]) real_lock.claim(Mock(), l.access('counting')) b.startBuild(FakeBuildStatus(), None, slavebuilder) self.assert_(('startStep', (slavebuilder.remote,), {}) not in step.method_calls) self.assertEquals(claimCount[0], 1) self.assert_(b.currentStep is None) self.assert_(b._acquiringLock is not None)
def testBuildLocksAcquired(self): b = self.build slavebuilder = Mock() l = SlaveLock('lock') claimCount = [0] lock_access = l.access('counting') l.access = lambda mode: lock_access real_lock = b.builder.botmaster.getLockByID(l).getLock(slavebuilder) def claim(owner, access): claimCount[0] += 1 return real_lock.old_claim(owner, access) real_lock.old_claim = real_lock.claim real_lock.claim = claim b.setLocks([l]) step = Mock() step.return_value = step step.startStep.return_value = SUCCESS b.setStepFactories([(step, {})]) b.startBuild(FakeBuildStatus(), None, slavebuilder) self.assertEqual(b.result, SUCCESS) self.assert_( ('startStep', (slavebuilder.remote,), {}) in step.method_calls) self.assertEquals(claimCount[0], 1)
def testBuildWaitingForLocks(self): b = self.build slavebuilder = Mock() l = SlaveLock('lock') claimCount = [0] lock_access = l.access('counting') l.access = lambda mode: lock_access real_lock = b.builder.botmaster.getLockByID(l).getLock( slavebuilder.slave) def claim(owner, access): claimCount[0] += 1 return real_lock.old_claim(owner, access) real_lock.old_claim = real_lock.claim real_lock.claim = claim b.setLocks([lock_access]) step = Mock() step.return_value = step step.startStep.return_value = SUCCESS b.setStepFactories([FakeStepFactory(step)]) real_lock.claim(Mock(), l.access('counting')) b.startBuild(FakeBuildStatus(), None, slavebuilder) self.assert_(('startStep', (slavebuilder.remote, ), {}) not in step.method_calls) self.assertEquals(claimCount[0], 1) self.assert_(b.currentStep is None) self.assert_(b._acquiringLock is not None)
def testBuildLocksOrder(self): """Test that locks are acquired in FIFO order; specifically that counting locks cannot jump ahead of exclusive locks""" eBuild = self.build cBuild = Build([self.request]) cBuilder = Mock() cBuilder.botmaster = self.master cBuild.setBuilder(cBuilder) eSlavebuilder = Mock() cSlavebuilder = Mock() slave = eSlavebuilder.slave cSlavebuilder.slave = slave l = SlaveLock('lock', 2) claimLog = [] realLock = self.master.getLockByID(l).getLock(slave) def claim(owner, access): claimLog.append(owner) return realLock.oldClaim(owner, access) realLock.oldClaim = realLock.claim realLock.claim = claim eBuild.setLocks([l.access('exclusive')]) cBuild.setLocks([l.access('counting')]) fakeBuild = Mock() fakeBuildAccess = l.access('counting') realLock.claim(fakeBuild, fakeBuildAccess) step = Mock() step.return_value = step step.startStep.return_value = SUCCESS eBuild.setStepFactories([FakeStepFactory(step)]) cBuild.setStepFactories([FakeStepFactory(step)]) e = eBuild.startBuild(FakeBuildStatus(), None, eSlavebuilder) c = cBuild.startBuild(FakeBuildStatus(), None, cSlavebuilder) d = defer.DeferredList([e, c]) realLock.release(fakeBuild, fakeBuildAccess) def check(ign): self.assertEqual(eBuild.result, SUCCESS) self.assertEqual(cBuild.result, SUCCESS) self.assertEquals(claimLog, [fakeBuild, eBuild, cBuild]) d.addCallback(check) return d
def testBuildLocksOrder(self): """Test that locks are acquired in FIFO order; specifically that counting locks cannot jump ahead of exclusive locks""" eBuild = self.build cBuilder = FakeBuilder(self.master) cBuild = Build([self.request]) cBuild.setBuilder(cBuilder) eSlavebuilder = Mock() cSlavebuilder = Mock() eSlavebuilder.slave = self.slave cSlavebuilder.slave = self.slave l = SlaveLock("lock", 2) claimLog = [] realLock = self.master.botmaster.getLockByID(l).getLock(self.slave) def claim(owner, access): claimLog.append(owner) return realLock.oldClaim(owner, access) realLock.oldClaim = realLock.claim realLock.claim = claim eBuild.setLocks([l.access("exclusive")]) cBuild.setLocks([l.access("counting")]) fakeBuild = Mock() fakeBuildAccess = l.access("counting") realLock.claim(fakeBuild, fakeBuildAccess) step = Mock() step.return_value = step step.startStep.return_value = SUCCESS eBuild.setStepFactories([FakeStepFactory(step)]) cBuild.setStepFactories([FakeStepFactory(step)]) e = eBuild.startBuild(FakeBuildStatus(), None, eSlavebuilder) c = cBuild.startBuild(FakeBuildStatus(), None, cSlavebuilder) d = defer.DeferredList([e, c]) realLock.release(fakeBuild, fakeBuildAccess) def check(ign): self.assertEqual(eBuild.result, SUCCESS) self.assertEqual(cBuild.result, SUCCESS) self.assertEquals(claimLog, [fakeBuild, eBuild, cBuild]) d.addCallback(check) return d
def test_acquire_build_Lock_step(self): b = self.build slavebuilder = Mock() l = SlaveLock("slave_builds", maxCount=1) self.assertEqual(len(b.locks), 0) step = artifact.AcquireBuildLocks(locks=[l.access('exclusive')]) b.setStepFactories([FakeStepFactory(step)]) b.startBuild(self.build_status, None, slavebuilder) b.currentStep.start() self.assertEqual(len(b.locks), 1) self.assertTrue(b.locks[0][0].owners[0][0], step)
def testStopBuildWaitingForStepLocks(self): r = FakeRequest() b = Build([r]) b.setBuilder(Mock()) b.builder.botmaster = FakeMaster() slavebuilder = Mock() status = Mock() l = SlaveLock("lock") lock_access = l.access("counting") l.access = lambda mode: lock_access real_lock = b.builder.botmaster.getLockByID(l).getLock(slavebuilder) step = LoggingBuildStep(locks=[lock_access]) def factory(*args): return step b.setStepFactories([(factory, {})]) real_lock.claim(Mock(), l.access("counting")) gotLocks = [False] def acquireLocks(res=None): gotLocks[0] = True retval = LoggingBuildStep.acquireLocks(step, res) self.assert_(b.currentStep is step) b.stopBuild("stop it") return retval step.acquireLocks = acquireLocks step.setStepStatus = Mock() step.step_status = Mock() step.step_status.addLog().chunkSize = 10 step.step_status.getLogs.return_value = [] b.startBuild(status, None, slavebuilder) self.assertEqual(gotLocks, [True]) self.assert_(("stepStarted", (), {}) in step.step_status.method_calls) self.assertEqual(b.result, EXCEPTION)
def testStopBuildWaitingForLocks(self): r = FakeRequest() b = Build([r]) b.setBuilder(Mock()) b.builder.botmaster = FakeMaster() slavebuilder = Mock() status = Mock() l = SlaveLock("lock") lock_access = l.access("counting") l.access = lambda mode: lock_access real_lock = b.builder.botmaster.getLockByID(l).getLock(slavebuilder) b.setLocks([l]) step = Mock() step.return_value = step step.startStep.return_value = SUCCESS step.alwaysRun = False b.setStepFactories([(step, {})]) real_lock.claim(Mock(), l.access("counting")) def acquireLocks(res=None): retval = Build.acquireLocks(b, res) b.stopBuild("stop it") return retval b.acquireLocks = acquireLocks b.startBuild(status, None, slavebuilder) self.assert_(("startStep", (b.remote,), {}) not in step.method_calls) self.assert_(b.currentStep is None) self.assertEqual(b.result, EXCEPTION) self.assert_(("interrupt", ("stop it",), {}) not in step.method_calls)
def __init__(self, Environ): # Max number of running builds build_lock = SlaveLock('build', maxCount=1, maxCountForSlave={ 'farmer-slave1': 1, }) # All repo all_repos = { 'quantum_espresso': { 'repository': 'https://gitlab.com/QEF/q-e.git', 'branch': 'develop', }, 'sternheimer_gw': { 'repository': 'https://github.com/mmdg-oxford/SternheimerGW.git', 'branch': 'develop', }, 'wannier90': { 'repository': 'https://github.com/wannier-developers/wannier90.git', 'branch': 'develop', }, } ############################################################################ # QE code ############################################################################ self.checkout_qe = [ steps.Git( name="checkout_qe", method="copy", repourl=all_repos["quantum_espresso"]["repository"], branch=all_repos["quantum_espresso"]["branch"], haltOnFailure=True, alwaysUseLatest=True, ) ] self.configure_qe = [ ShellCommand(name="configure_qe", command=["./configure"], env=Environ, workdir="build", locks=[build_lock.access('counting')], haltOnFailure=True, descriptionDone=["configure_qe"]) ] self.dep_qe = [ ShellCommand( name="dep_qe", #command=["make","depend"], # DBSP: Temporary until QE 6.2 is release command=["ls"], env=Environ, workdir="build", locks=[build_lock.access('counting')], haltOnFailure=True, descriptionDone=["dep_qe"]) ] self.make_pw = [ ShellCommand(name="make_pw", command=["make", "pwall", "cp", "ld1", "upf"], env=Environ, workdir="build", haltOnFailure=True, descriptionDone=["make_pw"], locks=[build_lock.access('counting')]) ] self.make_ph = [ ShellCommand(name="make_ph", command=["make", "ph"], env=Environ, workdir="build", haltOnFailure=True, descriptionDone=["make_ph"], locks=[build_lock.access('counting')]) ] self.make_epw0 = [ ShellCommand(name="make_epw0", command=["make"], env=Environ, workdir="build/EPW/src/", haltOnFailure=True, descriptionDone=["make_epw"], locks=[build_lock.access('counting')]) ] self.make_epw = [ ShellCommand(name="make_epw", command=["make", "epw"], env=Environ, workdir="build", haltOnFailure=True, descriptionDone=["make_epw"], locks=[build_lock.access('counting')]) ] self.make_lr = [ ShellCommand( name="make_lr", command=["make", "-j", "8", "lrmods"], env=Environ, workdir="build", haltOnFailure=True, descriptionDone=["make_lr"], locks=[build_lock.access('counting')], ) ] self.test_clean = [ ShellCommand( name="test_clean", command=["make", "clean"], env=Environ, workdir="build/test-suite", descriptionDone=["test_clean"], locks=[build_lock.access('counting')], ) ] self.clean = [ ShellCommand(command=["make", "veryclean"], alwaysRun=True, flunkOnFailure=False, workdir="build") ] self.test0 = [ ShellCommand(name="test_prolog", command=["make", "prolog"], env=Environ, workdir="build/test-suite", haltOnFailure=False, descriptionDone=["make prolog"], locks=[build_lock.access('counting')]) ] self.test_para_PW = [ ShellCommand(name="PW_para", command=["make", "run-tests-pw-parallel"], env=Environ, workdir="build/test-suite", haltOnFailure=False, descriptionDone=["PW para tests"], locks=[build_lock.access('counting')]) ] self.test_serial_PW = [ ShellCommand(name="PW_serial", command=["make", "run-tests-pw-serial"], env=Environ, workdir="build/test-suite", haltOnFailure=False, descriptionDone=["PW serial tests"], locks=[build_lock.access('counting')]) ] self.test_para_CP = [ ShellCommand(name="CP_para", command=["make", "run-tests-cp-parallel"], env=Environ, workdir="build/test-suite", haltOnFailure=False, descriptionDone=["CP para tests"], locks=[build_lock.access('counting')]) ] self.test_serial_CP = [ ShellCommand(name="CP_serial", command=["make", "run-tests-cp-serial"], env=Environ, workdir="build/test-suite", haltOnFailure=False, descriptionDone=["CP serial tests"], locks=[build_lock.access('counting')]) ] self.test_para_PH = [ ShellCommand(name="PH_para", command=["make", "run-tests-ph-parallel"], env=Environ, workdir="build/test-suite", haltOnFailure=False, descriptionDone=["PH para tests"], locks=[build_lock.access('counting')]) ] self.test_serial_PH = [ ShellCommand(name="PH_serial", command=["make", "run-tests-ph-serial"], env=Environ, workdir="build/test-suite", haltOnFailure=False, descriptionDone=["PH serial tests"], locks=[build_lock.access('counting')]) ] self.test_para_EPW = [ ShellCommand(name="EPW_para", command=["make", "run-tests-epw-parallel"], env=Environ, workdir="build/test-suite", haltOnFailure=False, descriptionDone=["EPW para tests"], locks=[build_lock.access('counting')]) ] self.test_serial_EPW = [ ShellCommand(name="EPW_serial", command=["make", "run-tests-epw-serial"], env=Environ, workdir="build/test-suite", haltOnFailure=False, descriptionDone=["EPW serial tests"], locks=[build_lock.access('counting')]) ] ############################################################################ # SGW code ############################################################################ self.configure_qe2 = [ ShellCommand(name="configure_qe", command=["./configure"], env=Environ, workdir="build", locks=[build_lock.access('counting')], haltOnFailure=True, descriptionDone=["configure_qe"]) ] self.make_pw2 = [ ShellCommand(name="make_pw", command=["make", "pw", "lrmods"], env=Environ, workdir="build", haltOnFailure=True, descriptionDone=["make_pw"], locks=[build_lock.access('counting')]) ] self.checkout_sgw = [ steps.Git( name="checkout_sgw", repourl=all_repos["sternheimer_gw"]["repository"], branch=all_repos["sternheimer_gw"]["branch"], workdir="build/SGW", haltOnFailure=True, alwaysUseLatest=True, ) ] self.make_sgw = [ ShellCommand( name="make_sgw", command=["make"], env=Environ, workdir="build/SGW", haltOnFailure=True, descriptionDone=["make_sgw"], locks=[build_lock.access('counting')], ) ] self.test_sgw = [ ShellCommand( name="test_sgw", command=["make", "run-tests"], env=Environ, workdir="build/SGW/test-suite", haltOnFailure=True, descriptionDone=["test_sgw"], locks=[build_lock.access('counting')], ) ] self.test_clean_sgw = [ ShellCommand( name="test_clean", command=["make", "clean"], env=Environ, workdir="build/SGW/test-suite", descriptionDone=["test_clean"], locks=[build_lock.access('counting')], ) ] ############################################################################ # Wannier code ############################################################################ self.checkout_wannier = [ steps.Git( name="checkout_wannier", method="copy", workdir="build/WAN", repourl=all_repos["wannier90"]["repository"], branch=all_repos["wannier90"]["branch"], haltOnFailure=True, alwaysUseLatest=True, ) ] self.cpconfig = [ ShellCommand( name="cp_config", command=[ "cp", "test-suite/config/EPW_testfarm/farmer_gcc485.inc", "make.inc" ], env=Environ, workdir="build/WAN", haltOnFailure=True, descriptionDone=["cp_config"], locks=[build_lock.access('counting')]) ] self.clean_wannier = [ ShellCommand( name="clean_wannier", command=["python", "clean_tests"], env=Environ, workdir="build/WAN/test-suite", haltOnFailure=True, descriptionDone=["clean_wannier"], locks=[build_lock.access('counting')], ) ] self.make_wannier = [ ShellCommand( name="make_wannier", command=["make"], env=Environ, workdir="build/WAN", haltOnFailure=True, descriptionDone=["make_wannier"], locks=[build_lock.access('counting')], ) ] self.make_wannier2 = [ ShellCommand( name="make_wannier2", command=["make", "default", "w90chk2chk"], env=Environ, workdir="build/WAN", haltOnFailure=True, descriptionDone=["make_wannier2"], locks=[build_lock.access('counting')], ) ] self.test_wannier_serial = [ ShellCommand( name="test_wannier_seq", command=["./run_tests", "--category=default"], env=Environ, workdir="build/WAN/test-suite", haltOnFailure=True, descriptionDone=["test_wannier_seq"], locks=[build_lock.access('counting')], ) ] self.test_wannier_para = [ ShellCommand( name="test_wannier_para", command=["./run_tests", "--category=default", "--numprocs=4"], env=Environ, workdir="build/WAN/test-suite", haltOnFailure=True, descriptionDone=["test_wannier_para"], locks=[build_lock.access('counting')], ) ]
haltOnFailure=True)) return factory from buildbot.config import BuilderConfig from buildbot.schedulers.basic import AnyBranchScheduler from buildbot.schedulers.forcesched import (CodebaseParameter, StringParameter, ForceScheduler, FixedParameter) from buildbot.locks import SlaveLock from ..steps import report_expected_failures_parameter from ..steps import idleSlave # A lock to prevent multiple functional tests running at the same time functionalLock = SlaveLock('functional-tests') OMNIBUS_DISTRIBUTIONS = [ 'ubuntu-14.04', 'ubuntu-15.10', 'centos-7', ] def getBuilders(slavenames): builders = [] for distribution in OMNIBUS_DISTRIBUTIONS: builders.append( BuilderConfig( name='flocker-omnibus-%s' % (distribution, ), slavenames=slavenames['aws/ubuntu-14.04'],
class FactoryCommands(object): # Base URL for performance test results. PERF_BASE_URL = config.Master.perf_base_url PERF_REPORT_URL_SUFFIX = config.Master.perf_report_url_suffix # Directory in which to save perf output data files. PERF_OUTPUT_DIR = config.Master.perf_output_dir # Use this to prevent steps which cannot be run on the same # slave from being done together (in the case where slaves are # shared by multiple builds). slave_exclusive_lock = SlaveLock('slave_exclusive', maxCount=1) # -------------------------------------------------------------------------- # PERF TEST SETTINGS # In each mapping below, the first key is the target and the second is the # perf_id. The value is the directory name in the results URL. # Configuration of most tests. PERF_TEST_MAPPINGS = { 'Release': { 'chrome-linux32-beta': 'linux32-beta', 'chrome-linux32-stable': 'linux32-stable', 'chrome-linux64-beta': 'linux64-beta', 'chrome-linux64-stable': 'linux64-stable', 'chrome-mac-beta': 'mac-beta', 'chrome-mac-stable': 'mac-stable', 'chrome-win-beta': 'win-beta', 'chrome-win-stable': 'win-stable', 'chromium-linux-targets': 'linux-targets', 'chromium-mac-targets': 'mac-targets', 'chromium-rel-frame': 'win-release-chrome-frame', 'chromium-rel-linux': 'linux-release', 'chromium-rel-linux-64': 'linux-release-64', 'chromium-rel-linux-hardy': 'linux-release-hardy', 'chromium-rel-linux-hardy-lowmem': 'linux-release-lowmem', 'chromium-rel-linux-memory': 'linux-release-memory', 'chromium-rel-linux-webkit': 'linux-release-webkit-latest', 'chromium-rel-mac': 'mac-release', 'chromium-rel-mac-memory': 'mac-release-memory', 'chromium-rel-mac5': 'mac-release-10.5', 'chromium-rel-mac6': 'mac-release-10.6', 'chromium-rel-mac5-v8': 'mac-release-10.5-v8-latest', 'chromium-rel-mac6-v8': 'mac-release-10.6-v8-latest', 'chromium-rel-mac6-webkit': 'mac-release-10.6-webkit-latest', 'chromium-rel-old-mac6': 'mac-release-old-10.6', 'chromium-rel-vista-dual': 'vista-release-dual-core', 'chromium-rel-vista-dual-v8': 'vista-release-v8-latest', 'chromium-rel-vista-memory': 'vista-release-memory', 'chromium-rel-vista-single': 'vista-release-single-core', 'chromium-rel-vista-webkit': 'vista-release-webkit-latest', 'chromium-rel-xp': 'xp-release', 'chromium-rel-xp-dual': 'xp-release-dual-core', 'chromium-rel-xp-single': 'xp-release-single-core', 'chromium-win-targets': 'win-targets', 'gpu-fyi-linux-release-ati': 'gpu-fyi-linux-release-ati', 'gpu-fyi-linux-release-intel': 'gpu-fyi-linux-release-intel', 'gpu-fyi-mac-release-ati': 'gpu-fyi-mac-release-ati', 'gpu-fyi-win7-release-ati': 'gpu-fyi-win7-release-ati', 'gpu-fyi-win7-release-intel': 'gpu-fyi-win7-release-intel', 'gpu-fyi-winxp-release-nvidia': 'gpu-fyi-winxp-release-nvidia', 'o3d-mac-experimental': 'o3d-mac-experimental', 'o3d-win-experimental': 'o3d-win-experimental', 'nacl-lucid64-spec-x86': 'nacl-lucid64-spec-x86', 'nacl-lucid64-spec-arm': 'nacl-lucid64-spec-arm', 'nacl-lucid64-spec-trans': 'nacl-lucid64-spec-trans', }, 'Debug': { 'chromium-dbg-linux': 'linux-debug', 'chromium-dbg-mac': 'mac-debug', 'chromium-dbg-xp': 'xp-debug', 'chromium-dbg-linux-try': 'linux-try-debug', 'gpu-fyi-winxp-debug-nvidia': 'gpu-fyi-winxp-debug-nvidia', }, } DEFAULT_GTEST_FILTER = '' # TODO(maruel): DEFAULT_GTEST_FILTER = '-*.FLAKY_*:*.FAILS_*' def __init__(self, factory=None, target=None, build_dir=None, target_platform=None, target_arch=None): """Initializes the SlaveCommands class. Args: factory: BuildFactory to configure. target: Build configuration, case-sensitive; probably 'Debug' or 'Release' build_dir: name of the directory within the buildbot working directory in which the solution, Debug, and Release directories are found. target_platform: Slave's OS. """ self._factory = factory self._target = target self._build_dir = build_dir self._target_platform = target_platform self._target_arch = target_arch # Starting from e.g. C:\b\build\slave\build_slave_path\build, find # C:\b\build\scripts\slave. self._script_dir = self.PathJoin('..', '..', '..', 'scripts', 'slave') self._private_script_dir = self.PathJoin(self._script_dir, '..', '..', '..', 'build_internal', 'scripts', 'slave') self._perl = self.GetExecutableName('perl') if self._target_platform == 'win32': # Steps run using a separate copy of python.exe, so it can be killed at # the start of a build. But the kill_processes (taskkill) step has to use # the original python.exe, or it kills itself. self._python = 'python_slave' else: self._python = 'python' self.working_dir = 'build' self._repository_root = 'src' self._kill_tool = self.PathJoin(self._script_dir, 'kill_processes.py') self._compile_tool = self.PathJoin(self._script_dir, 'compile.py') self._test_tool = self.PathJoin(self._script_dir, 'runtest.py') self._zip_tool = self.PathJoin(self._script_dir, 'zip_build.py') self._extract_tool = self.PathJoin(self._script_dir, 'extract_build.py') self._cleanup_temp_tool = self.PathJoin(self._script_dir, 'cleanup_temp.py') self._resource_sizes_tool = self.PathJoin(self._script_dir, 'resource_sizes.py') self._get_build_for_chromebot_tool = self.PathJoin( self._script_dir, 'get_build_for_chromebot.py') self._update_clang_tool = self.PathJoin(self._repository_root, 'tools', 'clang', 'scripts', 'update.sh') # chrome_staging directory, relative to the build directory. self._staging_dir = self.PathJoin('..', 'chrome_staging') # Util methods. def GetExecutableName(self, executable): """The executable name must be executable plus '.exe' on Windows, or else just the test name.""" if self._target_platform == 'win32': return executable + '.exe' return executable def PathJoin(self, *args): if self._target_platform == 'win32': return ntpath.normpath(ntpath.join(*args)) else: return posixpath.normpath(posixpath.join(*args)) # Basic commands def GetTestCommand(self, executable, arg_list=None, factory_properties=None, test_tool_arg_list=None): cmd = [ self._python, self._test_tool, '--target', self._target, '--build-dir', self._build_dir ] if executable in ('performance_ui_tests', 'performance_browser_tests', 'sync_performance_tests'): cmd = self.AddBuildProperties(cmd) cmd = self.AddFactoryProperties(factory_properties, cmd) # Must add test tool arg list before test arg list. if test_tool_arg_list: cmd.extend(test_tool_arg_list) cmd.append(self.GetExecutableName(executable)) if arg_list is not None: cmd.extend(arg_list) return cmd def GetPythonTestCommand(self, py_script, arg_list=None): cmd = [ self._python, self._test_tool, '--run-python-script', '--target=', '--build-dir=', py_script ] if arg_list is not None: cmd.extend(arg_list) return cmd def GetShellTestCommand(self, sh_script, arg_list=None, wrapper_args=None, factory_properties=None): """ As above, arg_list goes to the shell script, wrapper_args come before the script so the test tool uses them. """ cmd = [ self._python, self._test_tool, '--run-shell-script', '--target', self._target, '--build-dir', self._build_dir ] cmd = self.AddBuildProperties(cmd) cmd = self.AddFactoryProperties(factory_properties, cmd) if wrapper_args is not None: cmd.extend(wrapper_args) cmd.append(sh_script) if arg_list is not None: cmd.extend(arg_list) return cmd def AddBuildProperties(self, cmd=None): """Adds a WithProperties() call with build properties to cmd.""" # pylint: disable=R0201 cmd = cmd or [] # Create a WithProperties format string that includes build properties. build_properties = dict((prop, "%%(%s:-)s" % prop) for prop in ( 'blamelist', 'branch', 'buildername', 'buildnumber', 'got_revision', 'mastername', 'parentname', 'parentslavename', 'revision', 'scheduler', 'slavename', )) def gen_blamelist_string(build): blame = ','.join(build.getProperty('blamelist')) # Could be interpreted by the shell. return re.sub(r'[\&\|\^]', '', blame.replace('<', '[').replace('>', ']')) # The |separators| argument is to densify the command line. string = '--build-properties=' + json.dumps( build_properties, sort_keys=True, separators=(',', ':')) cmd.append(WithProperties(string, blamelist=gen_blamelist_string)) return cmd def AddFactoryProperties(self, factory_properties, cmd=None): """Adds factory properties to cmd.""" # pylint: disable=R0201 cmd = cmd or [] cmd.append('--factory-properties=' + json.dumps( factory_properties or {}, sort_keys=True, separators=(',', ':'))) return cmd def AddTestStep(self, command_class, test_name, test_command, test_description='', timeout=10 * 60, max_time=8 * 60 * 60, workdir=None, env=None, locks=None, halt_on_failure=False, do_step_if=True): """Adds a step to the factory to run a test. Args: command_class: the command type to run, such as shell.ShellCommand or gtest_command.GTestCommand test_name: a string describing the test, used to build its logfile name and its descriptions in the waterfall display timeout: the buildbot timeout for the test, in seconds. If it doesn't produce any output to stdout or stderr for this many seconds, buildbot will cancel it and call it a failure. max_time: the maxiumum time the command can run, in seconds. If the command doesn't return in this many seconds, buildbot will cancel it and call it a failure. test_command: the command list to run test_description: an auxiliary description to be appended to the test_name in the buildbot display; for example, ' (single process)' workdir: directory where the test executable will be launched. If None, step will use default directory. env: dictionary with environmental variable key value pairs that will be set or overridden before launching the test executable. Does not do anything if 'env' is None. locks: any locks to acquire for this test halt_on_failure: whether the current build should halt if this step fails """ assert timeout <= max_time do_step_if = do_step_if or self.TestStepFilter self._factory.addStep( command_class, name=test_name, timeout=timeout, maxTime=max_time, doStepIf=do_step_if, workdir=workdir, env=env, # TODO(bradnelson): FIXME #locks=locks, description='running %s%s' % (test_name, test_description), descriptionDone='%s%s' % (test_name, test_description), haltOnFailure=halt_on_failure, command=test_command) def TestStepFilter(self, bStep): return self.TestStepFilterImpl(bStep, True) def TestStepFilterGTestFilterRequired(self, bStep): return self.TestStepFilterImpl(bStep, False) def TestStepFilterImpl(self, bStep, default): """Examines the 'testfilter' property of the build and determines if the step should run; True for yes.""" # TODO(maruel): This is bad hygiene to modify the build properties on the # fly like this. There should be another way to communicate the command line # properly. bStep.setProperty('gtest_filter', None, "Factory") if 'testfilter' not in bStep.build.getProperties(): # Not a try job. return default filters = bStep.build.getProperties().getProperty('testfilter') # DEFAULT_TESTS is a marker to specify all tests should be run normally. # Add it if no filter is explicitly specified. filters = filters or [DEFAULT_TESTS] name = bStep.name # TODO(maruel): Fix the step name. if name.startswith('memory test: '): name = name[len('memory test: '):] filters = dict( i.split(':', 1) if ':' in i else (i, '') for i in filters) # Continue if: # - the test is specified in filters # - DEFAULT_TESTS is listed and default is True if not (name in filters or (DEFAULT_TESTS in filters and default)): return False # This is gtest specific, but other test types can safely ignore it. # Defaults to excluding FAILS and FLAKY test if none is specified. gtest_filter = filters.get(name, '') or self.DEFAULT_GTEST_FILTER if gtest_filter: flag = "--gtest_filter=%s" % gtest_filter bStep.setProperty('gtest_filter', flag, "Scheduler") return True def GetTestStepFilter(self, factory_properties): """Returns a TestStepFilter lambda with the right default according to non_default factory property. """ return lambda bStep: self.TestStepFilterImpl( bStep, bStep.name not in factory_properties.get('non_default', [])) def AddBasicGTestTestStep(self, test_name, factory_properties=None, description='', arg_list=None, total_shards=None, shard_index=None, test_tool_arg_list=None): """Adds a step to the factory to run the gtest tests. Args: test_name: If prefixed with DISABLED_ the prefix is removed, and the step is flagged for not running, but is still added. total_shards: Number of shards to split this test into. shard_index: Shard to run. Must be between 1 and total_shards. generate_gtest_json: generate JSON results file after running the tests. """ factory_properties = factory_properties or {} generate_json = factory_properties.get('generate_gtest_json') if not arg_list: arg_list = [] arg_list = arg_list[:] doStep = self.GetTestStepFilter(factory_properties) if test_name.startswith('DISABLED_'): test_name = test_name[len('DISABLED_'):] doStep = False cmd = [ self._python, self._test_tool, '--target', self._target, '--build-dir', self._build_dir ] cmd = self.AddBuildProperties(cmd) cmd = self.AddFactoryProperties(factory_properties, cmd) # Must add test tool arg list before test arg list. if test_tool_arg_list: cmd.extend(test_tool_arg_list) if generate_json: # test_result_dir (-o) specifies where we put the JSON output locally # on slaves. test_result_dir = 'gtest-results/%s' % test_name cmd.extend([ '--generate-json-file', '-o', test_result_dir, '--test-type', test_name, '--build-number', WithProperties("%(buildnumber)s"), '--builder-name', WithProperties("%(buildername)s"), ]) if total_shards and shard_index: cmd.extend([ '--total-shards', str(total_shards), '--shard-index', str(shard_index) ]) if test_name in factory_properties.get('sharded_tests', []): cmd.append('--parallel') sharding_args = factory_properties.get('sharding_args') if sharding_args: cmd.extend(['--sharding-args', sharding_args]) cmd.append(self.GetExecutableName(test_name)) arg_list.append('--gtest_print_time') arg_list.append(WithProperties("%(gtest_filter)s")) cmd.extend(arg_list) self.AddTestStep(gtest_command.GTestCommand, test_name, ListProperties(cmd), description, do_step_if=doStep) def AddBasicShellStep(self, test_name, timeout=600, arg_list=None): """Adds a step to the factory to run a simple shell test with standard defaults. """ self.AddTestStep(shell.ShellCommand, test_name, timeout=timeout, test_command=self.GetTestCommand(test_name, arg_list=arg_list)) # GClient related commands. def AddSvnKillStep(self): """Adds a step to the factory to kill svn.exe. Windows-only.""" self._factory.addStep( shell.ShellCommand, name='svnkill', description='svnkill', timeout=60, workdir='', # The build subdir may not exist yet. command=[ r'%WINDIR%\system32\taskkill', '/f', '/im', 'svn.exe', '||', 'set', 'ERRORLEVEL=0' ]) def AddTempCleanupStep(self): """Runs script to cleanup acculumated cruft, including tmp directory.""" # Use ReturnCodeCommand so we can indicate a "warning" status (orange). self._factory.addStep( retcode_command.ReturnCodeCommand, name='cleanup_temp', description='cleanup_temp', timeout=60, workdir='', # Doesn't really matter where we are. command=['python', self._cleanup_temp_tool]) def AddUpdateScriptStep(self, gclient_jobs=None): """Adds a step to the factory to update the script folder.""" # This will be run in the '..' directory to udpate the slave's own script # checkout. command = [ chromium_utils.GetGClientCommand(self._target_platform), 'sync', '--verbose' ] if gclient_jobs: command.append('-j%d' % gclient_jobs) self._factory.addStep(shell.ShellCommand, name='update_scripts', description='update_scripts', locks=[self.slave_exclusive_lock], timeout=60 * 5, workdir='..', command=command) def AddUpdateStep(self, gclient_spec, env=None, timeout=None, sudo_for_remove=False, gclient_deps=None, gclient_nohooks=False, no_gclient_branch=False, gclient_transitive=False, primary_repo=None, gclient_jobs=None): """Adds a step to the factory to update the workspace.""" env = env or {} env['DEPOT_TOOLS_UPDATE'] = '0' if timeout is None: # svn timeout is 2 min; we allow 5 timeout = 60 * 5 self._factory.addStep( chromium_step.GClient, gclient_spec=gclient_spec, gclient_deps=gclient_deps, # TODO(maruel): Kept for compatibility but will be removed. gclient_nohooks=gclient_nohooks, workdir=self.working_dir, mode='update', env=env, locks=[self.slave_exclusive_lock], retry=(60 * 5, 4), # Try 4+1=5 more times, 5 min apart timeout=timeout, gclient_jobs=gclient_jobs, sudo_for_remove=sudo_for_remove, rm_timeout=60 * 15, # The step can take a long time. no_gclient_branch=no_gclient_branch, gclient_transitive=gclient_transitive, primary_repo=primary_repo) def AddApplyIssueStep(self): """Adds a step to the factory to apply an issues from Rietveld. It is a conditional step that is only run on the try server if the following conditions are all true: - There are both build properties issue and patchset - There is no patch attached """ def do_step_if(bStep): build = bStep.build properties = build.getProperties() for prop in ('issue', 'patchset'): if prop not in properties or not properties.getProperty(prop): return False if build.getSourceStamp().patch: return False return True cmd = [ 'apply_issue.py', '-r', WithProperties('%(root:-)s'), '-i', WithProperties('%(issue:-)s'), '-p', WithProperties('%(patchset:-)s'), ] self._factory.addStep(shell.ShellCommand, haltOnFailure=True, name='apply_issue', description='apply patch', command=cmd, doStepIf=do_step_if) def AddRunHooksStep(self, env=None, timeout=None): """Adds a step to the factory to run the gclient hooks.""" env = env or {} env['DEPOT_TOOLS_UPDATE'] = '0' if timeout is None: # svn timeout is 2 min; we allow 5 timeout = 60 * 5 self._factory.addStep(shell.ShellCommand, haltOnFailure=True, name='runhooks', description='gclient hooks', env=env, locks=[self.slave_exclusive_lock], timeout=timeout, command=['gclient', 'runhooks']) def AddClobberTreeStep(self, gclient_spec, env=None, timeout=None, gclient_deps=None, gclient_nohooks=False, no_gclient_branch=None): """ This is not for pressing 'clobber' on the waterfall UI page. This is for clobbering all the sources. Using mode='clobber' causes the entire working directory to get moved aside (to build.dead) --OR-- if build.dead already exists, it deletes build.dead. Strange, but true. See GClient.doClobber() (for move vs. delete logic) or Gclient.start() (for mode='clobber' trigger) in chromium_commands.py. In theory, this means we can have a ClobberTree step at the beginning of a build to quickly move the existing workdir and do a full clean checkout. Then, if we add the same step at the end of a build, it will delete the moved-out-of-the-way directory. Presuming neither step fails or times out, this allows a builder to pull a full, clean tree for every build. This is exactly what we want for official release builds, so that the builder can refresh its entire tree based on a new buildspec (which might point to a completely different branch or an older revision than the last build on the machine). """ if env is None: env = {} env['DEPOT_TOOLS_UPDATE'] = '0' if timeout is None: # svn timeout is 2 min; we allow 5 timeout = 60 * 5 self._factory.addStep(chromium_step.GClient, gclient_spec=gclient_spec, gclient_deps=gclient_deps, gclient_nohooks=gclient_nohooks, no_gclient_branch=no_gclient_branch, workdir=self.working_dir, mode='clobber', env=env, timeout=timeout, rm_timeout=60 * 60) # We don't care how long it takes. def AddTaskkillStep(self): """Adds a step to kill the running processes before a build.""" # Use ReturnCodeCommand so we can indicate a "warning" status (orange). self._factory.addStep( retcode_command.ReturnCodeCommand, name='taskkill', description='taskkill', timeout=60, workdir='', # Doesn't really matter where we are. command=['python', self._kill_tool]) # Zip / Extract commands. def AddZipBuild(self, src_dir=None, include_files=None, halt_on_failure=False, factory_properties=None): factory_properties = factory_properties or {} cmd = [ self._python, self._zip_tool, '--target', self._target, '--build-dir', self._build_dir ] if 'webkit_dir' in factory_properties: cmd += ['--webkit-dir', factory_properties['webkit_dir']] if src_dir is not None: cmd += ['--src-dir', src_dir] cmd = self.AddBuildProperties(cmd) cmd = self.AddFactoryProperties(factory_properties, cmd) if include_files is not None: # Convert the include_files array into a quoted, comma-delimited list # for passing as a command-line argument. include_arg = '"' + ', '.join(include_files) + '"' cmd += ['--include-files', include_arg] self._factory.addStep(shell.ShellCommand, name='package_build', timeout=600, description='packaging build', descriptionDone='packaged build', haltOnFailure=halt_on_failure, command=cmd) def AddExtractBuild(self, build_url, factory_properties=None): """Extract a build. Assumes the zip file has a directory like src/xcodebuild which contains the actual build. """ factory_properties = factory_properties or {} cmd = [ self._python, self._extract_tool, '--build-dir', self._build_dir, '--target', self._target, '--build-url', build_url ] if 'webkit_dir' in factory_properties: cmd += ['--webkit-dir', factory_properties['webkit_dir']] cmd = self.AddBuildProperties(cmd) cmd = self.AddFactoryProperties(factory_properties, cmd) self.AddTestStep(retcode_command.ReturnCodeCommand, 'extract_build', cmd, halt_on_failure=True) def AddGetBuildForChromebot(self, platform, archive=False, extract=False, build_url=None, archive_url=None, build_id=None, build_dir=None): """Get a Chrome build for Chromebot. If |build_id| is omitted, latest build will be downloaded instead. Args: platform: The platform for Chrome build (win, linux, linux64). archive: If the |build_url| contains a list of builds. extract: Whether to extract the downloaded files. build_url: URL to the build. Default URL if None. build_id: Id of build. """ if not build_dir: build_dir = self._build_dir cmd = [ self._python, self._get_build_for_chromebot_tool, '--platform', platform, '--build-dir', build_dir, '--target-dir', self._target ] if extract: cmd += ['--extract'] if build_url: cmd += ['--build-url', build_url] if archive_url: cmd += ['--archive-url', archive_url] self.AddTestStep(SetBuildPropertyShellCommand, 'get_build', cmd, halt_on_failure=True) # Build commands. def GetBuildCommand(self, clobber, solution, mode, options=None): """Returns a command list to call the _compile_tool in the given build_dir, optionally clobbering the build (that is, deleting the build directory) first. if solution contains a ";", the second part is interpreted as the project. """ cmd = [self._python, self._compile_tool] if solution: split_solution = solution.split(';', 1) cmd.extend(['--solution', split_solution[0]]) if len(split_solution) == 2: cmd.extend(['--project', split_solution[1]]) cmd.extend(['--target', self._target, '--build-dir', self._build_dir]) if self._target_arch: cmd.extend(['--arch', self._target_arch]) if mode: cmd.extend(['--mode', mode]) if clobber: cmd.append('--clobber') else: # Below, WithProperties is appended to the cmd and rendered into a string # for each specific build at build-time. When clobber is None, it renders # to an empty string. When clobber is not None, it renders to the string # --clobber. Note: the :+ after clobber controls this behavior and is not # a typo. cmd.append(WithProperties('%s', 'clobber:+--clobber')) if options: cmd.extend(options) # Using ListProperties will process and discard None and '' values, # otherwise posix platforms will fail. return ListProperties(cmd) def AddCompileStep(self, solution, clobber=False, description='compiling', descriptionDone='compile', timeout=600, mode=None, options=None, haltOnFailure=True): """Adds a step to the factory to compile the solution. Args: solution: the solution/sub-project file to build clobber: if True, clobber the build (that is, delete the build directory) before building description: for the waterfall descriptionDone: for the waterfall timeout: if no output is received in this many seconds, the compile step will be killed mode: if given, this will be passed as the --mode option to the compile command options: list of additional options to pass to the compile command halfOnFailure: should stop the build if compile fails """ self._factory.addStep(cl_command.CLCommand, name='compile', enable_warnings=0, timeout=timeout, description=description, descriptionDone=descriptionDone, command=self.GetBuildCommand( clobber, solution, mode, options), haltOnFailure=haltOnFailure) def GetPerfStepClass(self, factory_properties, test_name, log_processor_class, command_class=None, **kwargs): """Selects the right build step for the specified perf test.""" factory_properties = factory_properties or {} perf_id = factory_properties.get('perf_id') show_results = factory_properties.get('show_perf_results') report_link = None output_dir = None perf_name = None if show_results: perf_name = perf_id if (self._target in self.PERF_TEST_MAPPINGS and perf_id in self.PERF_TEST_MAPPINGS[self._target]): perf_name = self.PERF_TEST_MAPPINGS[self._target][perf_id] report_link = '%s/%s/%s/%s' % (self.PERF_BASE_URL, perf_name, test_name, self.PERF_REPORT_URL_SUFFIX) output_dir = '%s/%s/%s' % (self.PERF_OUTPUT_DIR, perf_name, test_name) return CreatePerformanceStepClass( log_processor_class, report_link=report_link, output_dir=output_dir, factory_properties=factory_properties, perf_name=perf_name, test_name=test_name, command_class=command_class) # Checks out and builds clang def AddUpdateClangStep(self): cmd = [self._update_clang_tool] self._factory.addStep( shell.ShellCommand, name='update_clang', timeout=600, description='Updating and building clang and plugins', descriptionDone='clang updated', env={'LLVM_URL': config.Master.llvm_url}, command=cmd) def AddDownloadFileStep(self, mastersrc, slavedest): """Download a file from master.""" self._factory.addStep( FileDownload(mastersrc=mastersrc, slavedest=slavedest))
def __init__(self, Environ): # Max number of running builds build_lock = SlaveLock('build', maxCount=2, maxCountForSlave={ 'farmer-slave1': 2, }) # All repo all_repos = { 'quantum_espresso': { 'repository': 'https://gitlab.com/QEF/q-e.git', 'branch': 'develop', }, # 'sternheimer_gw': { # 'repository': 'https://github.com/mmdg-oxford/SternheimerGW.git', # 'branch': 'develop', # }, 'wannier90': { 'repository': 'https://github.com/wannier-developers/wannier90.git', # 'repository': 'https://github.com/sponce24/wannier90.git', 'branch': 'develop', }, } ############################################################################ # QE code ############################################################################ self.checkout_qe = [ steps.Git( name="checkout_qe", method="copy", repourl=all_repos["quantum_espresso"]["repository"], branch=all_repos["quantum_espresso"]["branch"], haltOnFailure=True, alwaysUseLatest=True, ) ] self.configure_qe = [ ShellCommand(name="configure_qe", command=["./configure"], env=Environ, workdir="build", locks=[build_lock.access('counting')], haltOnFailure=True, descriptionDone=["configure_qe"]) ] self.configure_qe_mp = [ ShellCommand(name="configure_qe", command=[ "./configure", "--enable-openmp", "--enable-parallel" ], env=Environ, workdir="build", locks=[build_lock.access('counting')], haltOnFailure=True, descriptionDone=["configure_qe_mp"]) ] self.dep_qe = [ ShellCommand( name="dep_qe", #command=["make","depend"], # DBSP: Temporary until QE 6.2 is release command=["ls"], env=Environ, workdir="build", locks=[build_lock.access('counting')], haltOnFailure=True, descriptionDone=["dep_qe"]) ] self.env_qe1 = [ ShellCommand( name="env_qe1", command=Interpolate( 'sed -i "s/TESTCODE_NPROCS=4/TESTCODE_NPROCS=2/g" ENVIRONMENT' ), env=Environ, workdir="build/test-suite/", locks=[build_lock.access('counting')], haltOnFailure=True, descriptionDone=["env_qe1"]) ] self.env_qe2 = [ ShellCommand( name="env_qe2", command=Interpolate( 'echo "export OMP_NUM_THREADS=2" >> ENVIRONMENT'), #command=["cat","'export OMP_NUM_THREADS=2'",">>", "ENVIRONMENT"], env=Environ, workdir="build/test-suite/", locks=[build_lock.access('counting')], haltOnFailure=True, descriptionDone=["env_qe2"]) ] self.make_pw = [ ShellCommand( name="make_pw", command=["make", "-j", "4", "pwall", "cp", "ld1", "upf"], env=Environ, workdir="build", haltOnFailure=True, descriptionDone=["make_pw"], locks=[build_lock.access('counting')]) ] self.make_ph = [ ShellCommand(name="make_ph", command=["make", "ph"], env=Environ, workdir="build", haltOnFailure=True, descriptionDone=["make_ph"], locks=[build_lock.access('counting')]) ] self.make_epw0 = [ ShellCommand(name="make_epw0", command=["make"], env=Environ, workdir="build/EPW/src/", haltOnFailure=True, descriptionDone=["make_epw"], locks=[build_lock.access('counting')]) ] self.make_epw = [ ShellCommand(name="make_epw", command=["make", "epw"], env=Environ, workdir="build", haltOnFailure=True, descriptionDone=["make_epw"], locks=[build_lock.access('counting')]) ] self.make_lr = [ ShellCommand( name="make_lr", command=["make", "-j", "8", "lrmods"], env=Environ, workdir="build", haltOnFailure=True, descriptionDone=["make_lr"], locks=[build_lock.access('counting')], ) ] self.test_clean = [ ShellCommand( name="test_clean", command=["make", "clean"], env=Environ, workdir="build/test-suite", descriptionDone=["test_clean"], locks=[build_lock.access('counting')], ) ] self.clean = [ ShellCommand(command=["make", "veryclean"], alwaysRun=True, flunkOnFailure=False, workdir="build") ] self.test0 = [ ShellCommand(name="test_prolog", command=["make", "prolog"], env=Environ, workdir="build/test-suite", haltOnFailure=False, descriptionDone=["make prolog"], locks=[build_lock.access('counting')]) ] self.test_para_PW = [ ShellCommand(name="PW_para", command=["make", "run-tests-pw-parallel"], env=Environ, workdir="build/test-suite", haltOnFailure=False, descriptionDone=["PW para tests"], locks=[build_lock.access('counting')]) ] self.test_serial_PW = [ ShellCommand(name="PW_serial", command=["make", "run-tests-pw-serial"], env=Environ, workdir="build/test-suite", haltOnFailure=False, descriptionDone=["PW serial tests"], locks=[build_lock.access('counting')]) ] self.test_para_CP = [ ShellCommand(name="CP_para", command=["make", "run-tests-cp-parallel"], env=Environ, workdir="build/test-suite", haltOnFailure=False, descriptionDone=["CP para tests"], locks=[build_lock.access('counting')]) ] self.test_serial_CP = [ ShellCommand(name="CP_serial", command=["make", "run-tests-cp-serial"], env=Environ, workdir="build/test-suite", haltOnFailure=False, descriptionDone=["CP serial tests"], locks=[build_lock.access('counting')]) ] self.test_para_PH = [ ShellCommand(name="PH_para", command=["make", "run-tests-ph-parallel"], env=Environ, workdir="build/test-suite", haltOnFailure=False, descriptionDone=["PH para tests"], locks=[build_lock.access('counting')]) ] self.test_serial_PH = [ ShellCommand(name="PH_serial", command=["make", "run-tests-ph-serial"], env=Environ, workdir="build/test-suite", haltOnFailure=False, descriptionDone=["PH serial tests"], locks=[build_lock.access('counting')]) ] self.test_para_EPW = [ ShellCommand(name="EPW_para", command=["make", "run-tests-epw-parallel"], env=Environ, workdir="build/test-suite", haltOnFailure=False, descriptionDone=["EPW para tests"], locks=[build_lock.access('counting')]) ] self.test_serial_EPW = [ ShellCommand(name="EPW_serial", command=["make", "run-tests-epw-serial"], env=Environ, workdir="build/test-suite", haltOnFailure=False, descriptionDone=["EPW serial tests"], locks=[build_lock.access('counting')]) ] ############################################################################ # SGW code ############################################################################ # self.configure_qe2 = [ShellCommand( # name="configure_qe", # command=["./configure"], # env=Environ, # workdir="build", # locks=[build_lock.access('counting')], # haltOnFailure = True,descriptionDone=["configure_qe"] # )] # # self.make_pw2 = [ShellCommand( # name="make_pw", # command=["make","pw","lrmods"], # env=Environ, # workdir="build", # haltOnFailure=True, descriptionDone=["make_pw"], # locks=[build_lock.access('counting')] # )] # # self.checkout_sgw = [steps.Git( # name="checkout_sgw", # repourl=all_repos["sternheimer_gw"]["repository"], # branch=all_repos["sternheimer_gw"]["branch"], # workdir="build/SGW", # haltOnFailure = True, # alwaysUseLatest = True, # )] # # self.make_clean = [ShellCommand( # name="make_clean", # command=["make", "clean"], # env=Environ, # workdir="build/SGW", # haltOnFailure = True, # descriptionDone = ["make_clean"], # locks=[build_lock.access('counting')], # )] # # self.make_sgw = [ShellCommand( # name="make_sgw", # command=["make"], # env=Environ, # workdir="build/SGW", # haltOnFailure = True, # descriptionDone = ["make_sgw"], # locks=[build_lock.access('counting')], # )] # # self.test_sgw = [ShellCommand( # name="test_sgw", # command=["make", "run-tests"], # env=Environ, # workdir="build/SGW/test-suite", # haltOnFailure = True, # descriptionDone = ["test_sgw"], # locks=[build_lock.access('counting')], # )] # # self.test_clean_sgw = [ShellCommand( # name="test_clean", # command=["make", "clean"], # env=Environ, # workdir="build/SGW/test-suite", # descriptionDone = ["test_clean"], # locks=[build_lock.access('counting')], # )] ############################################################################ # Wannier code ############################################################################ self.checkout_wannier = [ steps.Git( name="checkout_wannier", method="copy", workdir="build/WAN", repourl=all_repos["wannier90"]["repository"], branch=all_repos["wannier90"]["branch"], haltOnFailure=True, alwaysUseLatest=True, ) ] self.cpconfig = [ ShellCommand( name="cp_config", command=[ "cp", "test-suite/config/TestFarm/farmer_gcc640_serial.inc", "make.inc" ], env=Environ, workdir="build/WAN", haltOnFailure=True, descriptionDone=["cp_config"], locks=[build_lock.access('counting')]) ] self.cpgcc730 = [ ShellCommand( name="cp_config", command=[ "cp", "test-suite/config/TestFarm/farmer_gcc730_openmpi1107.inc", "make.inc" ], env=Environ, workdir="build/WAN", haltOnFailure=True, descriptionDone=["cp_config"], locks=[build_lock.access('counting')]) ] self.cpintel17 = [ ShellCommand( name="cp_config", command=[ "cp", "test-suite/config/TestFarm/farmer_intel17_openmpi313.inc", "make.inc" ], env=Environ, workdir="build/WAN", haltOnFailure=True, descriptionDone=["cp_config"], locks=[build_lock.access('counting')]) ] self.cpintel17i = [ ShellCommand( name="cp_config", command=[ "cp", "test-suite/config/TestFarm/farmer_intel17_impi.inc", "make.inc" ], env=Environ, workdir="build/WAN", haltOnFailure=True, descriptionDone=["cp_config"], locks=[build_lock.access('counting')]) ] self.cpintel18 = [ ShellCommand( name="cp_config", command=[ "cp", "test-suite/config/TestFarm/farmer_intel18_openmpi313.inc", "make.inc" ], env=Environ, workdir="build/WAN", haltOnFailure=True, descriptionDone=["cp_config"], locks=[build_lock.access('counting')]) ] self.cppgi18 = [ ShellCommand( name="cp_config", command=[ "cp", "test-suite/config/TestFarm/farmer_pgi18_mvapich23b.inc", "make.inc" ], env=Environ, workdir="build/WAN", haltOnFailure=True, descriptionDone=["cp_config"], locks=[build_lock.access('counting')]) ] self.clean_wannier = [ ShellCommand( name="clean_wannier", command=["make", "clean"], env=Environ, workdir="build/WAN", haltOnFailure=True, descriptionDone=["clean_wannier"], locks=[build_lock.access('counting')], ) ] self.clean_tests = [ ShellCommand( name="clean_tests", command=["python", "clean_tests"], env=Environ, workdir="build/WAN/test-suite", haltOnFailure=True, descriptionDone=["clean_tests"], locks=[build_lock.access('counting')], ) ] self.make_wannier = [ ShellCommand( name="make_wannier", command=["make"], env=Environ, workdir="build/WAN", haltOnFailure=True, descriptionDone=["make_wannier"], locks=[build_lock.access('counting')], ) ] self.make_wannier2 = [ ShellCommand( name="make_wannier2", command=["make", "default", "w90chk2chk"], env=Environ, workdir="build/WAN", haltOnFailure=True, descriptionDone=["make_wannier2"], locks=[build_lock.access('counting')], ) ] self.test_wannier_serial = [ ShellCommand( name="test_wannier_seq", command=["./run_tests", "--category=default"], env=Environ, workdir="build/WAN/test-suite", haltOnFailure=True, descriptionDone=["test_wannier_seq"], locks=[build_lock.access('counting')], ) ] self.test_wannier_para = [ ShellCommand( name="test_wannier_para", command=["./run_tests", "--category=default", "--numprocs=4"], env=Environ, workdir="build/WAN/test-suite", haltOnFailure=True, descriptionDone=["test_wannier_para"], locks=[build_lock.access('counting')], ) ]