def checkConfig(self, name, workdir=None, usePty=False, **kwargs):
Worker.checkConfig(self, name, None, **kwargs)
self.LocalWorkerFactory = None
try:
# importing here to avoid dependency on buildbot worker package
from buildslave.bot import LocalBuildSlave as RemoteLocalWorker
self.LocalWorkerFactory = RemoteLocalWorker
except ImportError:
error("LocalWorker needs the buildbot-slave package installed (pip install buildbot-slave)")
self.remote_worker = None
def checkConfig(self, name, workdir=None, usePty=False, **kwargs):
Worker.checkConfig(self, name, None, **kwargs)
self.LocalWorkerFactory = None
try:
# importing here to avoid dependency on buildbot worker package
from buildbot_worker.bot import LocalWorker as RemoteLocalWorker
self.LocalWorkerFactory = RemoteLocalWorker
except ImportError:
error("LocalWorker needs the buildbot-worker package installed "
"(pip install buildbot-worker)")
self.remote_worker = None
def setUp(self):
self.setUpTestReactor()
self.master = fakemaster.make_master(self,
wantData=True,
wantMq=True,
wantDb=True)
self.master.db.insertTestData([
fakedb.Builder(id=80, name='test'),
])
self.builder = builder.Builder('test')
self.builder._builderid = 80
self.builder.config_version = 0
self.builder.master = self.master
self.builder.botmaster = mock.Mock()
self.builder.botmaster.getLockFromLockAccesses = lambda l, c: []
yield self.builder.startService()
self.factory = factory.BuildFactory() # will have steps added later
new_config = config.MasterConfig()
new_config.builders.append(
config.BuilderConfig(name='test',
workername='testworker',
factory=self.factory))
yield self.builder.reconfigServiceWithBuildbotConfig(new_config)
self.worker = Worker('worker', 'pass')
self.worker.sendBuilderList = lambda: defer.succeed(None)
self.worker.parent = mock.Mock()
self.worker.master.botmaster = mock.Mock()
self.worker.botmaster.maybeStartBuildsForWorker = lambda w: None
self.worker.botmaster.getBuildersForWorker = lambda w: []
self.worker.parent = self.master
self.worker.startService()
self.conn = fakeprotocol.FakeConnection(self.master, self.worker)
yield self.worker.attached(self.conn)
wfb = self.workerforbuilder = workerforbuilder.WorkerForBuilder()
wfb.setBuilder(self.builder)
yield wfb.attached(self.worker, {})
# add the buildset/request
self.bsid, brids = yield self.master.db.buildsets.addBuildset(
sourcestamps=[{}],
reason='x',
properties={},
builderids=[80],
waited_for=False)
self.brdict = \
yield self.master.db.buildrequests.getBuildRequest(brids[80])
self.buildrequest = \
yield buildrequest.BuildRequest.fromBrdict(self.master, self.brdict)
def checkConfig(self, name, workdir=None, **kwargs):
kwargs['password'] = None
Worker.checkConfig(self, name, **kwargs)
self.LocalWorkerFactory = None
try:
# importing here to avoid dependency on buildbot worker package
from buildbot_worker.bot import LocalWorker as RemoteLocalWorker
self.LocalWorkerFactory = RemoteLocalWorker
except ImportError:
error("LocalWorker needs the buildbot-worker package installed "
"(pip install buildbot-worker)")
self.remote_worker = None
def reconfigService(self, name, workdir=None, usePty=False, **kwargs):
Worker.reconfigService(self, name, None, **kwargs)
if workdir is None:
workdir = name
workdir = os.path.abspath(os.path.join(self.master.basedir, "workers", workdir))
if not os.path.isdir(workdir):
os.makedirs(workdir)
if self.remote_worker is None:
# create the actual worker as a child service
# we only create at reconfig, to avoid poluting memory in case of reconfig
self.remote_worker = self.LocalWorkerFactory(name, workdir, usePty)
yield self.remote_worker.setServiceParent(self)
else:
# The case of a reconfig, we forward the parameters
self.remote_worker.bot.basedir = workdir
self.remote_worker.usePty = usePty
def reconfigService(self, name, workdir=None, usePty=False, **kwargs):
Worker.reconfigService(self, name, None, **kwargs)
if workdir is None:
workdir = name
workdir = os.path.abspath(
os.path.join(self.master.basedir, "workers", workdir))
if not os.path.isdir(workdir):
os.makedirs(workdir)
if self.remote_worker is None:
# create the actual worker as a child service
# we only create at reconfig, to avoid poluting memory in case of reconfig
self.remote_worker = self.LocalWorkerFactory(name, workdir, usePty)
yield self.remote_worker.setServiceParent(self)
else:
# The case of a reconfig, we forward the parameters
self.remote_worker.bot.basedir = workdir
self.remote_worker.usePty = usePty
def getBaseConfig(self):
return {
'builders': [
BuilderConfig(name="testy",
workernames=["local1", "local2"],
factory=BuildFactory([steps.ShellCommand(command='echo hello')])),
],
'workers': [Worker('local' + str(i), 'pass') for i in range(3)],
'schedulers': [
ForceScheduler(
name="force",
builderNames=["testy"])
],
'protocols': {'null': {}},
'multiMaster': True,
}
def setUp(self):
self.setUpTestReactor()
self.master = fakemaster.make_master(self, wantData=True,
wantMq=True, wantDb=True)
self.master.db.insertTestData([
fakedb.Builder(id=80, name='test'), ])
self.builder = builder.Builder('test')
self.builder._builderid = 80
self.builder.master = self.master
yield self.builder.startService()
self.factory = factory.BuildFactory() # will have steps added later
new_config = config.MasterConfig()
new_config.builders.append(
config.BuilderConfig(name='test', workername='testworker',
factory=self.factory))
yield self.builder.reconfigServiceWithBuildbotConfig(new_config)
self.worker = Worker('worker', 'pass')
self.worker.sendBuilderList = lambda: defer.succeed(None)
self.worker.parent = mock.Mock()
self.worker.master.botmaster = mock.Mock()
self.worker.botmaster.maybeStartBuildsForWorker = lambda w: None
self.worker.botmaster.getBuildersForWorker = lambda w: []
self.worker.parent = self.master
self.worker.startService()
self.conn = fakeprotocol.FakeConnection(self.master, self.worker)
yield self.worker.attached(self.conn)
wfb = self.workerforbuilder = workerforbuilder.WorkerForBuilder()
wfb.setBuilder(self.builder)
yield wfb.attached(self.worker, {})
# add the buildset/request
self.bsid, brids = yield self.master.db.buildsets.addBuildset(
sourcestamps=[{}], reason='x', properties={},
builderids=[80], waited_for=False)
self.brdict = \
yield self.master.db.buildrequests.getBuildRequest(brids[80])
self.buildrequest = \
yield buildrequest.BuildRequest.fromBrdict(self.master, self.brdict)
class RunSteps(unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
self.master = fakemaster.make_master(testcase=self,
wantData=True, wantMq=True, wantDb=True)
self.master.db.insertTestData([
fakedb.Builder(id=80, name='test'), ])
self.builder = builder.Builder('test')
self.builder._builderid = 80
self.builder.master = self.master
yield self.builder.startService()
self.factory = factory.BuildFactory() # will have steps added later
new_config = config.MasterConfig()
new_config.builders.append(
config.BuilderConfig(name='test', workername='testworker',
factory=self.factory))
yield self.builder.reconfigServiceWithBuildbotConfig(new_config)
self.worker = Worker('worker', 'pass')
self.worker.sendBuilderList = lambda: defer.succeed(None)
self.worker.parent = mock.Mock()
self.worker.master.botmaster = mock.Mock()
self.worker.botmaster.maybeStartBuildsForWorker = lambda w: None
self.worker.botmaster.getBuildersForWorker = lambda w: []
self.worker.parent = self.master
self.worker.startService()
self.conn = fakeprotocol.FakeConnection(self.master, self.worker)
yield self.worker.attached(self.conn)
wfb = self.workerforbuilder = workerforbuilder.WorkerForBuilder()
wfb.setBuilder(self.builder)
yield wfb.attached(self.worker, {})
# add the buildset/request
self.bsid, brids = yield self.master.db.buildsets.addBuildset(
sourcestamps=[{}], reason=u'x', properties={},
builderids=[80], waited_for=False)
self.brdict = \
yield self.master.db.buildrequests.getBuildRequest(brids[80])
self.buildrequest = \
yield buildrequest.BuildRequest.fromBrdict(self.master, self.brdict)
def tearDown(self):
return self.builder.stopService()
@defer.inlineCallbacks
def do_test_step(self):
# patch builder.buildFinished to signal us with a deferred
bfd = defer.Deferred()
old_buildFinished = self.builder.buildFinished
def buildFinished(*args):
old_buildFinished(*args)
bfd.callback(None)
self.builder.buildFinished = buildFinished
# start the builder
self.assertTrue((yield self.builder.maybeStartBuild(
self.workerforbuilder, [self.buildrequest])))
# and wait for completion
yield bfd
# then get the BuildStatus and return it
defer.returnValue(self.master.status.lastBuilderStatus.lastBuildStatus)
def assertLogs(self, exp_logs):
got_logs = {}
for id, l in iteritems(self.master.data.updates.logs):
self.assertTrue(l['finished'])
got_logs[l['name']] = ''.join(l['content'])
self.assertEqual(got_logs, exp_logs)
@defer.inlineCallbacks
def doOldStyleCustomBuildStep(self, slowDB=False):
# patch out addLog to delay until we're ready
newLogDeferreds = []
oldNewLog = self.master.data.updates.addLog
def finishNewLog(self):
for d in newLogDeferreds:
reactor.callLater(0, d.callback, None)
def delayedNewLog(*args, **kwargs):
d = defer.Deferred()
d.addCallback(lambda _: oldNewLog(*args, **kwargs))
newLogDeferreds.append(d)
return d
if slowDB:
self.patch(self.master.data.updates,
"addLog", delayedNewLog)
self.patch(OldStyleCustomBuildStep,
"_run_finished_hook", finishNewLog)
self.factory.addStep(OldStyleCustomBuildStep(arg1=1, arg2=2))
yield self.do_test_step()
self.assertLogs({
u'compl.html': u'<blink>A very short logfile</blink>\n',
# this is one of the things that differs independently of
# new/old style: encoding of logs and newlines
u'foo':
# 'stdout\n\xe2\x98\x83\nstderr\n',
u'ostdout\no\N{SNOWMAN}\nestderr\n',
u'obs':
# if slowDB, the observer won't see anything before the end of this
# instant step
u'Observer saw []\n' if slowDB else
# 'Observer saw [\'stdout\\n\', \'\\xe2\\x98\\x83\\n\']',
u'Observer saw [' + repr(u'stdout\n') + u", " + repr(u"\u2603\n") + u"]\n"
})
def test_OldStyleCustomBuildStep(self):
return self.doOldStyleCustomBuildStep(False)
def test_OldStyleCustomBuildStepSlowDB(self):
return self.doOldStyleCustomBuildStep(True)
@defer.inlineCallbacks
def test_OldStyleCustomBuildStep_failure(self):
self.factory.addStep(OldStyleCustomBuildStep(arg1=1, arg2=2, doFail=1))
bs = yield self.do_test_step()
self.assertEqual(len(self.flushLoggedErrors(RuntimeError)), 1)
self.assertEqual(bs.getResults(), results.EXCEPTION)
@defer.inlineCallbacks
def test_step_raising_buildstepfailed_in_start(self):
self.factory.addStep(FailingCustomStep())
bs = yield self.do_test_step()
self.assertEqual(bs.getResults(), results.FAILURE)
@defer.inlineCallbacks
def test_step_raising_exception_in_start(self):
self.factory.addStep(FailingCustomStep(exception=ValueError))
bs = yield self.do_test_step()
self.assertEqual(bs.getResults(), results.EXCEPTION)
self.assertEqual(len(self.flushLoggedErrors(ValueError)), 1)
@defer.inlineCallbacks
def test_step_raising_connectionlost_in_start(self):
self.factory.addStep(FailingCustomStep(exception=error.ConnectionLost))
bs = yield self.do_test_step()
self.assertEqual(bs.getResults(), results.RETRY)
@defer.inlineCallbacks
def test_Latin1ProducingCustomBuildStep(self):
self.factory.addStep(
Latin1ProducingCustomBuildStep(logEncoding='latin-1'))
yield self.do_test_step()
self.assertLogs({
u'xx': u'o\N{CENT SIGN}\n',
})
@defer.inlineCallbacks
def test_OldBuildEPYDoc(self):
# test old-style calls to log.getText, figuring readlines will be ok
self.factory.addStep(OldBuildEPYDoc())
bs = yield self.do_test_step()
self.assertEqual(bs.getResults(), results.FAILURE)
@defer.inlineCallbacks
def test_OldPerlModuleTest(self):
# test old-style calls to self.getLog
self.factory.addStep(OldPerlModuleTest())
bs = yield self.do_test_step()
self.assertEqual(bs.getResults(), results.SUCCESS)
class RunSteps(unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
self.master = fakemaster.make_master(testcase=self,
wantData=True, wantMq=True, wantDb=True)
self.master.db.insertTestData([
fakedb.Builder(id=80, name='test'), ])
self.builder = builder.Builder('test', _addServices=False)
self.builder._builderid = 80
self.builder.master = self.master
yield self.builder.startService()
self.factory = factory.BuildFactory() # will have steps added later
new_config = config.MasterConfig()
new_config.builders.append(
config.BuilderConfig(name='test', workername='testworker',
factory=self.factory))
yield self.builder.reconfigServiceWithBuildbotConfig(new_config)
self.worker = Worker('worker', 'pass')
self.worker.sendBuilderList = lambda: defer.succeed(None)
self.worker.parent = mock.Mock()
self.worker.master.botmaster = mock.Mock()
self.worker.botmaster.maybeStartBuildsForWorker = lambda w: None
self.worker.botmaster.getBuildersForWorker = lambda w: []
self.worker.parent = self.master
self.worker.startService()
self.conn = fakeprotocol.FakeConnection(self.master, self.worker)
yield self.worker.attached(self.conn)
sb = self.workerforbuilder = workerforbuilder.WorkerForBuilder()
sb.setBuilder(self.builder)
yield sb.attached(self.worker, {})
# add the buildset/request
self.bsid, brids = yield self.master.db.buildsets.addBuildset(
sourcestamps=[{}], reason=u'x', properties={},
builderids=[80], waited_for=False)
self.brdict = \
yield self.master.db.buildrequests.getBuildRequest(brids[80])
self.buildrequest = \
yield buildrequest.BuildRequest.fromBrdict(self.master, self.brdict)
def tearDown(self):
return self.builder.stopService()
@defer.inlineCallbacks
def do_test_step(self):
# patch builder.buildFinished to signal us with a deferred
bfd = defer.Deferred()
old_buildFinished = self.builder.buildFinished
def buildFinished(*args):
old_buildFinished(*args)
bfd.callback(None)
self.builder.buildFinished = buildFinished
# start the builder
self.assertTrue((yield self.builder.maybeStartBuild(
self.workerforbuilder, [self.buildrequest])))
# and wait for completion
yield bfd
# then get the BuildStatus and return it
defer.returnValue(self.master.status.lastBuilderStatus.lastBuildStatus)
def assertLogs(self, exp_logs):
got_logs = {}
for id, l in iteritems(self.master.data.updates.logs):
self.assertTrue(l['finished'])
got_logs[l['name']] = ''.join(l['content'])
self.assertEqual(got_logs, exp_logs)
@defer.inlineCallbacks
def doOldStyleCustomBuildStep(self, slowDB=False):
# patch out addLog to delay until we're ready
newLogDeferreds = []
oldNewLog = self.master.data.updates.addLog
def finishNewLog(self):
for d in newLogDeferreds:
reactor.callLater(0, d.callback, None)
def delayedNewLog(*args, **kwargs):
d = defer.Deferred()
d.addCallback(lambda _: oldNewLog(*args, **kwargs))
newLogDeferreds.append(d)
return d
if slowDB:
self.patch(self.master.data.updates,
"addLog", delayedNewLog)
self.patch(OldStyleCustomBuildStep,
"_run_finished_hook", finishNewLog)
self.factory.addStep(OldStyleCustomBuildStep(arg1=1, arg2=2))
yield self.do_test_step()
self.assertLogs({
u'compl.html': u'<blink>A very short logfile</blink>\n',
# this is one of the things that differs independently of
# new/old style: encoding of logs and newlines
u'foo':
# 'stdout\n\xe2\x98\x83\nstderr\n',
u'ostdout\no\N{SNOWMAN}\nestderr\n',
u'obs':
# if slowDB, the observer wont see anything before the end of this
# instant step
u'Observer saw []\n' if slowDB else
# 'Observer saw [\'stdout\\n\', \'\\xe2\\x98\\x83\\n\']',
u'Observer saw [u\'stdout\\n\', u\'\\u2603\\n\']\n',
})
def test_OldStyleCustomBuildStep(self):
return self.doOldStyleCustomBuildStep(False)
def test_OldStyleCustomBuildStepSlowDB(self):
return self.doOldStyleCustomBuildStep(True)
@defer.inlineCallbacks
def test_OldStyleCustomBuildStep_failure(self):
self.factory.addStep(OldStyleCustomBuildStep(arg1=1, arg2=2, doFail=1))
bs = yield self.do_test_step()
self.assertEqual(len(self.flushLoggedErrors(RuntimeError)), 1)
self.assertEqual(bs.getResults(), results.EXCEPTION)
@defer.inlineCallbacks
def test_step_raising_buildstepfailed_in_start(self):
self.factory.addStep(FailingCustomStep())
bs = yield self.do_test_step()
self.assertEqual(bs.getResults(), results.FAILURE)
@defer.inlineCallbacks
def test_step_raising_exception_in_start(self):
self.factory.addStep(FailingCustomStep(exception=ValueError))
bs = yield self.do_test_step()
self.assertEqual(bs.getResults(), results.EXCEPTION)
self.assertEqual(len(self.flushLoggedErrors(ValueError)), 1)
@defer.inlineCallbacks
def test_step_raising_connectionlost_in_start(self):
self.factory.addStep(FailingCustomStep(exception=error.ConnectionLost))
bs = yield self.do_test_step()
self.assertEqual(bs.getResults(), results.RETRY)
@defer.inlineCallbacks
def test_Latin1ProducingCustomBuildStep(self):
self.factory.addStep(
Latin1ProducingCustomBuildStep(logEncoding='latin-1'))
yield self.do_test_step()
self.assertLogs({
u'xx': u'o\N{CENT SIGN}\n',
})
@defer.inlineCallbacks
def test_OldBuildEPYDoc(self):
# test old-style calls to log.getText, figuring readlines will be ok
self.factory.addStep(OldBuildEPYDoc())
bs = yield self.do_test_step()
self.assertEqual(bs.getResults(), results.FAILURE)
@defer.inlineCallbacks
def test_OldPerlModuleTest(self):
# test old-style calls to self.getLog
self.factory.addStep(OldPerlModuleTest())
bs = yield self.do_test_step()
self.assertEqual(bs.getResults(), results.SUCCESS)