def do_test_master(self):
# create the master and set its config
m = BuildMaster(self.basedir, self.configfile)
m.config = config.MasterConfig.loadConfig(
self.basedir, self.configfile)
# update the DB
yield m.db.setup(check_version=False)
yield m.db.model.upgrade()
# stub out m.db.setup since it was already called above
m.db.setup = lambda : None
# mock reactor.stop (which trial *really* doesn't
# like test code to call!)
mock_reactor = mock.Mock(spec=reactor)
mock_reactor.callWhenRunning = reactor.callWhenRunning
# start the service
yield m.startService(_reactor=mock_reactor)
self.failIf(mock_reactor.stop.called,
"startService tried to stop the reactor; check logs")
# stop the service
yield m.stopService()
# and shutdown the db threadpool, as is normally done at reactor stop
m.db.pool.shutdown()
def do_test_master(self):
# create the master and set its config
m = BuildMaster(self.basedir, self.configfile)
# update the DB
yield m.db.setup(check_version=False)
yield m.db.model.upgrade()
# stub out m.db.setup since it was already called above
m.db.setup = lambda: None
# mock reactor.stop (which trial *really* doesn't
# like test code to call!)
mock_reactor = mock.Mock(spec=reactor)
mock_reactor.callWhenRunning = reactor.callWhenRunning
# start the service
yield m.startService(_reactor=mock_reactor)
self.failIf(mock_reactor.stop.called,
"startService tried to stop the reactor; check logs")
# hang out for a fraction of a second, to let startup processes run
d = defer.Deferred()
reactor.callLater(0.01, d.callback, None)
yield d
# stop the service
yield m.stopService()
# and shutdown the db threadpool, as is normally done at reactor stop
m.db.pool.shutdown()
def upgradeDatabase(config, master_cfg):
if not config['quiet']:
print("upgrading database (%s)"
% (stripUrlPassword(master_cfg.db['db_url'])))
print("Warning: Stopping this process might cause data loss")
def sighandler(signum, frame):
msg = " ".join("""
WARNING: ignoring signal %s.
This process should not be interrupted to avoid database corruption.
If you really need to terminate it, use SIGKILL.
""".split())
print(msg % signum)
prev_handlers = {}
try:
for signame in ("SIGTERM", "SIGINT", "SIGQUIT", "SIGHUP",
"SIGUSR1", "SIGUSR2", "SIGBREAK"):
if hasattr(signal, signame):
signum = getattr(signal, signame)
prev_handlers[signum] = signal.signal(signum, sighandler)
master = BuildMaster(config['basedir'])
master.config = master_cfg
master.db.disownServiceParent()
db = connector.DBConnector(basedir=config['basedir'])
db.setServiceParent(master)
yield db.setup(check_version=False, verbose=not config['quiet'])
yield db.model.upgrade()
yield db.masters.setAllMastersActiveLongTimeAgo()
finally:
# restore previous signal handlers
for signum, handler in prev_handlers.items():
signal.signal(signum, handler)
def do_test_master(self):
# create the master and set its config
m = BuildMaster(self.basedir, self.configfile)
m.config = config.MasterConfig.loadConfig(self.basedir,
self.configfile)
# update the DB
yield m.db.setup(check_version=False)
yield m.db.model.upgrade()
# stub out m.db.setup since it was already called above
m.db.setup = lambda: None
# mock reactor.stop (which trial *really* doesn't
# like test code to call!)
mock_reactor = mock.Mock(spec=reactor)
mock_reactor.callWhenRunning = reactor.callWhenRunning
# start the service
yield m.startService(_reactor=mock_reactor)
self.failIf(mock_reactor.stop.called,
"startService tried to stop the reactor; check logs")
# stop the service
yield m.stopService()
# and shutdown the db threadpool, as is normally done at reactor stop
m.db.pool.shutdown()
def do_test_master(self):
# create the master and set its config
m = BuildMaster(self.basedir, self.configfile)
# update the DB
yield m.db.setup(check_version=False)
yield m.db.model.upgrade()
# stub out m.db.setup since it was already called above
m.db.setup = lambda: None
# mock reactor.stop (which trial *really* doesn't
# like test code to call!)
mock_reactor = mock.Mock(spec=reactor)
mock_reactor.callWhenRunning = reactor.callWhenRunning
mock_reactor.getThreadPool = reactor.getThreadPool
mock_reactor.callFromThread = reactor.callFromThread
# start the service
yield m.startService(_reactor=mock_reactor)
self.failIf(mock_reactor.stop.called,
"startService tried to stop the reactor; check logs")
# hang out for a fraction of a second, to let startup processes run
d = defer.Deferred()
reactor.callLater(0.01, d.callback, None)
yield d
# stop the service
yield m.stopService()
# and shutdown the db threadpool, as is normally done at reactor stop
m.db.pool.shutdown()
def upgradeDatabase(config, master_cfg):
if not config['quiet']:
print("upgrading database (%s)" %
(stripUrlPassword(master_cfg.db['db_url'])))
print("Warning: Stopping this process might cause data loss")
def sighandler(signum, frame):
msg = " ".join("""
WARNING: ignoring signal %s.
This process should not be interrupted to avoid database corruption.
If you really need to terminate it, use SIGKILL.
""".split())
print(msg % signum)
for signame in ("SIGTERM", "SIGINT", "SIGQUIT", "SIGHUP", "SIGUSR1",
"SIGUSR2", "SIGBREAK"):
if hasattr(signal, signame):
signal.signal(getattr(signal, signame), sighandler)
master = BuildMaster(config['basedir'])
master.config = master_cfg
master.db.disownServiceParent()
db = connector.DBConnector(basedir=config['basedir'])
db.setServiceParent(master)
yield db.setup(check_version=False, verbose=not config['quiet'])
yield db.model.upgrade()
yield db.masters.setAllMastersActiveLongTimeAgo()
def doCleanupDatabase(config, master_cfg):
if not config['quiet']:
print("cleaning database (%s)" % (master_cfg.db['db_url']))
master = BuildMaster(config['basedir'])
master.config = master_cfg
print(master.config.logCompressionMethod)
db = master.db
yield db.setup(check_version=False, verbose=not config['quiet'])
res = yield db.logs.getLogs()
i = 0
percent = 0
saved = 0
for log in res:
saved += yield db.logs.compressLog(log['id'])
i += 1
if not config['quiet'] and percent != i * 100 / len(res):
percent = i * 100 / len(res)
print(" {0}% {1} saved".format(percent, saved))
saved = 0
sys.stdout.flush()
if master_cfg.db['db_url'].startswith("sqlite"):
if not config['quiet']:
print("executing sqlite vacuum function...")
# sqlite vacuum function rebuild the whole database to claim
# free disk space back
def thd(engine):
r = engine.execute("vacuum;")
r.close()
yield db.pool.do(thd)
def doCleanupDatabase(config, master_cfg):
if not config["quiet"]:
print("cleaning database (%s)" % (master_cfg.db["db_url"]))
master = BuildMaster(config["basedir"])
master.config = master_cfg
db = master.db
yield db.setup(check_version=False, verbose=not config["quiet"])
res = yield db.logs.getLogs()
i = 0
percent = 0
saved = 0
for log in res:
saved += yield db.logs.compressLog(log["id"], force=config["force"])
i += 1
if not config["quiet"] and percent != i * 100 / len(res):
percent = i * 100 / len(res)
print(" {0}% {1} saved".format(percent, saved))
saved = 0
sys.stdout.flush()
if master_cfg.db["db_url"].startswith("sqlite"):
if not config["quiet"]:
print("executing sqlite vacuum function...")
# sqlite vacuum function rebuild the whole database to claim
# free disk space back
def thd(engine):
r = engine.execute("vacuum;")
r.close()
yield db.pool.do(thd)
def getMaster(case, reactor, config_dict):
"""
Create a started ``BuildMaster`` with the given configuration.
"""
basedir = FilePath(case.mktemp())
basedir.createDirectory()
config_dict['buildbotNetUsageData'] = None
master = BuildMaster(
basedir.path, reactor=reactor, config_loader=DictLoader(config_dict))
if 'db_url' not in config_dict:
config_dict['db_url'] = 'sqlite://'
# TODO: Allow BuildMaster to transparently upgrade the database, at least
# for tests.
master.config.db['db_url'] = config_dict['db_url']
yield master.db.setup(check_version=False)
yield master.db.model.upgrade()
master.db.setup = lambda: None
yield master.startService()
# and shutdown the db threadpool, as is normally done at reactor stop
case.addCleanup(master.db.pool.shutdown)
case.addCleanup(master.stopService)
defer.returnValue(master)
def setupConfig(self, configFunc):
"""
Setup and start a master configured
by the function configFunc defined in the test module.
@type configFunc: string
@param configFunc: name of a function
without argument defined in the test module
that returns a BuildmasterConfig object.
"""
self.basedir = os.path.abspath('basdir')
self.setUpDirs(self.basedir)
self.configfile = os.path.join(self.basedir, 'master.cfg')
if self.proto == 'pb':
proto = '{"pb": {"port": "tcp:0:interface=127.0.0.1"}}'
elif self.proto == 'null':
proto = '{"null": {}}'
# We create a master.cfg, which loads the configuration from the
# test module. Only the slave config is kept there, as it should not
# be changed
open(self.configfile, "w").write(
textwrap.dedent("""
from buildbot.buildslave import BuildSlave
from %s import %s
c = BuildmasterConfig = %s()
c['slaves'] = [BuildSlave("local1", "localpw")]
c['protocols'] = %s
""" % (self.__class__.__module__, configFunc, configFunc, proto)))
# create the master and set its config
m = BuildMaster(self.basedir, self.configfile)
self.master = m
# update the DB
yield m.db.setup(check_version=False)
yield m.db.model.upgrade()
# stub out m.db.setup since it was already called above
m.db.setup = lambda: None
# mock reactor.stop (which trial *really* doesn't
# like test code to call!)
mock_reactor = mock.Mock(spec=reactor)
mock_reactor.callWhenRunning = reactor.callWhenRunning
# start the service
yield m.startService(_reactor=mock_reactor)
self.failIf(mock_reactor.stop.called,
"startService tried to stop the reactor; check logs")
if self.proto == 'pb':
# We find out the slave port automatically
slavePort = list(itervalues(
m.pbmanager.dispatchers))[0].port.getHost().port
# create a slave, and attach it to the master, it will be started, and stopped
# along with the master
s = BuildSlave("127.0.0.1", slavePort, "local1", "localpw",
self.basedir, False, False)
elif self.proto == 'null':
s = LocalBuildSlave("local1", self.basedir, False)
s.setServiceParent(m)
def setupConfig(self, configFunc):
"""
Setup and start a master configured
by the function configFunc defined in the test module.
@type configFunc: string
@param configFunc: name of a function
without argument defined in the test module
that returns a BuildmasterConfig object.
"""
self.basedir = os.path.abspath('basdir')
self.setUpDirs(self.basedir)
self.configfile = os.path.join(self.basedir, 'master.cfg')
if self.proto == 'pb':
proto = '{"pb": {"port": "tcp:0:interface=127.0.0.1"}}'
elif self.proto == 'null':
proto = '{"null": {}}'
# We create a master.cfg, which loads the configuration from the
# test module. Only the slave config is kept there, as it should not
# be changed
open(self.configfile, "w").write(textwrap.dedent("""
from buildbot.buildslave import BuildSlave
from %s import %s
c = BuildmasterConfig = %s()
c['slaves'] = [BuildSlave("local1", "localpw")]
c['protocols'] = %s
""" % (self.__class__.__module__,
configFunc, configFunc,
proto)))
# create the master and set its config
m = BuildMaster(self.basedir, self.configfile)
self.master = m
# update the DB
yield m.db.setup(check_version=False)
yield m.db.model.upgrade()
# stub out m.db.setup since it was already called above
m.db.setup = lambda: None
# mock reactor.stop (which trial *really* doesn't
# like test code to call!)
mock_reactor = mock.Mock(spec=reactor)
mock_reactor.callWhenRunning = reactor.callWhenRunning
# start the service
yield m.startService(_reactor=mock_reactor)
self.failIf(mock_reactor.stop.called,
"startService tried to stop the reactor; check logs")
if self.proto == 'pb':
# We find out the slave port automatically
slavePort = list(itervalues(m.pbmanager.dispatchers))[0].port.getHost().port
# create a slave, and attach it to the master, it will be started, and stopped
# along with the master
s = BuildSlave("127.0.0.1", slavePort, "local1", "localpw", self.basedir, False, False)
elif self.proto == 'null':
s = LocalBuildSlave("local1", self.basedir, False)
s.setServiceParent(m)
def setupConfig(self, config_dict):
"""
Setup and start a master configured
by the function configFunc defined in the test module.
@type config_dict: dict
@param configFunc: The BuildmasterConfig dictionary.
"""
self.basedir = os.path.abspath('basdir')
self.setUpDirs(self.basedir)
# mock reactor.stop (which trial *really* doesn't
# like test code to call!)
mock_reactor = mock.Mock(spec=reactor)
mock_reactor.callWhenRunning = reactor.callWhenRunning
mock_reactor.getThreadPool = reactor.getThreadPool
mock_reactor.callFromThread = reactor.callFromThread
workerclass = worker.Worker
if self.proto == 'pb':
proto = {"pb": {"port": "tcp:0:interface=127.0.0.1"}}
elif self.proto == 'null':
proto = {"null": {}}
workerclass = worker.LocalWorker
config_dict['workers'] = [workerclass("local1", "localpw")]
config_dict['protocols'] = proto
# create the master and set its config
m = BuildMaster(self.basedir,
reactor=mock_reactor,
config_loader=DictLoader(config_dict))
self.master = m
# update the DB
yield m.db.setup(check_version=False)
yield m.db.model.upgrade()
# stub out m.db.setup since it was already called above
m.db.setup = lambda: None
# start the service
yield m.startService()
self.failIf(mock_reactor.stop.called,
"startService tried to stop the reactor; check logs")
if self.proto == 'pb':
# We find out the worker port automatically
workerPort = list(itervalues(
m.pbmanager.dispatchers))[0].port.getHost().port
# create a worker, and attach it to the master, it will be started, and stopped
# along with the master
self.w = BuildSlave("127.0.0.1", workerPort, "local1", "localpw",
self.basedir, False, False)
elif self.proto == 'null':
self.w = None
if self.w is not None:
self.w.setServiceParent(m)
def getMaster(self, config_dict):
"""
Create a started ``BuildMaster`` with the given configuration.
"""
basedir = FilePath(self.mktemp())
basedir.createDirectory()
master = BuildMaster(
basedir.path, reactor=reactor, config_loader=DictLoader(config_dict))
master.config = master.config_loader.loadConfig()
return master
def getMaster(self, config_dict):
"""
Create a started ``BuildMaster`` with the given configuration.
"""
basedir = FilePath(self.mktemp())
basedir.createDirectory()
master = BuildMaster(
basedir.path, reactor=reactor, config_loader=DictLoader(config_dict))
master.config = master.config_loader.loadConfig()
return master
def upgradeDatabase(config, master_cfg):
if not config['quiet']:
print "upgrading database (%s)" % (master_cfg.db['db_url'])
master = BuildMaster(config['basedir'])
master.config = master_cfg
db = connector.DBConnector(master, basedir=config['basedir'])
yield db.setup(check_version=False, verbose=not config['quiet'])
yield db.model.upgrade()
def testSteps(self):
m = BuildMaster(".")
m.loadConfig(cfg1)
b = m.botmaster.builders["builder1"]
steps = b.buildFactory.steps
self.failUnlessEqual(len(steps), 4)
self.failUnlessExpectedShell(steps[0], command="echo yes")
self.failUnlessExpectedShell(steps[1], defaults=False, command="old-style")
self.failUnlessExpectedDarcs(steps[2], repourl="http://buildbot.net/repos/trunk")
self.failUnlessExpectedShell(steps[3], defaults=False, command="echo old-style")
def check_master_cfg(self):
"""Check the buildmaster configuration, returning a deferred that
fires with an approprate exit status (so 0=success)."""
from buildbot.master import BuildMaster
from twisted.python import log, failure
master_cfg = os.path.join(self.basedir, "master.cfg")
if not os.path.exists(master_cfg):
if not self.quiet:
print "No master.cfg found"
return defer.succeed(1)
# side-effects of loading the config file:
# for each Builder defined in c['builders'], if the status directory
# didn't already exist, it will be created, and the
# $BUILDERNAME/builder pickle might be created (with a single
# "builder created" event).
# we put basedir in front of sys.path, because that's how the
# buildmaster itself will run, and it is quite common to have the
# buildmaster import helper classes from other .py files in its
# basedir.
if sys.path[0] != self.basedir:
sys.path.insert(0, self.basedir)
m = BuildMaster(self.basedir)
# we need to route log.msg to stdout, so any problems can be seen
# there. But if everything goes well, I'd rather not clutter stdout
# with log messages. So instead we add a logObserver which gathers
# messages and only displays them if something goes wrong.
messages = []
log.addObserver(messages.append)
try:
# this will raise an exception if there's something wrong with
# the config file. Note that this BuildMaster instance is never
# started, so it won't actually do anything with the
# configuration.
return m.loadConfig(open(master_cfg, "r"), checkOnly=True)
except:
f = failure.Failure()
if not self.quiet:
print
for m in messages:
print "".join(m['message'])
print f
print
print "An error was detected in the master.cfg file."
print "Please correct the problem and run 'buildbot upgrade-master' again."
print
return 1
return 0
def upgradeDatabase(config, master_cfg):
if not config['quiet']:
print("upgrading database (%s)" % (master_cfg.db['db_url']))
master = BuildMaster(config['basedir'])
master.config = master_cfg
master.db.disownServiceParent()
db = connector.DBConnector(basedir=config['basedir'])
db.setServiceParent(master)
yield db.setup(check_version=False, verbose=not config['quiet'])
yield db.model.upgrade()
yield db.masters.setAllMastersActiveLongTimeAgo()
def upgradeDatabase(config, master_cfg):
if not config['quiet']:
print("upgrading database (%s)" % (master_cfg.db['db_url']))
master = BuildMaster(config['basedir'])
master.config = master_cfg
master.db.disownServiceParent()
db = connector.DBConnector(basedir=config['basedir'])
db.setServiceParent(master)
yield db.setup(check_version=False, verbose=not config['quiet'])
yield db.model.upgrade()
yield db.masters.setAllMastersActiveLongTimeAgo()
def check_master_cfg(self):
from buildbot.master import BuildMaster
from twisted.python import log, failure
master_cfg = os.path.join(self.basedir, "master.cfg")
if not os.path.exists(master_cfg):
if not self.quiet:
print "No master.cfg found"
return 1
# side-effects of loading the config file:
# for each Builder defined in c['builders'], if the status directory
# didn't already exist, it will be created, and the
# $BUILDERNAME/builder pickle might be created (with a single
# "builder created" event).
# we put basedir in front of sys.path, because that's how the
# buildmaster itself will run, and it is quite common to have the
# buildmaster import helper classes from other .py files in its
# basedir.
if sys.path[0] != self.basedir:
sys.path.insert(0, self.basedir)
m = BuildMaster(self.basedir)
# we need to route log.msg to stdout, so any problems can be seen
# there. But if everything goes well, I'd rather not clutter stdout
# with log messages. So instead we add a logObserver which gathers
# messages and only displays them if something goes wrong.
messages = []
log.addObserver(messages.append)
try:
# this will raise an exception if there's something wrong with
# the config file. Note that this BuildMaster instance is never
# started, so it won't actually do anything with the
# configuration.
m.loadConfig(open(master_cfg, "r"))
except:
f = failure.Failure()
if not self.quiet:
print
for m in messages:
print "".join(m['message'])
print f
print
print "An error was detected in the master.cfg file."
print "Please correct the problem and run 'buildbot upgrade-master' again."
print
return 1
return 0
def upgradeDatabase(config, master_cfg):
if not config['quiet']:
print("upgrading database (%s)" %
(stripUrlPassword(master_cfg.db['db_url'])))
print("Warning: Stopping this process might cause data loss")
master = BuildMaster(config['basedir'])
master.config = master_cfg
master.db.disownServiceParent()
db = connector.DBConnector(basedir=config['basedir'])
db.setServiceParent(master)
yield db.setup(check_version=False, verbose=not config['quiet'])
yield db.model.upgrade()
yield db.masters.setAllMastersActiveLongTimeAgo()
def createDB(config, _noMonkey=False):
# apply the db monkeypatches (and others - no harm)
if not _noMonkey: # pragma: no cover
monkeypatches.patch_all()
# create a master with the default configuration, but with db_url
# overridden
master_cfg = config_module.MasterConfig()
master_cfg.db['db_url'] = config['db']
master = BuildMaster(config['basedir'])
master.config = master_cfg
db = master.db
yield db.setup(check_version=False, verbose=not config['quiet'])
if not config['quiet']:
print("creating database (%s)" % (master_cfg.db['db_url'],))
yield db.model.upgrade()
def testSteps(self):
m = BuildMaster(".")
m.loadConfig(cfg1)
b = m.botmaster.builders["builder1"]
steps = b.buildFactory.steps
self.failUnlessEqual(len(steps), 4)
self.failUnlessExpectedShell(steps[0], command="echo yes")
self.failUnlessExpectedShell(steps[1],
defaults=False,
command="old-style")
self.failUnlessExpectedDarcs(steps[2],
repourl="http://buildbot.net/repos/trunk")
self.failUnlessExpectedShell(steps[3],
defaults=False,
command="echo old-style")
def createDB(config, _noMonkey=False):
# apply the db monkeypatches (and others - no harm)
if not _noMonkey: # pragma: no cover
monkeypatches.patch_all()
# create a master with the default configuration, but with db_url
# overridden
master_cfg = config_module.MasterConfig()
master_cfg.db['db_url'] = config['db']
master = BuildMaster(config['basedir'])
master.config = master_cfg
db = connector.DBConnector(master, config['basedir'])
yield db.setup(check_version=False, verbose=not config['quiet'])
if not config['quiet']:
print "creating database (%s)" % (master_cfg.db['db_url'],)
yield db.model.upgrade()
def create_db(self):
from buildbot.db import connector
from buildbot.master import BuildMaster
db = connector.DBConnector(BuildMaster(self.basedir),
self.config['db'], basedir=self.basedir)
if not self.config['quiet']: print "creating database"
d = db.model.upgrade()
return d
def create_db(self):
from buildbot.db import connector
from buildbot.master import BuildMaster
from buildbot import config as config_module
# create a master with the default configuration, but with db_url
# overridden
master_cfg = config_module.MasterConfig()
master_cfg.db['db_url'] = self.config['db']
master = BuildMaster(self.basedir)
master.config = master_cfg
db = connector.DBConnector(master, self.basedir)
d = db.setup(check_version=False)
if not self.config['quiet']:
print "creating database (%s)" % (master_cfg.db['db_url'], )
d = db.model.upgrade()
return d
def testStartService(self):
os.mkdir("test_ss")
self.master = m = BuildMaster("test_ss")
# inhibit the usual read-config-on-startup behavior
m.readConfig = True
m.startService()
d = m.loadConfig(startableEmptyCfg % 0)
d.addCallback(self._testStartService_0)
return d
def setUp(self):
self.basedir = os.path.abspath('basdir')
self.setUpDirs(self.basedir)
self.configfile = os.path.join(self.basedir, 'master.cfg')
# We create a master.cfg, which loads the configuration from the
# test module. Only the slave config is kept there, as it should not
# be changed
open(self.configfile, "w").write(textwrap.dedent("""
from buildbot.buildslave import BuildSlave
from %s import masterConfig
c = BuildmasterConfig = masterConfig()
c['slaves'] = [BuildSlave("local1", "localpw")]
c['protocols'] = {"pb": {"port": "tcp:0:interface=127.0.0.1"}}
""" % self.__class__.__module__))
# create the master and set its config
m = BuildMaster(self.basedir, self.configfile)
self.master = m
# update the DB
yield m.db.setup(check_version=False)
yield m.db.model.upgrade()
# stub out m.db.setup since it was already called above
m.db.setup = lambda: None
# mock reactor.stop (which trial *really* doesn't
# like test code to call!)
mock_reactor = mock.Mock(spec=reactor)
mock_reactor.callWhenRunning = reactor.callWhenRunning
# start the service
yield m.startService(_reactor=mock_reactor)
self.failIf(mock_reactor.stop.called,
"startService tried to stop the reactor; check logs")
# We find out the slave port automatically
slavePort = m.pbmanager.dispatchers.values()[0].port.getHost().port
# create a slave, and attach it to the master, it will be started, and stopped
# along with the master
s = BuildSlave("127.0.0.1", slavePort, "local1", "localpw", self.basedir, False, False)
s.setServiceParent(m)
def setUp(self):
self.basedir = os.path.abspath('basdir')
self.setUpDirs(self.basedir)
self.configfile = os.path.join(self.basedir, 'master.cfg')
# We create a master.cfg, which loads the configuration from the
# test module. Only the slave config is kept there, as it should not
# be changed
open(self.configfile, "w").write(textwrap.dedent("""
from buildbot.buildslave import BuildSlave
from %s import masterConfig
c = BuildmasterConfig = masterConfig()
c['slaves'] = [BuildSlave("local1", "localpw")]
c['protocols'] = {"pb": {"port": "tcp:0:interface=127.0.0.1"}}
""" % self.__class__.__module__))
# create the master and set its config
m = BuildMaster(self.basedir, self.configfile)
self.master = m
# update the DB
yield m.db.setup(check_version=False)
yield m.db.model.upgrade()
# stub out m.db.setup since it was already called above
m.db.setup = lambda: None
# mock reactor.stop (which trial *really* doesn't
# like test code to call!)
mock_reactor = mock.Mock(spec=reactor)
mock_reactor.callWhenRunning = reactor.callWhenRunning
# start the service
yield m.startService(_reactor=mock_reactor)
self.failIf(mock_reactor.stop.called,
"startService tried to stop the reactor; check logs")
# We find out the slave port automatically
slavePort = m.pbmanager.dispatchers.values()[0].port.getHost().port
# create a slave, and attach it to the master, it will be started, and stopped
# along with the master
s = BuildSlave("127.0.0.1", slavePort, "local1", "localpw", self.basedir, False, False)
s.setServiceParent(m)
def testFindConfigFile(self):
os.mkdir("test_cf")
open(os.path.join("test_cf", "master.cfg"), "w").write(emptyCfg)
slaveportCfg = emptyCfg + "c['slavePortnum'] = 9000\n"
open(os.path.join("test_cf", "alternate.cfg"), "w").write(slaveportCfg)
m = BuildMaster("test_cf")
m.loadTheConfigFile()
self.failUnlessEqual(m.slavePortnum, "tcp:9999")
m = BuildMaster("test_cf", "alternate.cfg")
m.loadTheConfigFile()
self.failUnlessEqual(m.slavePortnum, "tcp:9000")
def getMaster(case, reactor, config_dict):
"""
Create a started ``BuildMaster`` with the given configuration.
"""
basedir = FilePath(case.mktemp())
basedir.createDirectory()
master = BuildMaster(basedir.path, reactor=reactor, config_loader=DictLoader(config_dict))
if 'db_url' not in config_dict:
config_dict['db_url'] = 'sqlite://'
# TODO: Allow BuildMaster to transparently upgrade the database, at least for tests.
master.config.db['db_url'] = config_dict['db_url']
yield master.db.setup(check_version=False)
yield master.db.model.upgrade()
master.db.setup = lambda: None
yield master.startService()
defer.returnValue(master)
def doCleanupDatabase(config, master_cfg):
if not config['quiet']:
print("cleaning database (%s)" % (master_cfg.db['db_url']))
master = BuildMaster(config['basedir'])
master.config = master_cfg
db = master.db
yield db.setup(check_version=False, verbose=not config['quiet'])
res = yield db.logs.getLogs()
i = 0
percent = 0
saved = 0
for log in res:
saved += yield db.logs.compressLog(log['id'], force=config['force'])
i += 1
if not config['quiet'] and percent != i * 100 / len(res):
percent = i * 100 / len(res)
print(" {0}% {1} saved".format(percent, saved))
saved = 0
sys.stdout.flush()
if master_cfg.db['db_url'].startswith("sqlite"):
if not config['quiet']:
print("executing sqlite vacuum function...")
# sqlite vacuum function rebuild the whole database to claim
# free disk space back
def thd(engine):
# In Python 3.6 and higher, sqlite3 no longer commits an
# open transaction before DDL statements.
# It is necessary to set the isolation_level to none
# for auto-commit mode before doing a VACUUM.
# See: https://bugs.python.org/issue28518
# Get the underlying sqlite connection from SQLAlchemy.
sqlite_conn = engine.connection.connection
# Set isolation_level to 'auto-commit mode'
sqlite_conn.isolation_level = None
sqlite_conn.execute("vacuum;").close()
yield db.pool.do(thd)
def populate_database(config):
master = BuildMaster(config['baseDir'])
master.config = load_config(config, config['configFile'])
db = connector.DBConnector(master, basedir=config['baseDir'])
seed = int(time())
if config['seed']:
seed = int(config['seed'])
random.seed(seed)
if not config['quiet']:
print("Seed =", seed)
yield db.setup(check_version=False, verbose=not config['quiet'])
users = yield populate_user(db,
int(config['users']),
verbose=not config['quiet'])
yield populate_build(db,
int(config['builds']),
master.config.builders,
master.config.projects,
users,
verbose=not config['quiet'])
def _run_master(self, loaded_config):
# create the master
m = BuildMaster(self.basedir, self.configfile)
# update the DB
yield m.db.setup(check_version=False)
yield m.db.model.upgrade()
# stub out m.db.setup since it was already called above
m.db.setup = lambda: None
# mock reactor.stop (which trial *really* doesn't
# like test code to call!)
mock_reactor = mock.Mock(spec=reactor)
mock_reactor.callWhenRunning = reactor.callWhenRunning
mock_reactor.getThreadPool = reactor.getThreadPool
mock_reactor.callFromThread = reactor.callFromThread
# mock configuration loading
@classmethod
def loadConfig(cls, basedir, filename):
return loaded_config
with mock.patch('buildbot.config.MasterConfig.loadConfig', loadConfig):
# start the service
yield m.startService(_reactor=mock_reactor)
self.failIf(mock_reactor.stop.called,
"startService tried to stop the reactor; check logs")
# hang out for a fraction of a second, to let startup processes run
d = defer.Deferred()
reactor.callLater(0.01, d.callback, None)
yield d
# stop the service
yield m.stopService()
# and shutdown the db threadpool, as is normally done at reactor stop
m.db.pool.shutdown()
def _run_master(self, loaded_config):
# create the master
m = BuildMaster(self.basedir, self.configfile)
# update the DB
yield m.db.setup(check_version=False)
yield m.db.model.upgrade()
# stub out m.db.setup since it was already called above
m.db.setup = lambda: None
# mock reactor.stop (which trial *really* doesn't
# like test code to call!)
mock_reactor = mock.Mock(spec=reactor)
mock_reactor.callWhenRunning = reactor.callWhenRunning
mock_reactor.getThreadPool = reactor.getThreadPool
mock_reactor.callFromThread = reactor.callFromThread
# mock configuration loading
@classmethod
def loadConfig(cls, basedir, filename):
return loaded_config
with mock.patch('buildbot.config.MasterConfig.loadConfig', loadConfig):
# start the service
yield m.startService(_reactor=mock_reactor)
self.failIf(mock_reactor.stop.called,
"startService tried to stop the reactor; check logs")
# hang out for a fraction of a second, to let startup processes run
d = defer.Deferred()
reactor.callLater(0.01, d.callback, None)
yield d
# stop the service
yield m.stopService()
# and shutdown the db threadpool, as is normally done at reactor stop
m.db.pool.shutdown()
def upgradeMaster(config):
m = Maker(config)
if not config['quiet']: print "upgrading basedir"
basedir = os.path.expanduser(config['basedir'])
# TODO: check Makefile
# TODO: check TAC file
# check web files: index.html, default.css, robots.txt
m.upgrade_public_html({
'bg_gradient.jpg':
util.sibpath(__file__, "../status/web/files/bg_gradient.jpg"),
'default.css':
util.sibpath(__file__, "../status/web/files/default.css"),
'robots.txt':
util.sibpath(__file__, "../status/web/files/robots.txt"),
'favicon.ico':
util.sibpath(__file__, "../status/web/files/favicon.ico"),
})
m.populate_if_missing(os.path.join(basedir, "master.cfg.sample"),
util.sibpath(__file__, "sample.cfg"),
overwrite=True)
# if index.html exists, use it to override the root page tempalte
m.move_if_present(os.path.join(basedir, "public_html/index.html"),
os.path.join(basedir, "templates/root.html"))
if not config['quiet']: print "checking master.cfg"
wfd = defer.waitForDeferred(
m.check_master_cfg(expected_db_url=config['db']))
yield wfd
rc = wfd.getResult()
if rc == 0:
from buildbot.db import connector
from buildbot.master import BuildMaster
if not config['quiet']: print "upgrading database"
db = connector.DBConnector(BuildMaster(config['basedir']),
config['db'],
basedir=config['basedir'])
wfd = defer.waitForDeferred(db.model.upgrade())
yield wfd
wfd.getResult()
if not config['quiet']: print "upgrade complete"
yield 0
else:
yield rc
def __init__(self,
config,
reactor=None,
source='TestMaster',
log_handler=None,
attach_on=tuple()):
"""Lightweight in-process BuildMaster
Spins up a lightweight BuildMaster in the same process and can trigger
builders defined in the configuration. The TestMaster only pays
attention to the `workers`, `builders` and `schedulers` configuration
keys, so it doesn't configure non-essential services like the
reporters.
It is used in the CLI interface to locally reproduce specific builds,
but it is also suitable for general integration testing of the
builders.
Parameters
----------
config: MasterConfig
reactor: twisted.reactor, default None
source: str, default `TestMaster`
Used for highligting the origin or the build properties.
log_handler: Callable[[unseen_log_lines], None], default lambda _: None
A callback to handle the logs produced by the builder's buildsteps.
attach_on: List[Results], default []
If a build finishes with any of the listed states and it is
executed withing a DockerLatentWorker then start an interactive
shell session in the container. Use it with caution, because it
blocks the event loop.
"""
assert isinstance(config, MasterConfig)
assert all(result in ALL_RESULTS for result in attach_on)
self.config = config
self.attach_on = set(attach_on)
loader = EagerLoader(config, source=source)
if reactor is None:
from twisted.internet import reactor
self._source = source
self._master = BuildMaster('.', reactor=reactor, config_loader=loader)
self._log_handler = log_handler or (lambda _: None)
# state variable updated by the event handlers below
self._buildset = None
self._buildset_id = None
self._log_offset = 0
def testFindConfigFile(self):
os.mkdir("test_cf")
open(os.path.join("test_cf", "master.cfg"), "w").write(emptyCfg)
slaveportCfg = emptyCfg + "c['slavePortnum'] = 9000\n"
open(os.path.join("test_cf", "alternate.cfg"), "w").write(slaveportCfg)
m = BuildMaster("test_cf")
m.loadTheConfigFile()
self.failUnlessEqual(m.slavePortnum, "tcp:9999")
m = BuildMaster("test_cf", "alternate.cfg")
m.loadTheConfigFile()
self.failUnlessEqual(m.slavePortnum, "tcp:9000")
def check_master_cfg(self, expected_db_url=None):
"""Check the buildmaster configuration, returning a deferred that
fires with an approprate exit status (so 0=success)."""
from buildbot.master import BuildMaster
from twisted.python import log
master_cfg = os.path.join(self.basedir, "master.cfg")
if not os.path.exists(master_cfg):
if not self.quiet:
print "No master.cfg found"
return defer.succeed(1)
# side-effects of loading the config file:
# for each Builder defined in c['builders'], if the status directory
# didn't already exist, it will be created, and the
# $BUILDERNAME/builder pickle might be created (with a single
# "builder created" event).
# we put basedir in front of sys.path, because that's how the
# buildmaster itself will run, and it is quite common to have the
# buildmaster import helper classes from other .py files in its
# basedir.
if sys.path[0] != self.basedir:
sys.path.insert(0, self.basedir)
m = BuildMaster(self.basedir)
# we need to route log.msg to stdout, so any problems can be seen
# there. But if everything goes well, I'd rather not clutter stdout
# with log messages. So instead we add a logObserver which gathers
# messages and only displays them if something goes wrong.
messages = []
log.addObserver(messages.append)
# this will errback if there's something wrong with the config file.
# Note that this BuildMaster instance is never started, so it won't
# actually do anything with the configuration.
d = defer.maybeDeferred(lambda :
m.loadConfig(open(master_cfg, "r"), checkOnly=True))
def check_db_url(config):
if (expected_db_url and
config.get('db_url', 'sqlite:///state.sqlite') != expected_db_url):
raise ValueError("c['db_url'] in the config file ('%s') does"
" not match '%s'; please edit the configuration"
" file before upgrading." %
(config['db_url'], expected_db_url))
d.addCallback(check_db_url)
def cb(_):
return 0
def eb(f):
if not self.quiet:
print
for m in messages:
print "".join(m['message'])
f.printTraceback()
print
print "An error was detected in the master.cfg file."
print "Please correct the problem and run 'buildbot upgrade-master' again."
print
return 1
d.addCallbacks(cb, eb)
return d
def setupConfig(self, configFunc):
"""
Setup and start a master configured
by the function configFunc defined in the test module.
@type configFunc: string
@param configFunc: name of a function
without argument defined in the test module
that returns a BuildmasterConfig object.
"""
self.basedir = os.path.abspath("basdir")
self.setUpDirs(self.basedir)
self.configfile = os.path.join(self.basedir, "master.cfg")
workerclass = "BuildWorker"
if self.proto == "pb":
proto = '{"pb": {"port": "tcp:0:interface=127.0.0.1"}}'
elif self.proto == "null":
proto = '{"null": {}}'
workerclass = "LocalBuildWorker"
# We create a master.cfg, which loads the configuration from the
# test module. Only the worker config is kept there, as it should not
# be changed
open(self.configfile, "w").write(
textwrap.dedent(
"""
from buildbot.plugins import buildworker
from {module} import {configFunc}
c = BuildmasterConfig = {configFunc}()
c['workers'] = [buildworker.{workerclass}("local1", "localpw")]
c['protocols'] = {proto}
"""
).format(module=self.__class__.__module__, configFunc=configFunc, proto=proto, workerclass=workerclass)
)
# create the master and set its config
m = BuildMaster(self.basedir, self.configfile)
self.master = m
# update the DB
yield m.db.setup(check_version=False)
yield m.db.model.upgrade()
# stub out m.db.setup since it was already called above
m.db.setup = lambda: None
# mock reactor.stop (which trial *really* doesn't
# like test code to call!)
mock_reactor = mock.Mock(spec=reactor)
mock_reactor.callWhenRunning = reactor.callWhenRunning
# start the service
yield m.startService(_reactor=mock_reactor)
self.failIf(mock_reactor.stop.called, "startService tried to stop the reactor; check logs")
if self.proto == "pb":
# We find out the worker port automatically
workerPort = list(itervalues(m.pbmanager.dispatchers))[0].port.getHost().port
# create a worker, and attach it to the master, it will be started, and stopped
# along with the master
s = BuildWorker("127.0.0.1", workerPort, "local1", "localpw", self.basedir, False, False)
elif self.proto == "null":
s = None
if s is not None:
s.setServiceParent(m)
class ConfigTest(unittest.TestCase):
def setUp(self):
# this class generates several deprecation warnings, which the user
# doesn't need to see.
warnings.simplefilter('ignore', exceptions.DeprecationWarning)
self.buildmaster = BuildMaster(".")
def failUnlessListsEquivalent(self, list1, list2):
l1 = list1[:]
l1.sort()
l2 = list2[:]
l2.sort()
self.failUnlessEqual(l1, l2)
def servers(self, s, types):
# perform a recursive search of s.services, looking for instances of
# twisted.application.internet.TCPServer, then extract their .args
# values to find the TCP ports they want to listen on
for child in s:
if service.IServiceCollection.providedBy(child):
for gc in self.servers(child, types):
yield gc
if isinstance(child, types):
yield child
def TCPports(self, s):
return list(self.servers(s, internet.TCPServer))
def UNIXports(self, s):
return list(self.servers(s, internet.UNIXServer))
def TCPclients(self, s):
return list(self.servers(s, internet.TCPClient))
def checkPorts(self, svc, expected):
"""Verify that the TCPServer and UNIXServer children of the given
service have the expected portnum/pathname and factory classes. As a
side-effect, return a list of servers in the same order as the
'expected' list. This can be used to verify properties of the
factories contained therein."""
expTCP = [e for e in expected if type(e[0]) == int]
expUNIX = [e for e in expected if type(e[0]) == str]
haveTCP = [(p.args[0], p.args[1].__class__)
for p in self.TCPports(svc)]
haveUNIX = [(p.args[0], p.args[1].__class__)
for p in self.UNIXports(svc)]
self.failUnlessListsEquivalent(expTCP, haveTCP)
self.failUnlessListsEquivalent(expUNIX, haveUNIX)
ret = []
for e in expected:
for have in self.TCPports(svc) + self.UNIXports(svc):
if have.args[0] == e[0]:
ret.append(have)
continue
assert (len(ret) == len(expected))
return ret
def testEmpty(self):
self.failUnlessRaises(KeyError, self.buildmaster.loadConfig, "")
def testSimple(self):
# covers slavePortnum, base checker passwords
master = self.buildmaster
master.loadChanges()
master.loadConfig(emptyCfg)
# note: this doesn't actually start listening, because the app
# hasn't been started running
self.failUnlessEqual(master.slavePortnum, "tcp:9999")
self.checkPorts(master, [(9999, pb.PBServerFactory)])
self.failUnlessEqual(list(master.change_svc), [])
self.failUnlessEqual(master.botmaster.builders, {})
self.failUnlessEqual(master.checker.users, {"change": "changepw"})
self.failUnlessEqual(master.projectName, "dummy project")
self.failUnlessEqual(master.projectURL, "http://dummy.example.com")
self.failUnlessEqual(master.buildbotURL,
"http://dummy.example.com/buildbot")
def testSlavePortnum(self):
master = self.buildmaster
master.loadChanges()
master.loadConfig(emptyCfg)
self.failUnlessEqual(master.slavePortnum, "tcp:9999")
ports = self.checkPorts(master, [(9999, pb.PBServerFactory)])
p = ports[0]
master.loadConfig(emptyCfg)
self.failUnlessEqual(master.slavePortnum, "tcp:9999")
ports = self.checkPorts(master, [(9999, pb.PBServerFactory)])
self.failUnlessIdentical(p, ports[0],
"the slave port was changed even " + \
"though the configuration was not")
master.loadConfig(emptyCfg + "c['slavePortnum'] = 9000\n")
self.failUnlessEqual(master.slavePortnum, "tcp:9000")
ports = self.checkPorts(master, [(9000, pb.PBServerFactory)])
self.failIf(p is ports[0],
"slave port was unchanged but configuration was changed")
def testSlaves(self):
master = self.buildmaster
master.loadChanges()
master.loadConfig(emptyCfg)
self.failUnlessEqual(master.botmaster.builders, {})
self.failUnlessEqual(master.checker.users, {"change": "changepw"})
# 'botsCfg' is testing backwards compatibility, for 0.7.5 config
# files that have not yet been updated to 0.7.6 . This compatibility
# (and this test) is scheduled for removal in 0.8.0 .
botsCfg = (emptyCfg +
"c['bots'] = [('bot1', 'pw1'), ('bot2', 'pw2')]\n")
master.loadConfig(botsCfg)
self.failUnlessEqual(master.checker.users, {
"change": "changepw",
"bot1": "pw1",
"bot2": "pw2"
})
master.loadConfig(botsCfg)
self.failUnlessEqual(master.checker.users, {
"change": "changepw",
"bot1": "pw1",
"bot2": "pw2"
})
master.loadConfig(emptyCfg)
self.failUnlessEqual(master.checker.users, {"change": "changepw"})
slavesCfg = (emptyCfg + "from buildbot.buildslave import BuildSlave\n"
"c['slaves'] = [BuildSlave('bot1','pw1'), "
"BuildSlave('bot2','pw2')]\n")
master.loadConfig(slavesCfg)
self.failUnlessEqual(master.checker.users, {
"change": "changepw",
"bot1": "pw1",
"bot2": "pw2"
})
def testChangeSource(self):
master = self.buildmaster
master.loadChanges()
master.loadConfig(emptyCfg)
self.failUnlessEqual(list(master.change_svc), [])
sourcesCfg = emptyCfg + \
"""
from buildbot.changes.pb import PBChangeSource
c['change_source'] = PBChangeSource()
"""
d = master.loadConfig(sourcesCfg)
def _check1(res):
self.failUnlessEqual(len(list(self.buildmaster.change_svc)), 1)
s1 = list(self.buildmaster.change_svc)[0]
self.failUnless(isinstance(s1, PBChangeSource))
self.failUnlessEqual(s1, list(self.buildmaster.change_svc)[0])
self.failUnless(s1.parent)
# verify that unchanged sources are not interrupted
d1 = self.buildmaster.loadConfig(sourcesCfg)
def _check2(res):
self.failUnlessEqual(len(list(self.buildmaster.change_svc)), 1)
s2 = list(self.buildmaster.change_svc)[0]
self.failUnlessIdentical(s1, s2)
self.failUnless(s1.parent)
d1.addCallback(_check2)
return d1
d.addCallback(_check1)
# make sure we can get rid of the sources too
d.addCallback(lambda res: self.buildmaster.loadConfig(emptyCfg))
def _check3(res):
self.failUnlessEqual(list(self.buildmaster.change_svc), [])
d.addCallback(_check3)
return d
def testChangeSources(self):
# make sure we can accept a list
master = self.buildmaster
master.loadChanges()
master.loadConfig(emptyCfg)
self.failUnlessEqual(list(master.change_svc), [])
sourcesCfg = emptyCfg + \
"""
from buildbot.changes.pb import PBChangeSource
from buildbot.changes.mail import SyncmailMaildirSource
c['change_source'] = [PBChangeSource(),
SyncmailMaildirSource('.'),
]
"""
d = master.loadConfig(sourcesCfg)
def _check1(res):
self.failUnlessEqual(len(list(self.buildmaster.change_svc)), 2)
s1, s2 = list(self.buildmaster.change_svc)
if isinstance(s2, PBChangeSource):
s1, s2 = s2, s1
self.failUnless(isinstance(s1, PBChangeSource))
self.failUnless(s1.parent)
self.failUnless(isinstance(s2, SyncmailMaildirSource))
self.failUnless(s2.parent)
d.addCallback(_check1)
return d
def testSources(self):
# test backwards compatibility. c['sources'] is deprecated.
master = self.buildmaster
master.loadChanges()
master.loadConfig(emptyCfg)
self.failUnlessEqual(list(master.change_svc), [])
sourcesCfg = emptyCfg + \
"""
from buildbot.changes.pb import PBChangeSource
c['sources'] = [PBChangeSource()]
"""
d = master.loadConfig(sourcesCfg)
def _check1(res):
self.failUnlessEqual(len(list(self.buildmaster.change_svc)), 1)
s1 = list(self.buildmaster.change_svc)[0]
self.failUnless(isinstance(s1, PBChangeSource))
self.failUnless(s1.parent)
d.addCallback(_check1)
return d
def shouldBeFailure(self, res, *expected):
self.failUnless(isinstance(res, failure.Failure),
"we expected this to fail, not produce %s" % (res, ))
res.trap(*expected)
return None # all is good
def testSchedulerErrors(self):
master = self.buildmaster
master.loadChanges()
master.loadConfig(emptyCfg)
self.failUnlessEqual(master.allSchedulers(), [])
def _shouldBeFailure(res, hint=None):
self.shouldBeFailure(res, AssertionError, ValueError)
if hint:
self.failUnless(str(res).find(hint) != -1)
def _loadConfig(res, newcfg):
return self.buildmaster.loadConfig(newcfg)
d = defer.succeed(None)
# c['schedulers'] must be a list
badcfg = schedulersCfg + \
"""
c['schedulers'] = Scheduler('full', None, 60, ['builder1'])
"""
d.addCallback(_loadConfig, badcfg)
d.addBoth(_shouldBeFailure,
"c['schedulers'] must be a list of Scheduler instances")
# c['schedulers'] must be a list of IScheduler objects
badcfg = schedulersCfg + \
"""
c['schedulers'] = ['oops', 'problem']
"""
d.addCallback(_loadConfig, badcfg)
d.addBoth(_shouldBeFailure,
"c['schedulers'] must be a list of Scheduler instances")
# c['schedulers'] must point at real builders
badcfg = schedulersCfg + \
"""
c['schedulers'] = [Scheduler('full', None, 60, ['builder-bogus'])]
"""
d.addCallback(_loadConfig, badcfg)
d.addBoth(_shouldBeFailure, "uses unknown builder")
# builderNames= must be a list
badcfg = schedulersCfg + \
"""
c['schedulers'] = [Scheduler('full', None, 60, 'builder1')]
"""
d.addCallback(_loadConfig, badcfg)
d.addBoth(_shouldBeFailure,
"must be a list of Builder description names")
# builderNames= must be a list of strings, not dicts
badcfg = schedulersCfg + \
"""
c['schedulers'] = [Scheduler('full', None, 60, [b1])]
"""
d.addCallback(_loadConfig, badcfg)
d.addBoth(_shouldBeFailure,
"must be a list of Builder description names")
# builderNames= must be a list of strings, not a dict
badcfg = schedulersCfg + \
"""
c['schedulers'] = [Scheduler('full', None, 60, b1)]
"""
d.addCallback(_loadConfig, badcfg)
d.addBoth(_shouldBeFailure,
"must be a list of Builder description names")
# each Scheduler must have a unique name
badcfg = schedulersCfg + \
"""
c['schedulers'] = [Scheduler('dup', None, 60, []),
Scheduler('dup', None, 60, [])]
"""
d.addCallback(_loadConfig, badcfg)
d.addBoth(_shouldBeFailure, "Schedulers must have unique names")
return d
def testSchedulers(self):
master = self.buildmaster
master.loadChanges()
master.loadConfig(emptyCfg)
self.failUnlessEqual(master.allSchedulers(), [])
d = self.buildmaster.loadConfig(schedulersCfg)
d.addCallback(self._testSchedulers_1)
return d
def _testSchedulers_1(self, res):
sch = self.buildmaster.allSchedulers()
self.failUnlessEqual(len(sch), 1)
s = sch[0]
self.failUnless(isinstance(s, scheduler.Scheduler))
self.failUnlessEqual(s.name, "full")
self.failUnlessEqual(s.branch, None)
self.failUnlessEqual(s.treeStableTimer, 60)
self.failUnlessEqual(s.builderNames, ['builder1'])
newcfg = schedulersCfg + \
"""
s1 = Scheduler('full', None, 60, ['builder1'])
c['schedulers'] = [s1, Dependent('downstream', s1, ['builder1'])]
"""
d = self.buildmaster.loadConfig(newcfg)
d.addCallback(self._testSchedulers_2, newcfg)
return d
def _testSchedulers_2(self, res, newcfg):
sch = self.buildmaster.allSchedulers()
self.failUnlessEqual(len(sch), 2)
s = sch[0]
self.failUnless(isinstance(s, scheduler.Scheduler))
s = sch[1]
self.failUnless(isinstance(s, scheduler.Dependent))
self.failUnlessEqual(s.name, "downstream")
self.failUnlessEqual(s.builderNames, ['builder1'])
# reloading the same config file should leave the schedulers in place
d = self.buildmaster.loadConfig(newcfg)
d.addCallback(self._testSchedulers_3, sch)
return d
def _testSchedulers_3(self, res, sch1):
sch2 = self.buildmaster.allSchedulers()
self.failUnlessEqual(len(sch2), 2)
sch1.sort()
sch2.sort()
self.failUnlessEqual(sch1, sch2)
self.failUnlessIdentical(sch1[0], sch2[0])
self.failUnlessIdentical(sch1[1], sch2[1])
self.failUnlessIdentical(sch1[0].parent, self.buildmaster)
self.failUnlessIdentical(sch1[1].parent, self.buildmaster)
def testBuilders(self):
master = self.buildmaster
master.loadConfig(emptyCfg)
self.failUnlessEqual(master.botmaster.builders, {})
master.loadConfig(buildersCfg)
self.failUnlessEqual(master.botmaster.builderNames, ["builder1"])
self.failUnlessEqual(master.botmaster.builders.keys(), ["builder1"])
b = master.botmaster.builders["builder1"]
self.failUnless(isinstance(b, Builder))
self.failUnlessEqual(b.name, "builder1")
self.failUnlessEqual(b.slavenames, ["bot1"])
self.failUnlessEqual(b.builddir, "workdir")
f1 = b.buildFactory
self.failUnless(isinstance(f1, BasicBuildFactory))
steps = f1.steps
self.failUnlessEqual(len(steps), 3)
self.failUnlessEqual(steps[0], (CVS, {
'cvsroot': 'cvsroot',
'cvsmodule': 'cvsmodule',
'mode': 'clobber'
}))
self.failUnlessEqual(steps[1], (Compile, {'command': 'make all'}))
self.failUnlessEqual(steps[2], (Test, {'command': 'make check'}))
# make sure a reload of the same data doesn't interrupt the Builder
master.loadConfig(buildersCfg)
self.failUnlessEqual(master.botmaster.builderNames, ["builder1"])
self.failUnlessEqual(master.botmaster.builders.keys(), ["builder1"])
b2 = master.botmaster.builders["builder1"]
self.failUnlessIdentical(b, b2)
# TODO: test that the BuilderStatus object doesn't change
#statusbag2 = master.client_svc.statusbags["builder1"]
#self.failUnlessIdentical(statusbag, statusbag2)
# but changing something should result in a new Builder
master.loadConfig(buildersCfg2)
self.failUnlessEqual(master.botmaster.builderNames, ["builder1"])
self.failUnlessEqual(master.botmaster.builders.keys(), ["builder1"])
b3 = master.botmaster.builders["builder1"]
self.failIf(b is b3)
# the statusbag remains the same TODO
#statusbag3 = master.client_svc.statusbags["builder1"]
#self.failUnlessIdentical(statusbag, statusbag3)
# adding new builder
master.loadConfig(buildersCfg3)
self.failUnlessEqual(master.botmaster.builderNames,
["builder1", "builder2"])
self.failUnlessListsEquivalent(master.botmaster.builders.keys(),
["builder1", "builder2"])
b4 = master.botmaster.builders["builder1"]
self.failUnlessIdentical(b3, b4)
# changing first builder should leave it at the same place in the list
master.loadConfig(buildersCfg4)
self.failUnlessEqual(master.botmaster.builderNames,
["builder1", "builder2"])
self.failUnlessListsEquivalent(master.botmaster.builders.keys(),
["builder1", "builder2"])
b5 = master.botmaster.builders["builder1"]
self.failIf(b4 is b5)
master.loadConfig(buildersCfg5)
self.failUnlessEqual(master.botmaster.builderNames,
["builder1", "builder2"])
self.failUnlessListsEquivalent(master.botmaster.builders.keys(),
["builder1", "builder2"])
b5 = master.botmaster.builders["builder1"]
# and removing it should make the Builder go away
master.loadConfig(emptyCfg)
self.failUnlessEqual(master.botmaster.builderNames, [])
self.failUnlessEqual(master.botmaster.builders, {})
#self.failUnlessEqual(master.client_svc.statusbags, {}) # TODO
def testWithProperties(self):
master = self.buildmaster
master.loadConfig(wpCfg1)
self.failUnlessEqual(master.botmaster.builderNames, ["builder1"])
self.failUnlessEqual(master.botmaster.builders.keys(), ["builder1"])
b1 = master.botmaster.builders["builder1"]
# reloading the same config should leave the builder unchanged
master.loadConfig(wpCfg1)
b2 = master.botmaster.builders["builder1"]
self.failUnlessIdentical(b1, b2)
# but changing the parameters of the WithProperties should change it
master.loadConfig(wpCfg2)
b3 = master.botmaster.builders["builder1"]
self.failIf(b1 is b3)
# again, reloading same config should leave the builder unchanged
master.loadConfig(wpCfg2)
b4 = master.botmaster.builders["builder1"]
self.failUnlessIdentical(b3, b4)
def checkIRC(self, m, expected):
ircs = {}
for irc in self.servers(m, words.IRC):
ircs[irc.host] = (irc.nick, irc.channels)
self.failUnlessEqual(ircs, expected)
def testIRC(self):
if not words:
raise unittest.SkipTest("Twisted Words package is not installed")
master = self.buildmaster
master.loadChanges()
d = master.loadConfig(emptyCfg)
e1 = {}
d.addCallback(lambda res: self.checkIRC(master, e1))
d.addCallback(lambda res: master.loadConfig(ircCfg1))
e2 = {'irc.us.freenode.net': ('buildbot', ['twisted'])}
d.addCallback(lambda res: self.checkIRC(master, e2))
d.addCallback(lambda res: master.loadConfig(ircCfg2))
e3 = {
'irc.us.freenode.net': ('buildbot', ['twisted']),
'irc.example.com': ('otherbot', ['chan1', 'chan2'])
}
d.addCallback(lambda res: self.checkIRC(master, e3))
d.addCallback(lambda res: master.loadConfig(ircCfg3))
e4 = {'irc.us.freenode.net': ('buildbot', ['knotted'])}
d.addCallback(lambda res: self.checkIRC(master, e4))
d.addCallback(lambda res: master.loadConfig(ircCfg1))
e5 = {'irc.us.freenode.net': ('buildbot', ['twisted'])}
d.addCallback(lambda res: self.checkIRC(master, e5))
return d
def testWebPortnum(self):
master = self.buildmaster
master.loadChanges()
d = master.loadConfig(webCfg1)
def _check1(res):
ports = self.checkPorts(self.buildmaster,
[(9999, pb.PBServerFactory), (9980, Site)])
p = ports[1]
self.p = p
# nothing should be changed
d.addCallback(_check1)
d.addCallback(lambda res: self.buildmaster.loadConfig(webCfg1))
def _check2(res):
ports = self.checkPorts(self.buildmaster,
[(9999, pb.PBServerFactory), (9980, Site)])
self.failUnlessIdentical(
self.p, ports[1], "web port was changed even though "
"configuration was not")
# WebStatus is no longer a ComparableMixin, so it will be
# rebuilt on each reconfig
#d.addCallback(_check2)
d.addCallback(lambda res: self.buildmaster.loadConfig(webCfg2))
# changes port to 9981
def _check3(p):
ports = self.checkPorts(self.buildmaster,
[(9999, pb.PBServerFactory), (9981, Site)])
self.failIf(
self.p is ports[1],
"configuration was changed but web port was unchanged")
d.addCallback(_check3)
d.addCallback(lambda res: self.buildmaster.loadConfig(webCfg3))
# make 9981 on only localhost
def _check4(p):
ports = self.checkPorts(self.buildmaster,
[(9999, pb.PBServerFactory), (9981, Site)])
self.failUnlessEqual(ports[1].kwargs['interface'], "127.0.0.1")
d.addCallback(_check4)
d.addCallback(lambda res: self.buildmaster.loadConfig(emptyCfg))
d.addCallback(lambda res: self.checkPorts(self.buildmaster, [(
9999, pb.PBServerFactory)]))
return d
def testWebPathname(self):
master = self.buildmaster
master.loadChanges()
d = master.loadConfig(webNameCfg1)
def _check1(res):
self.checkPorts(self.buildmaster,
[(9999, pb.PBServerFactory),
('~/.twistd-web-pb', pb.PBServerFactory)])
unixports = self.UNIXports(self.buildmaster)
self.f = f = unixports[0].args[1]
self.failUnless(isinstance(f.root, ResourcePublisher))
d.addCallback(_check1)
d.addCallback(lambda res: self.buildmaster.loadConfig(webNameCfg1))
# nothing should be changed
def _check2(res):
self.checkPorts(self.buildmaster,
[(9999, pb.PBServerFactory),
('~/.twistd-web-pb', pb.PBServerFactory)])
newf = self.UNIXports(self.buildmaster)[0].args[1]
self.failUnlessIdentical(
self.f, newf, "web factory was changed even though "
"configuration was not")
# WebStatus is no longer a ComparableMixin, so it will be
# rebuilt on each reconfig
#d.addCallback(_check2)
d.addCallback(lambda res: self.buildmaster.loadConfig(webNameCfg2))
def _check3(res):
self.checkPorts(self.buildmaster,
[(9999, pb.PBServerFactory),
('./bar.socket', pb.PBServerFactory)])
newf = self.UNIXports(self.buildmaster)[0].args[1],
self.failIf(
self.f is newf, "web factory was unchanged but "
"configuration was changed")
d.addCallback(_check3)
d.addCallback(lambda res: self.buildmaster.loadConfig(emptyCfg))
d.addCallback(lambda res: self.checkPorts(self.buildmaster, [(
9999, pb.PBServerFactory)]))
return d
def testDebugPassword(self):
master = self.buildmaster
master.loadConfig(debugPasswordCfg)
self.failUnlessEqual(master.checker.users, {
"change": "changepw",
"debug": "sekrit"
})
master.loadConfig(debugPasswordCfg)
self.failUnlessEqual(master.checker.users, {
"change": "changepw",
"debug": "sekrit"
})
master.loadConfig(emptyCfg)
self.failUnlessEqual(master.checker.users, {"change": "changepw"})
def testLocks(self):
master = self.buildmaster
botmaster = master.botmaster
# make sure that c['interlocks'] is rejected properly
self.failUnlessRaises(KeyError, master.loadConfig, interlockCfgBad)
# and that duplicate-named Locks are caught
self.failUnlessRaises(ValueError, master.loadConfig, lockCfgBad1)
self.failUnlessRaises(ValueError, master.loadConfig, lockCfgBad2)
self.failUnlessRaises(ValueError, master.loadConfig, lockCfgBad3)
# create a Builder that uses Locks
master.loadConfig(lockCfg1a)
b1 = master.botmaster.builders["builder1"]
self.failUnlessEqual(len(b1.locks), 2)
# reloading the same config should not change the Builder
master.loadConfig(lockCfg1a)
self.failUnlessIdentical(b1, master.botmaster.builders["builder1"])
# but changing the set of locks used should change it
master.loadConfig(lockCfg1b)
self.failIfIdentical(b1, master.botmaster.builders["builder1"])
b1 = master.botmaster.builders["builder1"]
self.failUnlessEqual(len(b1.locks), 1)
# similar test with step-scoped locks
master.loadConfig(lockCfg2a)
b1 = master.botmaster.builders["builder1"]
# reloading the same config should not change the Builder
master.loadConfig(lockCfg2a)
self.failUnlessIdentical(b1, master.botmaster.builders["builder1"])
# but changing the set of locks used should change it
master.loadConfig(lockCfg2b)
self.failIfIdentical(b1, master.botmaster.builders["builder1"])
b1 = master.botmaster.builders["builder1"]
# remove the locks entirely
master.loadConfig(lockCfg2c)
self.failIfIdentical(b1, master.botmaster.builders["builder1"])
def testNoChangeHorizon(self):
master = self.buildmaster
master.loadChanges()
sourcesCfg = emptyCfg + \
"""
from buildbot.changes.pb import PBChangeSource
c['change_source'] = PBChangeSource()
"""
d = master.loadConfig(sourcesCfg)
def _check1(res):
self.failUnlessEqual(len(list(self.buildmaster.change_svc)), 1)
self.failUnlessEqual(self.buildmaster.change_svc.changeHorizon, 0)
d.addCallback(_check1)
return d
def testChangeHorizon(self):
master = self.buildmaster
master.loadChanges()
sourcesCfg = emptyCfg + \
"""
from buildbot.changes.pb import PBChangeSource
c['change_source'] = PBChangeSource()
c['changeHorizon'] = 5
"""
d = master.loadConfig(sourcesCfg)
def _check1(res):
self.failUnlessEqual(len(list(self.buildmaster.change_svc)), 1)
self.failUnlessEqual(self.buildmaster.change_svc.changeHorizon, 5)
d.addCallback(_check1)
return d
def setUp(self):
# this class generates several deprecation warnings, which the user
# doesn't need to see.
warnings.simplefilter('ignore', exceptions.DeprecationWarning)
self.buildmaster = BuildMaster(".")
def __init__(self, config_dict):
self.config_dict = config_dict
def loadConfig(self):
return MasterConfig.loadFromDict(self.config_dict, '<dict>')
@defer.inlineCallbacks
def getMaster(case, reactor, config_dict):
"""
Create a started ``BuildMaster`` with the given configuration.
"""
basedir = FilePath(case.mktemp())
basedir.createDirectory()
config_dict['buildbotNetUsageData'] = None
master = BuildMaster(
basedir.path, reactor=reactor, config_loader=DictLoader(config_dict))
if 'db_url' not in config_dict:
config_dict['db_url'] = 'sqlite://'
# TODO: Allow BuildMaster to transparently upgrade the database, at least
# for tests.
master.config.db['db_url'] = config_dict['db_url']
yield master.db.setup(check_version=False)
yield master.db.model.upgrade()
master.db.setup = lambda: None
yield master.startService()
case.addCleanup(master.stopService)
return master
'favicon.ico':
util.sibpath(__file__, "../status/web/files/favicon.ico"),
})
m.populate_if_missing(os.path.join(basedir, "master.cfg.sample"),
util.sibpath(__file__, "sample.cfg"),
overwrite=True)
# if index.html exists, use it to override the root page tempalte
m.move_if_present(os.path.join(basedir, "public_html/index.html"),
os.path.join(basedir, "templates/root.html"))
from buildbot.db import connector
from buildbot.master import BuildMaster
if not config['quiet']:
print "upgrading database (%s)" % (master_cfg.db['db_url'])
master = BuildMaster(config['basedir'])
master.config = master_cfg
db = connector.DBConnector(master, basedir=config['basedir'])
wfd = defer.waitForDeferred(
db.setup(check_version=False, verbose=not config['quiet']))
yield wfd
wfd.getResult()
wfd = defer.waitForDeferred(db.model.upgrade())
yield wfd
wfd.getResult()
if not config['quiet']: print "upgrade complete"
yield 0
class ConfigTest(unittest.TestCase):
def setUp(self):
# this class generates several deprecation warnings, which the user
# doesn't need to see.
warnings.simplefilter('ignore', exceptions.DeprecationWarning)
self.buildmaster = BuildMaster(".")
def failUnlessListsEquivalent(self, list1, list2):
l1 = list1[:]
l1.sort()
l2 = list2[:]
l2.sort()
self.failUnlessEqual(l1, l2)
def servers(self, s, types):
# perform a recursive search of s.services, looking for instances of
# twisted.application.internet.TCPServer, then extract their .args
# values to find the TCP ports they want to listen on
for child in s:
if service.IServiceCollection.providedBy(child):
for gc in self.servers(child, types):
yield gc
if isinstance(child, types):
yield child
def TCPports(self, s):
return list(self.servers(s, internet.TCPServer))
def UNIXports(self, s):
return list(self.servers(s, internet.UNIXServer))
def TCPclients(self, s):
return list(self.servers(s, internet.TCPClient))
def checkPorts(self, svc, expected):
"""Verify that the TCPServer and UNIXServer children of the given
service have the expected portnum/pathname and factory classes. As a
side-effect, return a list of servers in the same order as the
'expected' list. This can be used to verify properties of the
factories contained therein."""
expTCP = [e for e in expected if type(e[0]) == int]
expUNIX = [e for e in expected if type(e[0]) == str]
haveTCP = [(p.args[0], p.args[1].__class__)
for p in self.TCPports(svc)]
haveUNIX = [(p.args[0], p.args[1].__class__)
for p in self.UNIXports(svc)]
self.failUnlessListsEquivalent(expTCP, haveTCP)
self.failUnlessListsEquivalent(expUNIX, haveUNIX)
ret = []
for e in expected:
for have in self.TCPports(svc) + self.UNIXports(svc):
if have.args[0] == e[0]:
ret.append(have)
continue
assert(len(ret) == len(expected))
return ret
def testEmpty(self):
self.failUnlessRaises(KeyError, self.buildmaster.loadConfig, "")
def testSimple(self):
# covers slavePortnum, base checker passwords
master = self.buildmaster
master.loadChanges()
master.loadConfig(emptyCfg)
# note: this doesn't actually start listening, because the app
# hasn't been started running
self.failUnlessEqual(master.slavePortnum, "tcp:9999")
self.checkPorts(master, [(9999, pb.PBServerFactory)])
self.failUnlessEqual(list(master.change_svc), [])
self.failUnlessEqual(master.botmaster.builders, {})
self.failUnlessEqual(master.checker.users,
{"change": "changepw"})
self.failUnlessEqual(master.projectName, "dummy project")
self.failUnlessEqual(master.projectURL, "http://dummy.example.com")
self.failUnlessEqual(master.buildbotURL,
"http://dummy.example.com/buildbot")
def testSlavePortnum(self):
master = self.buildmaster
master.loadChanges()
master.loadConfig(emptyCfg)
self.failUnlessEqual(master.slavePortnum, "tcp:9999")
ports = self.checkPorts(master, [(9999, pb.PBServerFactory)])
p = ports[0]
master.loadConfig(emptyCfg)
self.failUnlessEqual(master.slavePortnum, "tcp:9999")
ports = self.checkPorts(master, [(9999, pb.PBServerFactory)])
self.failUnlessIdentical(p, ports[0],
"the slave port was changed even " + \
"though the configuration was not")
master.loadConfig(emptyCfg + "c['slavePortnum'] = 9000\n")
self.failUnlessEqual(master.slavePortnum, "tcp:9000")
ports = self.checkPorts(master, [(9000, pb.PBServerFactory)])
self.failIf(p is ports[0],
"slave port was unchanged but configuration was changed")
def testSlaves(self):
master = self.buildmaster
master.loadChanges()
master.loadConfig(emptyCfg)
self.failUnlessEqual(master.botmaster.builders, {})
self.failUnlessEqual(master.checker.users,
{"change": "changepw"})
# 'botsCfg' is testing backwards compatibility, for 0.7.5 config
# files that have not yet been updated to 0.7.6 . This compatibility
# (and this test) is scheduled for removal in 0.8.0 .
botsCfg = (emptyCfg +
"c['bots'] = [('bot1', 'pw1'), ('bot2', 'pw2')]\n")
master.loadConfig(botsCfg)
self.failUnlessEqual(master.checker.users,
{"change": "changepw",
"bot1": "pw1",
"bot2": "pw2"})
master.loadConfig(botsCfg)
self.failUnlessEqual(master.checker.users,
{"change": "changepw",
"bot1": "pw1",
"bot2": "pw2"})
master.loadConfig(emptyCfg)
self.failUnlessEqual(master.checker.users,
{"change": "changepw"})
slavesCfg = (emptyCfg +
"from buildbot.buildslave import BuildSlave\n"
"c['slaves'] = [BuildSlave('bot1','pw1'), "
"BuildSlave('bot2','pw2')]\n")
master.loadConfig(slavesCfg)
self.failUnlessEqual(master.checker.users,
{"change": "changepw",
"bot1": "pw1",
"bot2": "pw2"})
def testChangeSource(self):
master = self.buildmaster
master.loadChanges()
master.loadConfig(emptyCfg)
self.failUnlessEqual(list(master.change_svc), [])
sourcesCfg = emptyCfg + \
"""
from buildbot.changes.pb import PBChangeSource
c['change_source'] = PBChangeSource()
"""
d = master.loadConfig(sourcesCfg)
def _check1(res):
self.failUnlessEqual(len(list(self.buildmaster.change_svc)), 1)
s1 = list(self.buildmaster.change_svc)[0]
self.failUnless(isinstance(s1, PBChangeSource))
self.failUnlessEqual(s1, list(self.buildmaster.change_svc)[0])
self.failUnless(s1.parent)
# verify that unchanged sources are not interrupted
d1 = self.buildmaster.loadConfig(sourcesCfg)
def _check2(res):
self.failUnlessEqual(len(list(self.buildmaster.change_svc)), 1)
s2 = list(self.buildmaster.change_svc)[0]
self.failUnlessIdentical(s1, s2)
self.failUnless(s1.parent)
d1.addCallback(_check2)
return d1
d.addCallback(_check1)
# make sure we can get rid of the sources too
d.addCallback(lambda res: self.buildmaster.loadConfig(emptyCfg))
def _check3(res):
self.failUnlessEqual(list(self.buildmaster.change_svc), [])
d.addCallback(_check3)
return d
def testChangeSources(self):
# make sure we can accept a list
master = self.buildmaster
master.loadChanges()
master.loadConfig(emptyCfg)
self.failUnlessEqual(list(master.change_svc), [])
sourcesCfg = emptyCfg + \
"""
from buildbot.changes.pb import PBChangeSource
from buildbot.changes.mail import SyncmailMaildirSource
c['change_source'] = [PBChangeSource(),
SyncmailMaildirSource('.'),
]
"""
d = master.loadConfig(sourcesCfg)
def _check1(res):
self.failUnlessEqual(len(list(self.buildmaster.change_svc)), 2)
s1,s2 = list(self.buildmaster.change_svc)
if isinstance(s2, PBChangeSource):
s1,s2 = s2,s1
self.failUnless(isinstance(s1, PBChangeSource))
self.failUnless(s1.parent)
self.failUnless(isinstance(s2, SyncmailMaildirSource))
self.failUnless(s2.parent)
d.addCallback(_check1)
return d
def testSources(self):
# test backwards compatibility. c['sources'] is deprecated.
master = self.buildmaster
master.loadChanges()
master.loadConfig(emptyCfg)
self.failUnlessEqual(list(master.change_svc), [])
sourcesCfg = emptyCfg + \
"""
from buildbot.changes.pb import PBChangeSource
c['sources'] = [PBChangeSource()]
"""
d = master.loadConfig(sourcesCfg)
def _check1(res):
self.failUnlessEqual(len(list(self.buildmaster.change_svc)), 1)
s1 = list(self.buildmaster.change_svc)[0]
self.failUnless(isinstance(s1, PBChangeSource))
self.failUnless(s1.parent)
d.addCallback(_check1)
return d
def shouldBeFailure(self, res, *expected):
self.failUnless(isinstance(res, failure.Failure),
"we expected this to fail, not produce %s" % (res,))
res.trap(*expected)
return None # all is good
def testSchedulerErrors(self):
master = self.buildmaster
master.loadChanges()
master.loadConfig(emptyCfg)
self.failUnlessEqual(master.allSchedulers(), [])
def _shouldBeFailure(res, hint=None):
self.shouldBeFailure(res, AssertionError, ValueError)
if hint:
self.failUnless(str(res).find(hint) != -1)
def _loadConfig(res, newcfg):
return self.buildmaster.loadConfig(newcfg)
d = defer.succeed(None)
# c['schedulers'] must be a list
badcfg = schedulersCfg + \
"""
c['schedulers'] = Scheduler('full', None, 60, ['builder1'])
"""
d.addCallback(_loadConfig, badcfg)
d.addBoth(_shouldBeFailure,
"c['schedulers'] must be a list of Scheduler instances")
# c['schedulers'] must be a list of IScheduler objects
badcfg = schedulersCfg + \
"""
c['schedulers'] = ['oops', 'problem']
"""
d.addCallback(_loadConfig, badcfg)
d.addBoth(_shouldBeFailure,
"c['schedulers'] must be a list of Scheduler instances")
# c['schedulers'] must point at real builders
badcfg = schedulersCfg + \
"""
c['schedulers'] = [Scheduler('full', None, 60, ['builder-bogus'])]
"""
d.addCallback(_loadConfig, badcfg)
d.addBoth(_shouldBeFailure, "uses unknown builder")
# builderNames= must be a list
badcfg = schedulersCfg + \
"""
c['schedulers'] = [Scheduler('full', None, 60, 'builder1')]
"""
d.addCallback(_loadConfig, badcfg)
d.addBoth(_shouldBeFailure,
"must be a list of Builder description names")
# builderNames= must be a list of strings, not dicts
badcfg = schedulersCfg + \
"""
c['schedulers'] = [Scheduler('full', None, 60, [b1])]
"""
d.addCallback(_loadConfig, badcfg)
d.addBoth(_shouldBeFailure,
"must be a list of Builder description names")
# builderNames= must be a list of strings, not a dict
badcfg = schedulersCfg + \
"""
c['schedulers'] = [Scheduler('full', None, 60, b1)]
"""
d.addCallback(_loadConfig, badcfg)
d.addBoth(_shouldBeFailure,
"must be a list of Builder description names")
# each Scheduler must have a unique name
badcfg = schedulersCfg + \
"""
c['schedulers'] = [Scheduler('dup', None, 60, []),
Scheduler('dup', None, 60, [])]
"""
d.addCallback(_loadConfig, badcfg)
d.addBoth(_shouldBeFailure, "Schedulers must have unique names")
return d
def testSchedulers(self):
master = self.buildmaster
master.loadChanges()
master.loadConfig(emptyCfg)
self.failUnlessEqual(master.allSchedulers(), [])
d = self.buildmaster.loadConfig(schedulersCfg)
d.addCallback(self._testSchedulers_1)
return d
def _testSchedulers_1(self, res):
sch = self.buildmaster.allSchedulers()
self.failUnlessEqual(len(sch), 1)
s = sch[0]
self.failUnless(isinstance(s, scheduler.Scheduler))
self.failUnlessEqual(s.name, "full")
self.failUnlessEqual(s.branch, None)
self.failUnlessEqual(s.treeStableTimer, 60)
self.failUnlessEqual(s.builderNames, ['builder1'])
newcfg = schedulersCfg + \
"""
s1 = Scheduler('full', None, 60, ['builder1'])
c['schedulers'] = [s1, Dependent('downstream', s1, ['builder1'])]
"""
d = self.buildmaster.loadConfig(newcfg)
d.addCallback(self._testSchedulers_2, newcfg)
return d
def _testSchedulers_2(self, res, newcfg):
sch = self.buildmaster.allSchedulers()
self.failUnlessEqual(len(sch), 2)
s = sch[0]
self.failUnless(isinstance(s, scheduler.Scheduler))
s = sch[1]
self.failUnless(isinstance(s, scheduler.Dependent))
self.failUnlessEqual(s.name, "downstream")
self.failUnlessEqual(s.builderNames, ['builder1'])
# reloading the same config file should leave the schedulers in place
d = self.buildmaster.loadConfig(newcfg)
d.addCallback(self._testSchedulers_3, sch)
return d
def _testSchedulers_3(self, res, sch1):
sch2 = self.buildmaster.allSchedulers()
self.failUnlessEqual(len(sch2), 2)
sch1.sort()
sch2.sort()
self.failUnlessEqual(sch1, sch2)
self.failUnlessIdentical(sch1[0], sch2[0])
self.failUnlessIdentical(sch1[1], sch2[1])
self.failUnlessIdentical(sch1[0].parent, self.buildmaster)
self.failUnlessIdentical(sch1[1].parent, self.buildmaster)
def testBuilders(self):
master = self.buildmaster
master.loadConfig(emptyCfg)
self.failUnlessEqual(master.botmaster.builders, {})
master.loadConfig(buildersCfg)
self.failUnlessEqual(master.botmaster.builderNames, ["builder1"])
self.failUnlessEqual(master.botmaster.builders.keys(), ["builder1"])
b = master.botmaster.builders["builder1"]
self.failUnless(isinstance(b, Builder))
self.failUnlessEqual(b.name, "builder1")
self.failUnlessEqual(b.slavenames, ["bot1"])
self.failUnlessEqual(b.builddir, "workdir")
f1 = b.buildFactory
self.failUnless(isinstance(f1, BasicBuildFactory))
steps = f1.steps
self.failUnlessEqual(len(steps), 3)
self.failUnlessEqual(steps[0], (CVS,
{'cvsroot': 'cvsroot',
'cvsmodule': 'cvsmodule',
'mode': 'clobber'}))
self.failUnlessEqual(steps[1], (Compile,
{'command': 'make all'}))
self.failUnlessEqual(steps[2], (Test,
{'command': 'make check'}))
# make sure a reload of the same data doesn't interrupt the Builder
master.loadConfig(buildersCfg)
self.failUnlessEqual(master.botmaster.builderNames, ["builder1"])
self.failUnlessEqual(master.botmaster.builders.keys(), ["builder1"])
b2 = master.botmaster.builders["builder1"]
self.failUnlessIdentical(b, b2)
# TODO: test that the BuilderStatus object doesn't change
#statusbag2 = master.client_svc.statusbags["builder1"]
#self.failUnlessIdentical(statusbag, statusbag2)
# but changing something should result in a new Builder
master.loadConfig(buildersCfg2)
self.failUnlessEqual(master.botmaster.builderNames, ["builder1"])
self.failUnlessEqual(master.botmaster.builders.keys(), ["builder1"])
b3 = master.botmaster.builders["builder1"]
self.failIf(b is b3)
# the statusbag remains the same TODO
#statusbag3 = master.client_svc.statusbags["builder1"]
#self.failUnlessIdentical(statusbag, statusbag3)
# adding new builder
master.loadConfig(buildersCfg3)
self.failUnlessEqual(master.botmaster.builderNames, ["builder1",
"builder2"])
self.failUnlessListsEquivalent(master.botmaster.builders.keys(),
["builder1", "builder2"])
b4 = master.botmaster.builders["builder1"]
self.failUnlessIdentical(b3, b4)
# changing first builder should leave it at the same place in the list
master.loadConfig(buildersCfg4)
self.failUnlessEqual(master.botmaster.builderNames, ["builder1",
"builder2"])
self.failUnlessListsEquivalent(master.botmaster.builders.keys(),
["builder1", "builder2"])
b5 = master.botmaster.builders["builder1"]
self.failIf(b4 is b5)
master.loadConfig(buildersCfg5)
self.failUnlessEqual(master.botmaster.builderNames, ["builder1",
"builder2"])
self.failUnlessListsEquivalent(master.botmaster.builders.keys(),
["builder1", "builder2"])
b5 = master.botmaster.builders["builder1"]
# and removing it should make the Builder go away
master.loadConfig(emptyCfg)
self.failUnlessEqual(master.botmaster.builderNames, [])
self.failUnlessEqual(master.botmaster.builders, {})
#self.failUnlessEqual(master.client_svc.statusbags, {}) # TODO
def testWithProperties(self):
master = self.buildmaster
master.loadConfig(wpCfg1)
self.failUnlessEqual(master.botmaster.builderNames, ["builder1"])
self.failUnlessEqual(master.botmaster.builders.keys(), ["builder1"])
b1 = master.botmaster.builders["builder1"]
# reloading the same config should leave the builder unchanged
master.loadConfig(wpCfg1)
b2 = master.botmaster.builders["builder1"]
self.failUnlessIdentical(b1, b2)
# but changing the parameters of the WithProperties should change it
master.loadConfig(wpCfg2)
b3 = master.botmaster.builders["builder1"]
self.failIf(b1 is b3)
# again, reloading same config should leave the builder unchanged
master.loadConfig(wpCfg2)
b4 = master.botmaster.builders["builder1"]
self.failUnlessIdentical(b3, b4)
def checkIRC(self, m, expected):
ircs = {}
for irc in self.servers(m, words.IRC):
ircs[irc.host] = (irc.nick, irc.channels)
self.failUnlessEqual(ircs, expected)
def testIRC(self):
if not words:
raise unittest.SkipTest("Twisted Words package is not installed")
master = self.buildmaster
master.loadChanges()
d = master.loadConfig(emptyCfg)
e1 = {}
d.addCallback(lambda res: self.checkIRC(master, e1))
d.addCallback(lambda res: master.loadConfig(ircCfg1))
e2 = {'irc.us.freenode.net': ('buildbot', ['twisted'])}
d.addCallback(lambda res: self.checkIRC(master, e2))
d.addCallback(lambda res: master.loadConfig(ircCfg2))
e3 = {'irc.us.freenode.net': ('buildbot', ['twisted']),
'irc.example.com': ('otherbot', ['chan1', 'chan2'])}
d.addCallback(lambda res: self.checkIRC(master, e3))
d.addCallback(lambda res: master.loadConfig(ircCfg3))
e4 = {'irc.us.freenode.net': ('buildbot', ['knotted'])}
d.addCallback(lambda res: self.checkIRC(master, e4))
d.addCallback(lambda res: master.loadConfig(ircCfg1))
e5 = {'irc.us.freenode.net': ('buildbot', ['twisted'])}
d.addCallback(lambda res: self.checkIRC(master, e5))
return d
def testWebPortnum(self):
master = self.buildmaster
master.loadChanges()
d = master.loadConfig(webCfg1)
def _check1(res):
ports = self.checkPorts(self.buildmaster,
[(9999, pb.PBServerFactory), (9980, Site)])
p = ports[1]
self.p = p
# nothing should be changed
d.addCallback(_check1)
d.addCallback(lambda res: self.buildmaster.loadConfig(webCfg1))
def _check2(res):
ports = self.checkPorts(self.buildmaster,
[(9999, pb.PBServerFactory), (9980, Site)])
self.failUnlessIdentical(self.p, ports[1],
"web port was changed even though "
"configuration was not")
# WebStatus is no longer a ComparableMixin, so it will be
# rebuilt on each reconfig
#d.addCallback(_check2)
d.addCallback(lambda res: self.buildmaster.loadConfig(webCfg2))
# changes port to 9981
def _check3(p):
ports = self.checkPorts(self.buildmaster,
[(9999, pb.PBServerFactory), (9981, Site)])
self.failIf(self.p is ports[1],
"configuration was changed but web port was unchanged")
d.addCallback(_check3)
d.addCallback(lambda res: self.buildmaster.loadConfig(webCfg3))
# make 9981 on only localhost
def _check4(p):
ports = self.checkPorts(self.buildmaster,
[(9999, pb.PBServerFactory), (9981, Site)])
self.failUnlessEqual(ports[1].kwargs['interface'], "127.0.0.1")
d.addCallback(_check4)
d.addCallback(lambda res: self.buildmaster.loadConfig(emptyCfg))
d.addCallback(lambda res:
self.checkPorts(self.buildmaster,
[(9999, pb.PBServerFactory)]))
return d
def testWebPathname(self):
master = self.buildmaster
master.loadChanges()
d = master.loadConfig(webNameCfg1)
def _check1(res):
self.checkPorts(self.buildmaster,
[(9999, pb.PBServerFactory),
('~/.twistd-web-pb', pb.PBServerFactory)])
unixports = self.UNIXports(self.buildmaster)
self.f = f = unixports[0].args[1]
self.failUnless(isinstance(f.root, ResourcePublisher))
d.addCallback(_check1)
d.addCallback(lambda res: self.buildmaster.loadConfig(webNameCfg1))
# nothing should be changed
def _check2(res):
self.checkPorts(self.buildmaster,
[(9999, pb.PBServerFactory),
('~/.twistd-web-pb', pb.PBServerFactory)])
newf = self.UNIXports(self.buildmaster)[0].args[1]
self.failUnlessIdentical(self.f, newf,
"web factory was changed even though "
"configuration was not")
# WebStatus is no longer a ComparableMixin, so it will be
# rebuilt on each reconfig
#d.addCallback(_check2)
d.addCallback(lambda res: self.buildmaster.loadConfig(webNameCfg2))
def _check3(res):
self.checkPorts(self.buildmaster,
[(9999, pb.PBServerFactory),
('./bar.socket', pb.PBServerFactory)])
newf = self.UNIXports(self.buildmaster)[0].args[1],
self.failIf(self.f is newf,
"web factory was unchanged but "
"configuration was changed")
d.addCallback(_check3)
d.addCallback(lambda res: self.buildmaster.loadConfig(emptyCfg))
d.addCallback(lambda res:
self.checkPorts(self.buildmaster,
[(9999, pb.PBServerFactory)]))
return d
def testDebugPassword(self):
master = self.buildmaster
master.loadConfig(debugPasswordCfg)
self.failUnlessEqual(master.checker.users,
{"change": "changepw",
"debug": "sekrit"})
master.loadConfig(debugPasswordCfg)
self.failUnlessEqual(master.checker.users,
{"change": "changepw",
"debug": "sekrit"})
master.loadConfig(emptyCfg)
self.failUnlessEqual(master.checker.users,
{"change": "changepw"})
def testLocks(self):
master = self.buildmaster
botmaster = master.botmaster
# make sure that c['interlocks'] is rejected properly
self.failUnlessRaises(KeyError, master.loadConfig, interlockCfgBad)
# and that duplicate-named Locks are caught
self.failUnlessRaises(ValueError, master.loadConfig, lockCfgBad1)
self.failUnlessRaises(ValueError, master.loadConfig, lockCfgBad2)
self.failUnlessRaises(ValueError, master.loadConfig, lockCfgBad3)
# create a Builder that uses Locks
master.loadConfig(lockCfg1a)
b1 = master.botmaster.builders["builder1"]
self.failUnlessEqual(len(b1.locks), 2)
# reloading the same config should not change the Builder
master.loadConfig(lockCfg1a)
self.failUnlessIdentical(b1, master.botmaster.builders["builder1"])
# but changing the set of locks used should change it
master.loadConfig(lockCfg1b)
self.failIfIdentical(b1, master.botmaster.builders["builder1"])
b1 = master.botmaster.builders["builder1"]
self.failUnlessEqual(len(b1.locks), 1)
# similar test with step-scoped locks
master.loadConfig(lockCfg2a)
b1 = master.botmaster.builders["builder1"]
# reloading the same config should not change the Builder
master.loadConfig(lockCfg2a)
self.failUnlessIdentical(b1, master.botmaster.builders["builder1"])
# but changing the set of locks used should change it
master.loadConfig(lockCfg2b)
self.failIfIdentical(b1, master.botmaster.builders["builder1"])
b1 = master.botmaster.builders["builder1"]
# remove the locks entirely
master.loadConfig(lockCfg2c)
self.failIfIdentical(b1, master.botmaster.builders["builder1"])
def testNoChangeHorizon(self):
master = self.buildmaster
master.loadChanges()
sourcesCfg = emptyCfg + \
"""
from buildbot.changes.pb import PBChangeSource
c['change_source'] = PBChangeSource()
"""
d = master.loadConfig(sourcesCfg)
def _check1(res):
self.failUnlessEqual(len(list(self.buildmaster.change_svc)), 1)
self.failUnlessEqual(self.buildmaster.change_svc.changeHorizon, 0)
d.addCallback(_check1)
return d
def testChangeHorizon(self):
master = self.buildmaster
master.loadChanges()
sourcesCfg = emptyCfg + \
"""
from buildbot.changes.pb import PBChangeSource
c['change_source'] = PBChangeSource()
c['changeHorizon'] = 5
"""
d = master.loadConfig(sourcesCfg)
def _check1(res):
self.failUnlessEqual(len(list(self.buildmaster.change_svc)), 1)
self.failUnlessEqual(self.buildmaster.change_svc.changeHorizon, 5)
d.addCallback(_check1)
return d
def setUp(self):
# this class generates several deprecation warnings, which the user
# doesn't need to see.
warnings.simplefilter('ignore', exceptions.DeprecationWarning)
self.buildmaster = BuildMaster(".")
basedir = os.path.abspath(os.path.dirname(__file__))
configfile = 'master.cfg'
# Default umask for server
umask = None
# note: this line is matched against to check that this is a buildmaster
# directory; do not edit it.
application = service.Application('buildmaster')
import sys
from twisted.python.log import ILogObserver, FileLogObserver
application.setComponent(ILogObserver, FileLogObserver(sys.stdout).emit)
m = BuildMaster(basedir, configfile, umask)
m.setServiceParent(application)
# and slave on the same process!
buildmaster_host = 'localhost'
port = 19989
slavename = 'example-slave'
passwd = 'pass'
keepalive = 600
usepty = 0
umask = None
maxdelay = 300
allow_shutdown = None
slavedir = os.path.join(basedir, "slave")
if not os.path.exists(slavedir):
def setupConfig(self, config_dict, startWorker=True):
"""
Setup and start a master configured
by the function configFunc defined in the test module.
@type config_dict: dict
@param configFunc: The BuildmasterConfig dictionary.
"""
self.basedir = os.path.abspath('basdir')
self.setUpDirs(self.basedir)
self.addCleanup(self.tearDownDirs)
# mock reactor.stop (which trial *really* doesn't
# like test code to call!)
stop = mock.create_autospec(reactor.stop)
self.patch(reactor, 'stop', stop)
if startWorker:
if self.proto == 'pb':
proto = {"pb": {"port": "tcp:0:interface=127.0.0.1"}}
workerclass = worker.Worker
elif self.proto == 'null':
proto = {"null": {}}
workerclass = worker.LocalWorker
config_dict['workers'] = [workerclass("local1", "localpw")]
config_dict['protocols'] = proto
# create the master and set its config
m = BuildMaster(self.basedir, reactor=reactor, config_loader=DictLoader(config_dict))
self.master = m
# update the DB
yield m.db.setup(check_version=False)
yield m.db.model.upgrade()
# stub out m.db.setup since it was already called above
m.db.setup = lambda: None
# start the service
yield m.startService()
self.failIf(stop.called,
"startService tried to stop the reactor; check logs")
# and shutdown the db threadpool, as is normally done at reactor stop
self.addCleanup(m.db.pool.shutdown)
self.addCleanup(m.stopService)
if not startWorker:
return
if self.proto == 'pb':
# We find out the worker port automatically
workerPort = list(itervalues(m.pbmanager.dispatchers))[
0].port.getHost().port
# create a worker, and attach it to the master, it will be started, and stopped
# along with the master
self.w = BuildSlave(
"127.0.0.1", workerPort, "local1", "localpw", self.basedir, False, False)
elif self.proto == 'null':
self.w = None
if self.w is not None:
self.w.startService()
self.addCleanup(self.w.stopService)
@defer.inlineCallbacks
def dump():
if not self._passed:
dump = StringIO.StringIO()
print("FAILED! dumping build db for debug", file=dump)
builds = yield self.master.data.get(("builds",))
for build in builds:
yield self.printBuild(build, dump, withLogs=True)
raise self.failureException(dump.getvalue())
self.addCleanup(dump)
def setupConfig(self, config_dict, startWorker=True):
"""
Setup and start a master configured
by the function configFunc defined in the test module.
@type config_dict: dict
@param configFunc: The BuildmasterConfig dictionary.
"""
self.basedir = os.path.abspath('basdir')
self.setUpDirs(self.basedir)
self.addCleanup(self.tearDownDirs)
# mock reactor.stop (which trial *really* doesn't
# like test code to call!)
stop = mock.create_autospec(reactor.stop)
self.patch(reactor, 'stop', stop)
if startWorker:
if self.proto == 'pb':
proto = {"pb": {"port": "tcp:0:interface=127.0.0.1"}}
workerclass = worker.Worker
elif self.proto == 'null':
proto = {"null": {}}
workerclass = worker.LocalWorker
config_dict['workers'] = [workerclass("local1", "localpw")]
config_dict['protocols'] = proto
# create the master and set its config
m = BuildMaster(self.basedir,
reactor=reactor,
config_loader=DictLoader(config_dict))
self.master = m
# update the DB
yield m.db.setup(check_version=False)
yield m.db.model.upgrade()
# stub out m.db.setup since it was already called above
m.db.setup = lambda: None
# start the service
yield m.startService()
self.failIf(stop.called,
"startService tried to stop the reactor; check logs")
# and shutdown the db threadpool, as is normally done at reactor stop
self.addCleanup(m.db.pool.shutdown)
self.addCleanup(m.stopService)
if not startWorker:
return
if self.proto == 'pb':
# We find out the worker port automatically
workerPort = list(itervalues(
m.pbmanager.dispatchers))[0].port.getHost().port
# create a worker, and attach it to the master, it will be started, and stopped
# along with the master
self.w = BuildSlave("127.0.0.1", workerPort, "local1", "localpw",
self.basedir, False, False)
elif self.proto == 'null':
self.w = None
if self.w is not None:
self.w.startService()
self.addCleanup(self.w.stopService)
@defer.inlineCallbacks
def dump():
if not self._passed:
dump = StringIO.StringIO()
print("FAILED! dumping build db for debug", file=dump)
builds = yield self.master.data.get(("builds", ))
for build in builds:
yield self.printBuild(build, dump, withLogs=True)
raise self.failureException(dump.getvalue())
self.addCleanup(dump)
def get_master(basedir):
cfg = cm.MasterConfig.loadConfig(basedir, 'master.cfg')
master = BuildMaster(basedir)
master.config = cfg
return master
