def testFindConfigFile(self):
os.mkdir("test_cf")
open(os.path.join("test_cf", "master.cfg"), "w").write(emptyCfg)
slaveportCfg = emptyCfg + "c['slavePortnum'] = 9000\n"
open(os.path.join("test_cf", "alternate.cfg"), "w").write(slaveportCfg)
m = BuildMaster("test_cf")
m.loadTheConfigFile()
self.failUnlessEqual(m.slavePortnum, "tcp:9999")
m = BuildMaster("test_cf", "alternate.cfg")
m.loadTheConfigFile()
self.failUnlessEqual(m.slavePortnum, "tcp:9000")
def upgradeDatabase(config, master_cfg):
if not config['quiet']:
print("upgrading database (%s)" %
(stripUrlPassword(master_cfg.db['db_url'])))
print("Warning: Stopping this process might cause data loss")
def sighandler(signum, frame):
msg = " ".join("""
WARNING: ignoring signal %s.
This process should not be interrupted to avoid database corruption.
If you really need to terminate it, use SIGKILL.
""".split())
print(msg % signum)
for signame in ("SIGTERM", "SIGINT", "SIGQUIT", "SIGHUP", "SIGUSR1",
"SIGUSR2", "SIGBREAK"):
if hasattr(signal, signame):
signal.signal(getattr(signal, signame), sighandler)
master = BuildMaster(config['basedir'])
master.config = master_cfg
master.db.disownServiceParent()
db = connector.DBConnector(basedir=config['basedir'])
db.setServiceParent(master)
yield db.setup(check_version=False, verbose=not config['quiet'])
yield db.model.upgrade()
yield db.masters.setAllMastersActiveLongTimeAgo()
def do_test_master(self):
# create the master and set its config
m = BuildMaster(self.basedir, self.configfile)
# update the DB
yield m.db.setup(check_version=False)
yield m.db.model.upgrade()
# stub out m.db.setup since it was already called above
m.db.setup = lambda: None
# mock reactor.stop (which trial *really* doesn't
# like test code to call!)
mock_reactor = mock.Mock(spec=reactor)
mock_reactor.callWhenRunning = reactor.callWhenRunning
mock_reactor.getThreadPool = reactor.getThreadPool
mock_reactor.callFromThread = reactor.callFromThread
# start the service
yield m.startService(_reactor=mock_reactor)
self.failIf(mock_reactor.stop.called,
"startService tried to stop the reactor; check logs")
# hang out for a fraction of a second, to let startup processes run
d = defer.Deferred()
reactor.callLater(0.01, d.callback, None)
yield d
# stop the service
yield m.stopService()
# and shutdown the db threadpool, as is normally done at reactor stop
m.db.pool.shutdown()
def setupConfig(self, configFunc):
"""
Setup and start a master configured
by the function configFunc defined in the test module.
@type configFunc: string
@param configFunc: name of a function
without argument defined in the test module
that returns a BuildmasterConfig object.
"""
self.basedir = os.path.abspath('basdir')
self.setUpDirs(self.basedir)
self.configfile = os.path.join(self.basedir, 'master.cfg')
if self.proto == 'pb':
proto = '{"pb": {"port": "tcp:0:interface=127.0.0.1"}}'
elif self.proto == 'null':
proto = '{"null": {}}'
# We create a master.cfg, which loads the configuration from the
# test module. Only the slave config is kept there, as it should not
# be changed
open(self.configfile, "w").write(
textwrap.dedent("""
from buildbot.buildslave import BuildSlave
from %s import %s
c = BuildmasterConfig = %s()
c['slaves'] = [BuildSlave("local1", "localpw")]
c['protocols'] = %s
""" % (self.__class__.__module__, configFunc, configFunc, proto)))
# create the master and set its config
m = BuildMaster(self.basedir, self.configfile)
self.master = m
# update the DB
yield m.db.setup(check_version=False)
yield m.db.model.upgrade()
# stub out m.db.setup since it was already called above
m.db.setup = lambda: None
# mock reactor.stop (which trial *really* doesn't
# like test code to call!)
mock_reactor = mock.Mock(spec=reactor)
mock_reactor.callWhenRunning = reactor.callWhenRunning
# start the service
yield m.startService(_reactor=mock_reactor)
self.failIf(mock_reactor.stop.called,
"startService tried to stop the reactor; check logs")
if self.proto == 'pb':
# We find out the slave port automatically
slavePort = list(itervalues(
m.pbmanager.dispatchers))[0].port.getHost().port
# create a slave, and attach it to the master, it will be started, and stopped
# along with the master
s = BuildSlave("127.0.0.1", slavePort, "local1", "localpw",
self.basedir, False, False)
elif self.proto == 'null':
s = LocalBuildSlave("local1", self.basedir, False)
s.setServiceParent(m)
def do_test_master(self):
# create the master and set its config
m = BuildMaster(self.basedir, self.configfile)
m.config = config.MasterConfig.loadConfig(self.basedir,
self.configfile)
# update the DB
yield m.db.setup(check_version=False)
yield m.db.model.upgrade()
# stub out m.db.setup since it was already called above
m.db.setup = lambda: None
# mock reactor.stop (which trial *really* doesn't
# like test code to call!)
mock_reactor = mock.Mock(spec=reactor)
mock_reactor.callWhenRunning = reactor.callWhenRunning
# start the service
yield m.startService(_reactor=mock_reactor)
self.failIf(mock_reactor.stop.called,
"startService tried to stop the reactor; check logs")
# stop the service
yield m.stopService()
# and shutdown the db threadpool, as is normally done at reactor stop
m.db.pool.shutdown()
def doCleanupDatabase(config, master_cfg):
if not config['quiet']:
print("cleaning database (%s)" % (master_cfg.db['db_url']))
master = BuildMaster(config['basedir'])
master.config = master_cfg
print(master.config.logCompressionMethod)
db = master.db
yield db.setup(check_version=False, verbose=not config['quiet'])
res = yield db.logs.getLogs()
i = 0
percent = 0
saved = 0
for log in res:
saved += yield db.logs.compressLog(log['id'])
i += 1
if not config['quiet'] and percent != i * 100 / len(res):
percent = i * 100 / len(res)
print(" {0}% {1} saved".format(percent, saved))
saved = 0
sys.stdout.flush()
if master_cfg.db['db_url'].startswith("sqlite"):
if not config['quiet']:
print("executing sqlite vacuum function...")
# sqlite vacuum function rebuild the whole database to claim
# free disk space back
def thd(engine):
r = engine.execute("vacuum;")
r.close()
yield db.pool.do(thd)
def create_db(self):
from buildbot.db import connector
from buildbot.master import BuildMaster
db = connector.DBConnector(BuildMaster(self.basedir),
self.config['db'], basedir=self.basedir)
if not self.config['quiet']: print "creating database"
d = db.model.upgrade()
return d
def testStartService(self):
os.mkdir("test_ss")
self.master = m = BuildMaster("test_ss")
# inhibit the usual read-config-on-startup behavior
m.readConfig = True
m.startService()
d = m.loadConfig(startableEmptyCfg % 0)
d.addCallback(self._testStartService_0)
return d
def setupConfig(self, config_dict):
"""
Setup and start a master configured
by the function configFunc defined in the test module.
@type config_dict: dict
@param configFunc: The BuildmasterConfig dictionary.
"""
self.basedir = os.path.abspath('basdir')
self.setUpDirs(self.basedir)
# mock reactor.stop (which trial *really* doesn't
# like test code to call!)
mock_reactor = mock.Mock(spec=reactor)
mock_reactor.callWhenRunning = reactor.callWhenRunning
mock_reactor.getThreadPool = reactor.getThreadPool
mock_reactor.callFromThread = reactor.callFromThread
workerclass = worker.Worker
if self.proto == 'pb':
proto = {"pb": {"port": "tcp:0:interface=127.0.0.1"}}
elif self.proto == 'null':
proto = {"null": {}}
workerclass = worker.LocalWorker
config_dict['workers'] = [workerclass("local1", "localpw")]
config_dict['protocols'] = proto
# create the master and set its config
m = BuildMaster(self.basedir,
reactor=mock_reactor,
config_loader=DictLoader(config_dict))
self.master = m
# update the DB
yield m.db.setup(check_version=False)
yield m.db.model.upgrade()
# stub out m.db.setup since it was already called above
m.db.setup = lambda: None
# start the service
yield m.startService()
self.failIf(mock_reactor.stop.called,
"startService tried to stop the reactor; check logs")
if self.proto == 'pb':
# We find out the worker port automatically
workerPort = list(itervalues(
m.pbmanager.dispatchers))[0].port.getHost().port
# create a worker, and attach it to the master, it will be started, and stopped
# along with the master
self.w = BuildSlave("127.0.0.1", workerPort, "local1", "localpw",
self.basedir, False, False)
elif self.proto == 'null':
self.w = None
if self.w is not None:
self.w.setServiceParent(m)
def upgradeDatabase(config, master_cfg):
if not config['quiet']:
print "upgrading database (%s)" % (master_cfg.db['db_url'])
master = BuildMaster(config['basedir'])
master.config = master_cfg
db = connector.DBConnector(master, basedir=config['basedir'])
yield db.setup(check_version=False, verbose=not config['quiet'])
yield db.model.upgrade()
def getMaster(self, config_dict):
"""
Create a started ``BuildMaster`` with the given configuration.
"""
basedir = FilePath(self.mktemp())
basedir.createDirectory()
master = BuildMaster(
basedir.path, reactor=reactor, config_loader=DictLoader(config_dict))
master.config = master.config_loader.loadConfig()
return master
def upgradeDatabase(config, master_cfg):
if not config['quiet']:
print("upgrading database (%s)" % (master_cfg.db['db_url']))
master = BuildMaster(config['basedir'])
master.config = master_cfg
master.db.disownServiceParent()
db = connector.DBConnector(basedir=config['basedir'])
db.setServiceParent(master)
yield db.setup(check_version=False, verbose=not config['quiet'])
yield db.model.upgrade()
yield db.masters.setAllMastersActiveLongTimeAgo()
def check_master_cfg(self):
"""Check the buildmaster configuration, returning a deferred that
fires with an approprate exit status (so 0=success)."""
from buildbot.master import BuildMaster
from twisted.python import log, failure
master_cfg = os.path.join(self.basedir, "master.cfg")
if not os.path.exists(master_cfg):
if not self.quiet:
print "No master.cfg found"
return defer.succeed(1)
# side-effects of loading the config file:
# for each Builder defined in c['builders'], if the status directory
# didn't already exist, it will be created, and the
# $BUILDERNAME/builder pickle might be created (with a single
# "builder created" event).
# we put basedir in front of sys.path, because that's how the
# buildmaster itself will run, and it is quite common to have the
# buildmaster import helper classes from other .py files in its
# basedir.
if sys.path[0] != self.basedir:
sys.path.insert(0, self.basedir)
m = BuildMaster(self.basedir)
# we need to route log.msg to stdout, so any problems can be seen
# there. But if everything goes well, I'd rather not clutter stdout
# with log messages. So instead we add a logObserver which gathers
# messages and only displays them if something goes wrong.
messages = []
log.addObserver(messages.append)
try:
# this will raise an exception if there's something wrong with
# the config file. Note that this BuildMaster instance is never
# started, so it won't actually do anything with the
# configuration.
return m.loadConfig(open(master_cfg, "r"), checkOnly=True)
except:
f = failure.Failure()
if not self.quiet:
print
for m in messages:
print "".join(m['message'])
print f
print
print "An error was detected in the master.cfg file."
print "Please correct the problem and run 'buildbot upgrade-master' again."
print
return 1
return 0
def upgradeDatabase(config, master_cfg):
if not config['quiet']:
print("upgrading database (%s)" %
(stripUrlPassword(master_cfg.db['db_url'])))
print("Warning: Stopping this process might cause data loss")
master = BuildMaster(config['basedir'])
master.config = master_cfg
master.db.disownServiceParent()
db = connector.DBConnector(basedir=config['basedir'])
db.setServiceParent(master)
yield db.setup(check_version=False, verbose=not config['quiet'])
yield db.model.upgrade()
yield db.masters.setAllMastersActiveLongTimeAgo()
def upgradeMaster(config):
m = Maker(config)
if not config['quiet']: print "upgrading basedir"
basedir = os.path.expanduser(config['basedir'])
# TODO: check Makefile
# TODO: check TAC file
# check web files: index.html, default.css, robots.txt
m.upgrade_public_html({
'bg_gradient.jpg':
util.sibpath(__file__, "../status/web/files/bg_gradient.jpg"),
'default.css':
util.sibpath(__file__, "../status/web/files/default.css"),
'robots.txt':
util.sibpath(__file__, "../status/web/files/robots.txt"),
'favicon.ico':
util.sibpath(__file__, "../status/web/files/favicon.ico"),
})
m.populate_if_missing(os.path.join(basedir, "master.cfg.sample"),
util.sibpath(__file__, "sample.cfg"),
overwrite=True)
# if index.html exists, use it to override the root page tempalte
m.move_if_present(os.path.join(basedir, "public_html/index.html"),
os.path.join(basedir, "templates/root.html"))
if not config['quiet']: print "checking master.cfg"
wfd = defer.waitForDeferred(
m.check_master_cfg(expected_db_url=config['db']))
yield wfd
rc = wfd.getResult()
if rc == 0:
from buildbot.db import connector
from buildbot.master import BuildMaster
if not config['quiet']: print "upgrading database"
db = connector.DBConnector(BuildMaster(config['basedir']),
config['db'],
basedir=config['basedir'])
wfd = defer.waitForDeferred(db.model.upgrade())
yield wfd
wfd.getResult()
if not config['quiet']: print "upgrade complete"
yield 0
else:
yield rc
def __init__(self,
config,
reactor=None,
source='TestMaster',
log_handler=None,
attach_on=tuple()):
"""Lightweight in-process BuildMaster
Spins up a lightweight BuildMaster in the same process and can trigger
builders defined in the configuration. The TestMaster only pays
attention to the `workers`, `builders` and `schedulers` configuration
keys, so it doesn't configure non-essential services like the
reporters.
It is used in the CLI interface to locally reproduce specific builds,
but it is also suitable for general integration testing of the
builders.
Parameters
----------
config: MasterConfig
reactor: twisted.reactor, default None
source: str, default `TestMaster`
Used for highligting the origin or the build properties.
log_handler: Callable[[unseen_log_lines], None], default lambda _: None
A callback to handle the logs produced by the builder's buildsteps.
attach_on: List[Results], default []
If a build finishes with any of the listed states and it is
executed withing a DockerLatentWorker then start an interactive
shell session in the container. Use it with caution, because it
blocks the event loop.
"""
assert isinstance(config, MasterConfig)
assert all(result in ALL_RESULTS for result in attach_on)
self.config = config
self.attach_on = set(attach_on)
loader = EagerLoader(config, source=source)
if reactor is None:
from twisted.internet import reactor
self._source = source
self._master = BuildMaster('.', reactor=reactor, config_loader=loader)
self._log_handler = log_handler or (lambda _: None)
# state variable updated by the event handlers below
self._buildset = None
self._buildset_id = None
self._log_offset = 0
def createDB(config, _noMonkey=False):
# apply the db monkeypatches (and others - no harm)
if not _noMonkey: # pragma: no cover
monkeypatches.patch_all()
# create a master with the default configuration, but with db_url
# overridden
master_cfg = config_module.MasterConfig()
master_cfg.db['db_url'] = config['db']
master = BuildMaster(config['basedir'])
master.config = master_cfg
db = master.db
yield db.setup(check_version=False, verbose=not config['quiet'])
if not config['quiet']:
print("creating database (%s)" % (master_cfg.db['db_url'],))
yield db.model.upgrade()
def testSteps(self):
m = BuildMaster(".")
m.loadConfig(cfg1)
b = m.botmaster.builders["builder1"]
steps = b.buildFactory.steps
self.failUnlessEqual(len(steps), 4)
self.failUnlessExpectedShell(steps[0], command="echo yes")
self.failUnlessExpectedShell(steps[1],
defaults=False,
command="old-style")
self.failUnlessExpectedDarcs(steps[2],
repourl="http://buildbot.net/repos/trunk")
self.failUnlessExpectedShell(steps[3],
defaults=False,
command="echo old-style")
def create_db(self):
from buildbot.db import connector
from buildbot.master import BuildMaster
from buildbot import config as config_module
# create a master with the default configuration, but with db_url
# overridden
master_cfg = config_module.MasterConfig()
master_cfg.db['db_url'] = self.config['db']
master = BuildMaster(self.basedir)
master.config = master_cfg
db = connector.DBConnector(master, self.basedir)
d = db.setup(check_version=False)
if not self.config['quiet']:
print "creating database (%s)" % (master_cfg.db['db_url'], )
d = db.model.upgrade()
return d
def setUp(self):
self.basedir = os.path.abspath('basdir')
self.setUpDirs(self.basedir)
self.configfile = os.path.join(self.basedir, 'master.cfg')
# We create a master.cfg, which loads the configuration from the
# test module. Only the slave config is kept there, as it should not
# be changed
open(self.configfile, "w").write(textwrap.dedent("""
from buildbot.buildslave import BuildSlave
from %s import masterConfig
c = BuildmasterConfig = masterConfig()
c['slaves'] = [BuildSlave("local1", "localpw")]
c['protocols'] = {"pb": {"port": "tcp:0:interface=127.0.0.1"}}
""" % self.__class__.__module__))
# create the master and set its config
m = BuildMaster(self.basedir, self.configfile)
self.master = m
# update the DB
yield m.db.setup(check_version=False)
yield m.db.model.upgrade()
# stub out m.db.setup since it was already called above
m.db.setup = lambda: None
# mock reactor.stop (which trial *really* doesn't
# like test code to call!)
mock_reactor = mock.Mock(spec=reactor)
mock_reactor.callWhenRunning = reactor.callWhenRunning
# start the service
yield m.startService(_reactor=mock_reactor)
self.failIf(mock_reactor.stop.called,
"startService tried to stop the reactor; check logs")
# We find out the slave port automatically
slavePort = m.pbmanager.dispatchers.values()[0].port.getHost().port
# create a slave, and attach it to the master, it will be started, and stopped
# along with the master
s = BuildSlave("127.0.0.1", slavePort, "local1", "localpw", self.basedir, False, False)
s.setServiceParent(m)
def doCleanupDatabase(config, master_cfg):
if not config['quiet']:
print("cleaning database (%s)" % (master_cfg.db['db_url']))
master = BuildMaster(config['basedir'])
master.config = master_cfg
db = master.db
yield db.setup(check_version=False, verbose=not config['quiet'])
res = yield db.logs.getLogs()
i = 0
percent = 0
saved = 0
for log in res:
saved += yield db.logs.compressLog(log['id'], force=config['force'])
i += 1
if not config['quiet'] and percent != i * 100 / len(res):
percent = i * 100 / len(res)
print(" {0}% {1} saved".format(percent, saved))
saved = 0
sys.stdout.flush()
if master_cfg.db['db_url'].startswith("sqlite"):
if not config['quiet']:
print("executing sqlite vacuum function...")
# sqlite vacuum function rebuild the whole database to claim
# free disk space back
def thd(engine):
# In Python 3.6 and higher, sqlite3 no longer commits an
# open transaction before DDL statements.
# It is necessary to set the isolation_level to none
# for auto-commit mode before doing a VACUUM.
# See: https://bugs.python.org/issue28518
# Get the underlying sqlite connection from SQLAlchemy.
sqlite_conn = engine.connection.connection
# Set isolation_level to 'auto-commit mode'
sqlite_conn.isolation_level = None
sqlite_conn.execute("vacuum;").close()
yield db.pool.do(thd)
def populate_database(config):
master = BuildMaster(config['baseDir'])
master.config = load_config(config, config['configFile'])
db = connector.DBConnector(master, basedir=config['baseDir'])
seed = int(time())
if config['seed']:
seed = int(config['seed'])
random.seed(seed)
if not config['quiet']:
print("Seed =", seed)
yield db.setup(check_version=False, verbose=not config['quiet'])
users = yield populate_user(db,
int(config['users']),
verbose=not config['quiet'])
yield populate_build(db,
int(config['builds']),
master.config.builders,
master.config.projects,
users,
verbose=not config['quiet'])
def _run_master(self, loaded_config):
# create the master
m = BuildMaster(self.basedir, self.configfile)
# update the DB
yield m.db.setup(check_version=False)
yield m.db.model.upgrade()
# stub out m.db.setup since it was already called above
m.db.setup = lambda: None
# mock reactor.stop (which trial *really* doesn't
# like test code to call!)
mock_reactor = mock.Mock(spec=reactor)
mock_reactor.callWhenRunning = reactor.callWhenRunning
mock_reactor.getThreadPool = reactor.getThreadPool
mock_reactor.callFromThread = reactor.callFromThread
# mock configuration loading
@classmethod
def loadConfig(cls, basedir, filename):
return loaded_config
with mock.patch('buildbot.config.MasterConfig.loadConfig', loadConfig):
# start the service
yield m.startService(_reactor=mock_reactor)
self.failIf(mock_reactor.stop.called,
"startService tried to stop the reactor; check logs")
# hang out for a fraction of a second, to let startup processes run
d = defer.Deferred()
reactor.callLater(0.01, d.callback, None)
yield d
# stop the service
yield m.stopService()
# and shutdown the db threadpool, as is normally done at reactor stop
m.db.pool.shutdown()
def check_master_cfg(self, expected_db_url=None):
"""Check the buildmaster configuration, returning a deferred that
fires with an approprate exit status (so 0=success)."""
from buildbot.master import BuildMaster
from twisted.python import log
master_cfg = os.path.join(self.basedir, "master.cfg")
if not os.path.exists(master_cfg):
if not self.quiet:
print "No master.cfg found"
return defer.succeed(1)
# side-effects of loading the config file:
# for each Builder defined in c['builders'], if the status directory
# didn't already exist, it will be created, and the
# $BUILDERNAME/builder pickle might be created (with a single
# "builder created" event).
# we put basedir in front of sys.path, because that's how the
# buildmaster itself will run, and it is quite common to have the
# buildmaster import helper classes from other .py files in its
# basedir.
if sys.path[0] != self.basedir:
sys.path.insert(0, self.basedir)
m = BuildMaster(self.basedir)
# we need to route log.msg to stdout, so any problems can be seen
# there. But if everything goes well, I'd rather not clutter stdout
# with log messages. So instead we add a logObserver which gathers
# messages and only displays them if something goes wrong.
messages = []
log.addObserver(messages.append)
# this will errback if there's something wrong with the config file.
# Note that this BuildMaster instance is never started, so it won't
# actually do anything with the configuration.
d = defer.maybeDeferred(lambda :
m.loadConfig(open(master_cfg, "r"), checkOnly=True))
def check_db_url(config):
if (expected_db_url and
config.get('db_url', 'sqlite:///state.sqlite') != expected_db_url):
raise ValueError("c['db_url'] in the config file ('%s') does"
" not match '%s'; please edit the configuration"
" file before upgrading." %
(config['db_url'], expected_db_url))
d.addCallback(check_db_url)
def cb(_):
return 0
def eb(f):
if not self.quiet:
print
for m in messages:
print "".join(m['message'])
f.printTraceback()
print
print "An error was detected in the master.cfg file."
print "Please correct the problem and run 'buildbot upgrade-master' again."
print
return 1
d.addCallbacks(cb, eb)
return d
basedir = os.path.abspath(os.path.dirname(__file__))
configfile = 'master.cfg'
# Default umask for server
umask = None
# note: this line is matched against to check that this is a buildmaster
# directory; do not edit it.
application = service.Application('buildmaster')
import sys
from twisted.python.log import ILogObserver, FileLogObserver
application.setComponent(ILogObserver, FileLogObserver(sys.stdout).emit)
m = BuildMaster(basedir, configfile, umask)
m.setServiceParent(application)
# and slave on the same process!
buildmaster_host = 'localhost'
port = 19989
slavename = 'example-slave'
passwd = 'pass'
keepalive = 600
usepty = 0
umask = None
maxdelay = 300
allow_shutdown = None
slavedir = os.path.join(basedir, "slave")
if not os.path.exists(slavedir):
def testBadAddStepArguments(self):
m = BuildMaster(".")
self.failUnlessRaises(ArgumentsInTheWrongPlace, m.loadConfig, cfg1_bad)
def __init__(self, config_dict):
self.config_dict = config_dict
def loadConfig(self):
return MasterConfig.loadFromDict(self.config_dict, '<dict>')
@defer.inlineCallbacks
def getMaster(case, reactor, config_dict):
"""
Create a started ``BuildMaster`` with the given configuration.
"""
basedir = FilePath(case.mktemp())
basedir.createDirectory()
config_dict['buildbotNetUsageData'] = None
master = BuildMaster(
basedir.path, reactor=reactor, config_loader=DictLoader(config_dict))
if 'db_url' not in config_dict:
config_dict['db_url'] = 'sqlite://'
# TODO: Allow BuildMaster to transparently upgrade the database, at least
# for tests.
master.config.db['db_url'] = config_dict['db_url']
yield master.db.setup(check_version=False)
yield master.db.model.upgrade()
master.db.setup = lambda: None
yield master.startService()
case.addCleanup(master.stopService)
return master
'favicon.ico':
util.sibpath(__file__, "../status/web/files/favicon.ico"),
})
m.populate_if_missing(os.path.join(basedir, "master.cfg.sample"),
util.sibpath(__file__, "sample.cfg"),
overwrite=True)
# if index.html exists, use it to override the root page tempalte
m.move_if_present(os.path.join(basedir, "public_html/index.html"),
os.path.join(basedir, "templates/root.html"))
from buildbot.db import connector
from buildbot.master import BuildMaster
if not config['quiet']:
print "upgrading database (%s)" % (master_cfg.db['db_url'])
master = BuildMaster(config['basedir'])
master.config = master_cfg
db = connector.DBConnector(master, basedir=config['basedir'])
wfd = defer.waitForDeferred(
db.setup(check_version=False, verbose=not config['quiet']))
yield wfd
wfd.getResult()
wfd = defer.waitForDeferred(db.model.upgrade())
yield wfd
wfd.getResult()
if not config['quiet']: print "upgrade complete"
yield 0
def setupConfig(self, config_dict, startWorker=True):
"""
Setup and start a master configured
by the function configFunc defined in the test module.
@type config_dict: dict
@param configFunc: The BuildmasterConfig dictionary.
"""
self.basedir = os.path.abspath('basdir')
self.setUpDirs(self.basedir)
self.addCleanup(self.tearDownDirs)
# mock reactor.stop (which trial *really* doesn't
# like test code to call!)
stop = mock.create_autospec(reactor.stop)
self.patch(reactor, 'stop', stop)
if startWorker:
if self.proto == 'pb':
proto = {"pb": {"port": "tcp:0:interface=127.0.0.1"}}
workerclass = worker.Worker
elif self.proto == 'null':
proto = {"null": {}}
workerclass = worker.LocalWorker
config_dict['workers'] = [workerclass("local1", "localpw")]
config_dict['protocols'] = proto
# create the master and set its config
m = BuildMaster(self.basedir,
reactor=reactor,
config_loader=DictLoader(config_dict))
self.master = m
# update the DB
yield m.db.setup(check_version=False)
yield m.db.model.upgrade()
# stub out m.db.setup since it was already called above
m.db.setup = lambda: None
# start the service
yield m.startService()
self.failIf(stop.called,
"startService tried to stop the reactor; check logs")
# and shutdown the db threadpool, as is normally done at reactor stop
self.addCleanup(m.db.pool.shutdown)
self.addCleanup(m.stopService)
if not startWorker:
return
if self.proto == 'pb':
# We find out the worker port automatically
workerPort = list(itervalues(
m.pbmanager.dispatchers))[0].port.getHost().port
# create a worker, and attach it to the master, it will be started, and stopped
# along with the master
self.w = BuildSlave("127.0.0.1", workerPort, "local1", "localpw",
self.basedir, False, False)
elif self.proto == 'null':
self.w = None
if self.w is not None:
self.w.startService()
self.addCleanup(self.w.stopService)
@defer.inlineCallbacks
def dump():
if not self._passed:
dump = StringIO.StringIO()
print("FAILED! dumping build db for debug", file=dump)
builds = yield self.master.data.get(("builds", ))
for build in builds:
yield self.printBuild(build, dump, withLogs=True)
raise self.failureException(dump.getvalue())
self.addCleanup(dump)
def setUp(self):
# this class generates several deprecation warnings, which the user
# doesn't need to see.
warnings.simplefilter('ignore', exceptions.DeprecationWarning)
self.buildmaster = BuildMaster(".")
