def upgradeDatabase(config, master_cfg):
if not config['quiet']:
print("upgrading database (%s)" %
(stripUrlPassword(master_cfg.db['db_url'])))
print("Warning: Stopping this process might cause data loss")
def sighandler(signum, frame):
msg = " ".join("""
WARNING: ignoring signal %s.
This process should not be interrupted to avoid database corruption.
If you really need to terminate it, use SIGKILL.
""".split())
print(msg % signum)
for signame in ("SIGTERM", "SIGINT", "SIGQUIT", "SIGHUP", "SIGUSR1",
"SIGUSR2", "SIGBREAK"):
if hasattr(signal, signame):
signal.signal(getattr(signal, signame), sighandler)
master = BuildMaster(config['basedir'])
master.config = master_cfg
master.db.disownServiceParent()
db = connector.DBConnector(basedir=config['basedir'])
db.setServiceParent(master)
yield db.setup(check_version=False, verbose=not config['quiet'])
yield db.model.upgrade()
yield db.masters.setAllMastersActiveLongTimeAgo()
def do_test_master(self):
# create the master and set its config
m = BuildMaster(self.basedir, self.configfile)
m.config = config.MasterConfig.loadConfig(
self.basedir, self.configfile)
# update the DB
yield m.db.setup(check_version=False)
yield m.db.model.upgrade()
# stub out m.db.setup since it was already called above
m.db.setup = lambda : None
# mock reactor.stop (which trial *really* doesn't
# like test code to call!)
mock_reactor = mock.Mock(spec=reactor)
mock_reactor.callWhenRunning = reactor.callWhenRunning
# start the service
yield m.startService(_reactor=mock_reactor)
self.failIf(mock_reactor.stop.called,
"startService tried to stop the reactor; check logs")
# stop the service
yield m.stopService()
# and shutdown the db threadpool, as is normally done at reactor stop
m.db.pool.shutdown()
def upgradeDatabase(config, master_cfg):
if not config['quiet']:
print("upgrading database (%s)"
% (stripUrlPassword(master_cfg.db['db_url'])))
print("Warning: Stopping this process might cause data loss")
def sighandler(signum, frame):
msg = " ".join("""
WARNING: ignoring signal %s.
This process should not be interrupted to avoid database corruption.
If you really need to terminate it, use SIGKILL.
""".split())
print(msg % signum)
prev_handlers = {}
try:
for signame in ("SIGTERM", "SIGINT", "SIGQUIT", "SIGHUP",
"SIGUSR1", "SIGUSR2", "SIGBREAK"):
if hasattr(signal, signame):
signum = getattr(signal, signame)
prev_handlers[signum] = signal.signal(signum, sighandler)
master = BuildMaster(config['basedir'])
master.config = master_cfg
master.db.disownServiceParent()
db = connector.DBConnector(basedir=config['basedir'])
db.setServiceParent(master)
yield db.setup(check_version=False, verbose=not config['quiet'])
yield db.model.upgrade()
yield db.masters.setAllMastersActiveLongTimeAgo()
finally:
# restore previous signal handlers
for signum, handler in prev_handlers.items():
signal.signal(signum, handler)
def doCleanupDatabase(config, master_cfg):
if not config['quiet']:
print("cleaning database (%s)" % (master_cfg.db['db_url']))
master = BuildMaster(config['basedir'])
master.config = master_cfg
print(master.config.logCompressionMethod)
db = master.db
yield db.setup(check_version=False, verbose=not config['quiet'])
res = yield db.logs.getLogs()
i = 0
percent = 0
saved = 0
for log in res:
saved += yield db.logs.compressLog(log['id'])
i += 1
if not config['quiet'] and percent != i * 100 / len(res):
percent = i * 100 / len(res)
print(" {0}% {1} saved".format(percent, saved))
saved = 0
sys.stdout.flush()
if master_cfg.db['db_url'].startswith("sqlite"):
if not config['quiet']:
print("executing sqlite vacuum function...")
# sqlite vacuum function rebuild the whole database to claim
# free disk space back
def thd(engine):
r = engine.execute("vacuum;")
r.close()
yield db.pool.do(thd)
def doCleanupDatabase(config, master_cfg):
if not config["quiet"]:
print("cleaning database (%s)" % (master_cfg.db["db_url"]))
master = BuildMaster(config["basedir"])
master.config = master_cfg
db = master.db
yield db.setup(check_version=False, verbose=not config["quiet"])
res = yield db.logs.getLogs()
i = 0
percent = 0
saved = 0
for log in res:
saved += yield db.logs.compressLog(log["id"], force=config["force"])
i += 1
if not config["quiet"] and percent != i * 100 / len(res):
percent = i * 100 / len(res)
print(" {0}% {1} saved".format(percent, saved))
saved = 0
sys.stdout.flush()
if master_cfg.db["db_url"].startswith("sqlite"):
if not config["quiet"]:
print("executing sqlite vacuum function...")
# sqlite vacuum function rebuild the whole database to claim
# free disk space back
def thd(engine):
r = engine.execute("vacuum;")
r.close()
yield db.pool.do(thd)
def do_test_master(self):
# create the master and set its config
m = BuildMaster(self.basedir, self.configfile)
m.config = config.MasterConfig.loadConfig(self.basedir,
self.configfile)
# update the DB
yield m.db.setup(check_version=False)
yield m.db.model.upgrade()
# stub out m.db.setup since it was already called above
m.db.setup = lambda: None
# mock reactor.stop (which trial *really* doesn't
# like test code to call!)
mock_reactor = mock.Mock(spec=reactor)
mock_reactor.callWhenRunning = reactor.callWhenRunning
# start the service
yield m.startService(_reactor=mock_reactor)
self.failIf(mock_reactor.stop.called,
"startService tried to stop the reactor; check logs")
# hang out for a fraction of a second, to let startup processes run
d = defer.Deferred()
reactor.callLater(0.01, d.callback, None)
yield d
# stop the service
yield m.stopService()
# and shutdown the db threadpool, as is normally done at reactor stop
m.db.pool.shutdown()
def getMaster(self, config_dict):
"""
Create a started ``BuildMaster`` with the given configuration.
"""
basedir = FilePath(self.mktemp())
basedir.createDirectory()
master = BuildMaster(
basedir.path, reactor=reactor, config_loader=DictLoader(config_dict))
master.config = master.config_loader.loadConfig()
return master
def upgradeDatabase(config, master_cfg):
if not config['quiet']:
print "upgrading database (%s)" % (master_cfg.db['db_url'])
master = BuildMaster(config['basedir'])
master.config = master_cfg
db = connector.DBConnector(master, basedir=config['basedir'])
yield db.setup(check_version=False, verbose=not config['quiet'])
yield db.model.upgrade()
def getMaster(self, config_dict):
"""
Create a started ``BuildMaster`` with the given configuration.
"""
basedir = FilePath(self.mktemp())
basedir.createDirectory()
master = BuildMaster(
basedir.path, reactor=reactor, config_loader=DictLoader(config_dict))
master.config = master.config_loader.loadConfig()
return master
def upgradeDatabase(config, master_cfg):
if not config['quiet']:
print("upgrading database (%s)" % (master_cfg.db['db_url']))
master = BuildMaster(config['basedir'])
master.config = master_cfg
master.db.disownServiceParent()
db = connector.DBConnector(basedir=config['basedir'])
db.setServiceParent(master)
yield db.setup(check_version=False, verbose=not config['quiet'])
yield db.model.upgrade()
yield db.masters.setAllMastersActiveLongTimeAgo()
def upgradeDatabase(config, master_cfg):
if not config['quiet']:
print("upgrading database (%s)" % (master_cfg.db['db_url']))
master = BuildMaster(config['basedir'])
master.config = master_cfg
master.db.disownServiceParent()
db = connector.DBConnector(basedir=config['basedir'])
db.setServiceParent(master)
yield db.setup(check_version=False, verbose=not config['quiet'])
yield db.model.upgrade()
yield db.masters.setAllMastersActiveLongTimeAgo()
def upgradeDatabase(config, master_cfg):
if not config['quiet']:
print("upgrading database (%s)" %
(stripUrlPassword(master_cfg.db['db_url'])))
print("Warning: Stopping this process might cause data loss")
master = BuildMaster(config['basedir'])
master.config = master_cfg
master.db.disownServiceParent()
db = connector.DBConnector(basedir=config['basedir'])
db.setServiceParent(master)
yield db.setup(check_version=False, verbose=not config['quiet'])
yield db.model.upgrade()
yield db.masters.setAllMastersActiveLongTimeAgo()
def createDB(config, _noMonkey=False):
# apply the db monkeypatches (and others - no harm)
if not _noMonkey: # pragma: no cover
monkeypatches.patch_all()
# create a master with the default configuration, but with db_url
# overridden
master_cfg = config_module.MasterConfig()
master_cfg.db['db_url'] = config['db']
master = BuildMaster(config['basedir'])
master.config = master_cfg
db = connector.DBConnector(master, config['basedir'])
yield db.setup(check_version=False, verbose=not config['quiet'])
if not config['quiet']:
print "creating database (%s)" % (master_cfg.db['db_url'],)
yield db.model.upgrade()
def createDB(config, _noMonkey=False):
# apply the db monkeypatches (and others - no harm)
if not _noMonkey: # pragma: no cover
monkeypatches.patch_all()
# create a master with the default configuration, but with db_url
# overridden
master_cfg = config_module.MasterConfig()
master_cfg.db['db_url'] = config['db']
master = BuildMaster(config['basedir'])
master.config = master_cfg
db = master.db
yield db.setup(check_version=False, verbose=not config['quiet'])
if not config['quiet']:
print("creating database (%s)" % (master_cfg.db['db_url'],))
yield db.model.upgrade()
def create_db(self):
from buildbot.db import connector
from buildbot.master import BuildMaster
from buildbot import config as config_module
# create a master with the default configuration, but with db_url
# overridden
master_cfg = config_module.MasterConfig()
master_cfg.db['db_url'] = self.config['db']
master = BuildMaster(self.basedir)
master.config = master_cfg
db = connector.DBConnector(master, self.basedir)
d = db.setup(check_version=False)
if not self.config['quiet']:
print "creating database (%s)" % (master_cfg.db['db_url'], )
d = db.model.upgrade()
return d
def doCleanupDatabase(config, master_cfg):
if not config['quiet']:
print("cleaning database (%s)" % (master_cfg.db['db_url']))
master = BuildMaster(config['basedir'])
master.config = master_cfg
db = master.db
yield db.setup(check_version=False, verbose=not config['quiet'])
res = yield db.logs.getLogs()
i = 0
percent = 0
saved = 0
for log in res:
saved += yield db.logs.compressLog(log['id'], force=config['force'])
i += 1
if not config['quiet'] and percent != i * 100 / len(res):
percent = i * 100 / len(res)
print(" {0}% {1} saved".format(percent, saved))
saved = 0
sys.stdout.flush()
if master_cfg.db['db_url'].startswith("sqlite"):
if not config['quiet']:
print("executing sqlite vacuum function...")
# sqlite vacuum function rebuild the whole database to claim
# free disk space back
def thd(engine):
# In Python 3.6 and higher, sqlite3 no longer commits an
# open transaction before DDL statements.
# It is necessary to set the isolation_level to none
# for auto-commit mode before doing a VACUUM.
# See: https://bugs.python.org/issue28518
# Get the underlying sqlite connection from SQLAlchemy.
sqlite_conn = engine.connection.connection
# Set isolation_level to 'auto-commit mode'
sqlite_conn.isolation_level = None
sqlite_conn.execute("vacuum;").close()
yield db.pool.do(thd)
def populate_database(config):
master = BuildMaster(config['baseDir'])
master.config = load_config(config, config['configFile'])
db = connector.DBConnector(master, basedir=config['baseDir'])
seed = int(time())
if config['seed']:
seed = int(config['seed'])
random.seed(seed)
if not config['quiet']:
print("Seed =", seed)
yield db.setup(check_version=False, verbose=not config['quiet'])
users = yield populate_user(db,
int(config['users']),
verbose=not config['quiet'])
yield populate_build(db,
int(config['builds']),
master.config.builders,
master.config.projects,
users,
verbose=not config['quiet'])
util.sibpath(__file__, "../status/web/files/favicon.ico"),
})
m.populate_if_missing(os.path.join(basedir, "master.cfg.sample"),
util.sibpath(__file__, "sample.cfg"),
overwrite=True)
# if index.html exists, use it to override the root page tempalte
m.move_if_present(os.path.join(basedir, "public_html/index.html"),
os.path.join(basedir, "templates/root.html"))
from buildbot.db import connector
from buildbot.master import BuildMaster
if not config['quiet']:
print "upgrading database (%s)" % (master_cfg.db['db_url'])
master = BuildMaster(config['basedir'])
master.config = master_cfg
db = connector.DBConnector(master, basedir=config['basedir'])
wfd = defer.waitForDeferred(
db.setup(check_version=False, verbose=not config['quiet']))
yield wfd
wfd.getResult()
wfd = defer.waitForDeferred(db.model.upgrade())
yield wfd
wfd.getResult()
if not config['quiet']: print "upgrade complete"
yield 0
def get_master(basedir):
cfg = cm.MasterConfig.loadConfig(basedir, 'master.cfg')
master = BuildMaster(basedir)
master.config = cfg
return master