def upgradeMaster(config):
from buildbot import config as config_module
from buildbot import monkeypatches
import traceback
monkeypatches.patch_all()
m = Maker(config)
basedir = os.path.expanduser(config['basedir'])
if runtime.platformType != 'win32': # no pids on win32
if not config['quiet']: print "checking for running master"
pidfile = os.path.join(basedir, 'twistd.pid')
if os.path.exists(pidfile):
print "'%s' exists - is this master still running?" % (pidfile, )
yield 1
return
if not config['quiet']: print "checking master.cfg"
try:
master_cfg = config_module.MasterConfig.loadConfig(
basedir, 'master.cfg')
except config_module.ConfigErrors, e:
print "Errors loading configuration:"
for msg in e.errors:
print " " + msg
yield 1
return
def upgradeMaster(config, _noMonkey=False):
if not _noMonkey: # pragma: no cover
monkeypatches.patch_all()
if not base.checkBasedir(config):
defer.returnValue(1)
return
os.chdir(config['basedir'])
try:
configFile = base.getConfigFileFromTac(config['basedir'])
except (SyntaxError, ImportError) as e:
print("Unable to load 'buildbot.tac' from '%s':" % config['basedir'])
print(e)
defer.returnValue(1)
return
master_cfg = base.loadConfig(config, configFile)
if not master_cfg:
defer.returnValue(1)
return
upgradeFiles(config)
yield upgradeDatabase(config, master_cfg)
if not config['quiet']:
print("upgrade complete")
defer.returnValue(0)
def startService(self, _reactor=reactor):
assert not self._already_started, "can only start the master once"
self._already_started = True
log.msg("Starting BuildMaster -- buildbot.version: %s" %
buildbot.version)
# Set umask
if self.umask is not None:
os.umask(self.umask)
# first, apply all monkeypatches
monkeypatches.patch_all()
# we want to wait until the reactor is running, so we can call
# reactor.stop() for fatal errors
d = defer.Deferred()
_reactor.callWhenRunning(d.callback, None)
yield d
try:
# load the configuration file, treating errors as fatal
try:
self.config = config.MasterConfig.loadConfig(
self.basedir, self.configFileName)
except config.ConfigErrors, e:
log.msg("Configuration Errors:")
for msg in e.errors:
log.msg(" " + msg)
log.msg("Halting master.")
_reactor.stop()
return
except:
log.err(failure.Failure(), 'while starting BuildMaster')
_reactor.stop()
return
# set up services that need access to the config before everything else
# gets told to reconfig
try:
yield self.db.setup()
except connector.DatabaseNotReadyError:
# (message was already logged)
_reactor.stop()
return
if hasattr(signal, "SIGHUP"):
def sighup(*args):
_reactor.callLater(0, self.reconfig)
signal.signal(signal.SIGHUP, sighup)
# call the parent method
yield defer.maybeDeferred(
lambda: service.MultiService.startService(self))
# give all services a chance to load the new configuration, rather than
# the base configuration
yield self.reconfigService(self.config)
def upgradeMaster(config, _noMonkey=False):
if not _noMonkey: # pragma: no cover
monkeypatches.patch_all()
if not checkBasedir(config):
defer.returnValue(1)
return
os.chdir(config['basedir'])
configFile = base.getConfigFileWithFallback(config['basedir'])
master_cfg = loadConfig(config, configFile)
if not master_cfg:
defer.returnValue(1)
return
upgradeFiles(config)
try:
yield upgradeDatabase(config, master_cfg)
except Exception as e:
print "UNEXPECTED ERROR: %s" % str(e)
print "Buildmaster failed to upgrade." % str(e)
defer.returnValue(1)
return
if not config['quiet']:
print "upgrade complete"
defer.returnValue(0)
def upgradeMaster(config, _noMonkey=False):
if not _noMonkey: # pragma: no cover
monkeypatches.patch_all()
if not base.checkBasedir(config):
defer.returnValue(1)
return
os.chdir(config['basedir'])
try:
configFile = base.getConfigFileFromTac(config['basedir'])
except (SyntaxError, ImportError) as e:
print("Unable to load 'buildbot.tac' from '%s':" % config['basedir'])
print(e)
defer.returnValue(1)
return
master_cfg = base.loadConfig(config, configFile)
if not master_cfg:
defer.returnValue(1)
return
upgradeFiles(config)
yield upgradeDatabase(config, master_cfg)
if not config['quiet']:
print("upgrade complete")
defer.returnValue(0)
def startService(self, _reactor=reactor):
assert not self._already_started, "can only start the master once"
self._already_started = True
log.msg("Starting BuildMaster -- buildbot.version: %s" %
buildbot.version)
# Set umask
if self.umask is not None:
os.umask(self.umask)
# first, apply all monkeypatches
monkeypatches.patch_all()
# we want to wait until the reactor is running, so we can call
# reactor.stop() for fatal errors
d = defer.Deferred()
_reactor.callWhenRunning(d.callback, None)
yield d
try:
# load the configuration file, treating errors as fatal
try:
self.config = config.MasterConfig.loadConfig(self.basedir,
self.configFileName)
except config.ConfigErrors, e:
log.msg("Configuration Errors:")
for msg in e.errors:
log.msg(" " + msg)
log.msg("Halting master.")
_reactor.stop()
return
except:
log.err(failure.Failure(), 'while starting BuildMaster')
_reactor.stop()
return
# set up services that need access to the config before everything else
# gets told to reconfig
try:
yield self.db.setup()
except connector.DatabaseNotReadyError:
# (message was already logged)
_reactor.stop()
return
if hasattr(signal, "SIGHUP"):
def sighup(*args):
_reactor.callLater(0, self.reconfig)
signal.signal(signal.SIGHUP, sighup)
# call the parent method
yield defer.maybeDeferred(lambda :
service.MultiService.startService(self))
# give all services a chance to load the new configuration, rather than
# the base configuration
yield self.reconfigService(self.config)
def createDB(config, _noMonkey=False):
# apply the db monkeypatches (and others - no harm)
if not _noMonkey: # pragma: no cover
monkeypatches.patch_all()
# create a master with the default configuration, but with db_url
# overridden
master_cfg = config_module.MasterConfig()
master_cfg.db['db_url'] = config['db']
master = BuildMaster(config['basedir'])
master.config = master_cfg
db = master.db
yield db.setup(check_version=False, verbose=not config['quiet'])
if not config['quiet']:
print("creating database (%s)" % (master_cfg.db['db_url'],))
yield db.model.upgrade()
def createDB(config, _noMonkey=False):
# apply the db monkeypatches (and others - no harm)
if not _noMonkey: # pragma: no cover
monkeypatches.patch_all()
# create a master with the default configuration, but with db_url
# overridden
master_cfg = config_module.MasterConfig()
master_cfg.db['db_url'] = config['db']
master = BuildMaster(config['basedir'])
master.config = master_cfg
db = connector.DBConnector(master, config['basedir'])
yield db.setup(check_version=False, verbose=not config['quiet'])
if not config['quiet']:
print "creating database (%s)" % (master_cfg.db['db_url'],)
yield db.model.upgrade()
def upgradeMaster(config, _noMonkey=False):
if not _noMonkey: # pragma: no cover
monkeypatches.patch_all()
if not checkBasedir(config):
defer.returnValue(1)
return
os.chdir(config['basedir'])
try:
configFile = base.getConfigFileFromTac(config['basedir'])
except (SyntaxError, ImportError), e:
print "Unable to load 'buildbot.tac' from '%s':" % config['basedir']
print e
defer.returnValue(1)
return
def upgradeMaster(config, _noMonkey=False):
if not _noMonkey: # pragma: no cover
monkeypatches.patch_all()
if not base.checkBasedir(config):
return defer.succeed(1)
os.chdir(config["basedir"])
try:
configFile = base.getConfigFileFromTac(config["basedir"])
except (SyntaxError, ImportError):
print("Unable to load 'buildbot.tac' from '%s':" % config["basedir"], file=sys.stderr)
e = traceback.format_exc()
print(e, file=sys.stderr)
return defer.succeed(1)
master_cfg = base.loadConfig(config, configFile)
if not master_cfg:
return defer.succeed(1)
return _upgradeMaster(config, master_cfg)
def upgradeMaster(config):
from buildbot import config as config_module
from buildbot import monkeypatches
import traceback
monkeypatches.patch_all()
m = Maker(config)
basedir = os.path.expanduser(config['basedir'])
if not config['quiet']: print "checking master.cfg"
try:
master_cfg = config_module.MasterConfig.loadConfig(
basedir, 'master.cfg')
except config_module.ConfigErrors, e:
print "Errors loading configuration:"
for msg in e.errors:
print " " + msg
yield 1
return
def upgradeMaster(config, _noMonkey=False):
if not _noMonkey: # pragma: no cover
monkeypatches.patch_all()
if not base.checkBasedir(config):
return defer.succeed(1)
os.chdir(config['basedir'])
try:
configFile = base.getConfigFileFromTac(config['basedir'])
except (SyntaxError, ImportError):
print("Unable to load 'buildbot.tac' from '{}':".format(config['basedir'], file=sys.stderr))
e = traceback.format_exc()
print(e, file=sys.stderr)
return defer.succeed(1)
master_cfg = base.loadConfig(config, configFile)
if not master_cfg:
return defer.succeed(1)
return _upgradeMaster(config, master_cfg)
def create_db(self):
from buildbot.db import connector
from buildbot.master import BuildMaster
from buildbot import config as config_module
from buildbot import monkeypatches
monkeypatches.patch_all()
# create a master with the default configuration, but with db_url
# overridden
master_cfg = config_module.MasterConfig()
master_cfg.db['db_url'] = self.config['db']
master = BuildMaster(self.basedir)
master.config = master_cfg
db = connector.DBConnector(master, self.basedir)
d = db.setup(check_version=False, verbose=not self.config['quiet'])
if not self.config['quiet']:
print "creating database (%s)" % (master_cfg.db['db_url'], )
d = db.model.upgrade()
return d
def upgradeMaster(config, _noMonkey=False):
if not _noMonkey: # pragma: no cover
monkeypatches.patch_all()
if not checkBasedir(config):
defer.returnValue(1)
return
os.chdir(config['basedir'])
master_cfg = loadConfig(config)
if not master_cfg:
defer.returnValue(1)
return
upgradeFiles(config)
yield upgradeDatabase(config, master_cfg)
if not config['quiet']:
print "upgrade complete"
defer.returnValue(0)
def upgradeMaster(config, _noMonkey=False):
if not _noMonkey: # pragma: no cover
monkeypatches.patch_all()
if not checkBasedir(config):
defer.returnValue(1)
return
os.chdir(config['basedir'])
configFile = base.getConfigFileWithFallback(config['basedir'])
master_cfg = loadConfig(config, configFile)
if not master_cfg:
defer.returnValue(1)
return
upgradeFiles(config)
yield upgradeDatabase(config, master_cfg)
if not config['quiet']:
print "upgrade complete"
defer.returnValue(0)
# This file is part of Buildbot. Buildbot is free software: you can # redistribute it and/or modify it under the terms of the GNU General Public # License as published by the Free Software Foundation, version 2. # # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more # details. # # You should have received a copy of the GNU General Public License along with # this program; if not, write to the Free Software Foundation, Inc., 51 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # # Copyright Buildbot Team Members # apply the same patches the buildmaster does when it starts from buildbot import monkeypatches monkeypatches.patch_all() # and some extras to aid debugging from buildbot.util import monkeypatches monkeypatches.add_debugging_monkeypatches()
def startService(self):
assert not self._already_started, "can only start the master once"
self._already_started = True
log.msg("Starting BuildMaster -- buildbot.version: %s" %
buildbot.version)
# Set umask
if self.umask is not None:
os.umask(self.umask)
# first, apply all monkeypatches
monkeypatches.patch_all()
# we want to wait until the reactor is running, so we can call
# reactor.stop() for fatal errors
d = defer.Deferred()
self.reactor.callWhenRunning(d.callback, None)
yield d
startup_succeed = False
try:
# load the configuration file, treating errors as fatal
try:
# run the master.cfg in thread, so that it can use blocking
# code
self.config = yield threads.deferToThreadPool(
self.reactor, self.reactor.getThreadPool(),
self.config_loader.loadConfig)
except config.ConfigErrors as e:
log.msg("Configuration Errors:")
for msg in e.errors:
log.msg(" " + msg)
log.msg("Halting master.")
self.reactor.stop()
return
except Exception:
log.err(failure.Failure(), 'while starting BuildMaster')
self.reactor.stop()
return
# set up services that need access to the config before everything
# else gets told to reconfig
try:
yield self.db.setup()
except exceptions.DatabaseNotReadyError:
# (message was already logged)
self.reactor.stop()
return
self.mq.setup()
if hasattr(signal, "SIGHUP"):
def sighup(*args):
eventually(self.reconfig)
signal.signal(signal.SIGHUP, sighup)
if hasattr(signal, "SIGUSR1"):
def sigusr1(*args):
self.reactor.callLater(0, self.botmaster.cleanShutdown)
signal.signal(signal.SIGUSR1, sigusr1)
# get the masterid so other services can use it in
# startup/reconfig. This goes directly to the DB since the data
# API isn't initialized yet, and anyway, this method is aware of
# the DB API since it just called its setup function
self.masterid = yield self.db.masters.findMasterId(
name=self.name)
# mark this master as stopped, in case it crashed before
yield self.data.updates.masterStopped(name=self.name,
masterid=self.masterid)
# call the parent method
yield service.AsyncMultiService.startService(self)
# give all services a chance to load the new configuration, rather
# than the base configuration
yield self.reconfigServiceWithBuildbotConfig(self.config)
# mark the master as active now that mq is running
yield self.data.updates.masterActive(
name=self.name,
masterid=self.masterid)
startup_succeed = True
except Exception:
f = failure.Failure()
log.err(f, 'while starting BuildMaster')
self.reactor.stop()
finally:
if startup_succeed:
log.msg("BuildMaster is running")
else:
log.msg("BuildMaster startup failed")
self._master_initialized = True
def startService(self):
assert not self._already_started, "can only start the master once"
self._already_started = True
# ensure child services have been set up. Normally we would do this in serServiceParent,
# but buildmaster is used in contexts we can't control.
if self._services_d is not None:
yield self._services_d
self._services_d = None
log.msg(f"Starting BuildMaster -- buildbot.version: {buildbot.version}")
# Set umask
if self.umask is not None:
os.umask(self.umask)
# first, apply all monkeypatches
monkeypatches.patch_all()
# we want to wait until the reactor is running, so we can call
# reactor.stop() for fatal errors
d = defer.Deferred()
self.reactor.callWhenRunning(d.callback, None)
yield d
startup_succeed = False
try:
yield self.initLock.acquire()
# load the configuration file, treating errors as fatal
try:
# run the master.cfg in thread, so that it can use blocking
# code
self.config = yield threads.deferToThreadPool(
self.reactor, self.reactor.getThreadPool(),
self.config_loader.loadConfig)
except config.ConfigErrors as e:
log.msg("Configuration Errors:")
for msg in e.errors:
log.msg(" " + msg)
log.msg("Halting master.")
self.reactor.stop()
return
except Exception:
log.err(failure.Failure(), 'while starting BuildMaster')
self.reactor.stop()
return
# set up services that need access to the config before everything
# else gets told to reconfig
try:
yield self.db.setup()
except exceptions.DatabaseNotReadyError:
# (message was already logged)
self.reactor.stop()
return
yield self.mq.setup()
# the buildbot scripts send the SIGHUP signal to reconfig master
if hasattr(signal, "SIGHUP"):
def sighup(*args):
eventually(self.reconfig)
signal.signal(signal.SIGHUP, sighup)
# the buildbot scripts send the SIGUSR1 signal to stop master
if hasattr(signal, "SIGUSR1"):
def sigusr1(*args):
eventually(self.botmaster.cleanShutdown)
signal.signal(signal.SIGUSR1, sigusr1)
# get the masterid so other services can use it in
# startup/reconfig. This goes directly to the DB since the data
# API isn't initialized yet, and anyway, this method is aware of
# the DB API since it just called its setup function
self.masterid = yield self.db.masters.findMasterId(
name=self.name)
# mark this master as stopped, in case it crashed before
yield self.data.updates.masterStopped(name=self.name,
masterid=self.masterid)
# call the parent method
yield super().startService()
# We make sure the housekeeping is done before configuring in order to cleanup
# any remaining claimed schedulers or change sources from zombie
# masters
yield self.data.updates.expireMasters(forceHouseKeeping=True)
# give all services a chance to load the new configuration, rather
# than the base configuration
yield self.reconfigServiceWithBuildbotConfig(self.config)
# Mark the master as active now that mq is running
yield self.data.updates.masterActive(name=self.name,
masterid=self.masterid)
# Start the heartbeat timer
yield self.masterHeartbeatService.setServiceParent(self)
# send the statistics to buildbot.net, without waiting
self.sendBuildbotNetUsageData()
startup_succeed = True
except Exception:
f = failure.Failure()
log.err(f, 'while starting BuildMaster')
self.reactor.stop()
finally:
if startup_succeed:
log.msg("BuildMaster is running")
else:
log.msg("BuildMaster startup failed")
yield self.initLock.release()
self._master_initialized = True
def startService(self):
assert not self._already_started, "can only start the master once"
self._already_started = True
log.msg("Starting BuildMaster -- buildbot.version: %s" %
buildbot.version)
# Set umask
if self.umask is not None:
os.umask(self.umask)
# first, apply all monkeypatches
monkeypatches.patch_all()
# we want to wait until the reactor is running, so we can call
# reactor.stop() for fatal errors
d = defer.Deferred()
self.reactor.callWhenRunning(d.callback, None)
yield d
startup_succeed = False
try:
# load the configuration file, treating errors as fatal
try:
# run the master.cfg in thread, so that it can use blocking
# code
self.config = yield threads.deferToThreadPool(
self.reactor, self.reactor.getThreadPool(),
self.config_loader.loadConfig)
except config.ConfigErrors as e:
log.msg("Configuration Errors:")
for msg in e.errors:
log.msg(" " + msg)
log.msg("Halting master.")
self.reactor.stop()
return
except Exception:
log.err(failure.Failure(), 'while starting BuildMaster')
self.reactor.stop()
return
# set up services that need access to the config before everything
# else gets told to reconfig
try:
yield self.db.setup()
except exceptions.DatabaseNotReadyError:
# (message was already logged)
self.reactor.stop()
return
self.mq.setup()
if hasattr(signal, "SIGHUP"):
def sighup(*args):
eventually(self.reconfig)
signal.signal(signal.SIGHUP, sighup)
if hasattr(signal, "SIGUSR1"):
def sigusr1(*args):
self.reactor.callLater(0, self.botmaster.cleanShutdown)
signal.signal(signal.SIGUSR1, sigusr1)
# get the masterid so other services can use it in
# startup/reconfig. This goes directly to the DB since the data
# API isn't initialized yet, and anyway, this method is aware of
# the DB API since it just called its setup function
self.masterid = yield self.db.masters.findMasterId(name=self.name)
# mark this master as stopped, in case it crashed before
yield self.data.updates.masterStopped(name=self.name,
masterid=self.masterid)
# call the parent method
yield service.AsyncMultiService.startService(self)
# give all services a chance to load the new configuration, rather
# than the base configuration
yield self.reconfigServiceWithBuildbotConfig(self.config)
# mark the master as active now that mq is running
yield self.data.updates.masterActive(name=self.name,
masterid=self.masterid)
startup_succeed = True
except Exception:
f = failure.Failure()
log.err(f, 'while starting BuildMaster')
self.reactor.stop()
finally:
if startup_succeed:
log.msg("BuildMaster is running")
else:
log.msg("BuildMaster startup failed")
self._master_initialized = True
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
import sys
import warnings
from distutils.version import LooseVersion
from buildbot import monkeypatches
# apply the same patches the buildmaster does when it starts
monkeypatches.patch_all(for_tests=True)
# enable deprecation warnings
warnings.filterwarnings('always', category=DeprecationWarning)
if sys.version_info[:2] < (3, 2):
# Setup logging unhandled messages to stderr.
# Since Python 3.2 similar functionality implemented through
# logging.lastResort handler.
# Significant difference between this approach and Python 3.2 last resort
# approach is that in the current approach only records with log level
# equal or above to the root logger log level will be printed (WARNING by
# default). For example, there still will be warnings about missing
# handler for log if INFO or DEBUG records will be logged (but at least
# WARNINGs and ERRORs will be printed).
import logging
def cleanupDatabase(config, _noMonkey=False): # pragma: no cover
# we separate the actual implementation to protect unit tests
# from @in_reactor which stops the reactor
if not _noMonkey:
monkeypatches.patch_all()
return _cleanupDatabase(config, _noMonkey=False)
# This file is part of Buildbot. Buildbot is free software: you can # redistribute it and/or modify it under the terms of the GNU General Public # License as published by the Free Software Foundation, version 2. # # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more # details. # # You should have received a copy of the GNU General Public License along with # this program; if not, write to the Free Software Foundation, Inc., 51 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # # Copyright Buildbot Team Members # apply the same patches the buildmaster does when it starts from buildbot import monkeypatches monkeypatches.patch_all() # and some extras to aid debugging from buildbot.util import monkeypatches monkeypatches.add_debugging_monkeypatches()
from buildbot import monkeypatches
from buildbot.test.util.warnings import assertProducesWarning # noqa pylint: disable=wrong-import-position
from buildbot.test.util.warnings import assertProducesWarnings # noqa pylint: disable=wrong-import-position
from buildbot.warnings import DeprecatedApiWarning # noqa pylint: disable=wrong-import-position
# import mock so we bail out early if it's not installed
try:
import mock
[mock]
except ImportError as e:
raise ImportError("\nBuildbot tests require the 'mock' module; "
"try 'pip install mock'") from e
# apply the same patches the buildmaster does when it starts
monkeypatches.patch_all(for_tests=True)
# enable deprecation warnings
warnings.filterwarnings('always', category=DeprecationWarning)
if parse_version(mock.__version__) < parse_version("0.8"):
raise ImportError("\nBuildbot tests require mock version 0.8.0 or "
"higher; try 'pip install -U mock'")
[setuptools] # force use for pylint
# This is where we load deprecated module-level APIs to ignore warning produced by importing them.
# After the deprecated API has been removed, leave at least one instance of the import in a
# commented state as reference.
# with assertProducesWarnings(DeprecatedApiWarning,
def cleanupDatabase(config, _noMonkey=False): # pragma: no cover
# we separate the actual implementation to protect unit tests
# from @in_reactor which stops the reactor
if not _noMonkey:
monkeypatches.patch_all()
return _cleanupDatabase(config, _noMonkey=False)