def upgradeMaster(config): from buildbot import config as config_module from buildbot import monkeypatches import traceback monkeypatches.patch_all() m = Maker(config) basedir = os.path.expanduser(config['basedir']) if runtime.platformType != 'win32': # no pids on win32 if not config['quiet']: print "checking for running master" pidfile = os.path.join(basedir, 'twistd.pid') if os.path.exists(pidfile): print "'%s' exists - is this master still running?" % (pidfile, ) yield 1 return if not config['quiet']: print "checking master.cfg" try: master_cfg = config_module.MasterConfig.loadConfig( basedir, 'master.cfg') except config_module.ConfigErrors, e: print "Errors loading configuration:" for msg in e.errors: print " " + msg yield 1 return
def upgradeMaster(config, _noMonkey=False): if not _noMonkey: # pragma: no cover monkeypatches.patch_all() if not base.checkBasedir(config): defer.returnValue(1) return os.chdir(config['basedir']) try: configFile = base.getConfigFileFromTac(config['basedir']) except (SyntaxError, ImportError) as e: print("Unable to load 'buildbot.tac' from '%s':" % config['basedir']) print(e) defer.returnValue(1) return master_cfg = base.loadConfig(config, configFile) if not master_cfg: defer.returnValue(1) return upgradeFiles(config) yield upgradeDatabase(config, master_cfg) if not config['quiet']: print("upgrade complete") defer.returnValue(0)
def startService(self, _reactor=reactor): assert not self._already_started, "can only start the master once" self._already_started = True log.msg("Starting BuildMaster -- buildbot.version: %s" % buildbot.version) # Set umask if self.umask is not None: os.umask(self.umask) # first, apply all monkeypatches monkeypatches.patch_all() # we want to wait until the reactor is running, so we can call # reactor.stop() for fatal errors d = defer.Deferred() _reactor.callWhenRunning(d.callback, None) yield d try: # load the configuration file, treating errors as fatal try: self.config = config.MasterConfig.loadConfig( self.basedir, self.configFileName) except config.ConfigErrors, e: log.msg("Configuration Errors:") for msg in e.errors: log.msg(" " + msg) log.msg("Halting master.") _reactor.stop() return except: log.err(failure.Failure(), 'while starting BuildMaster') _reactor.stop() return # set up services that need access to the config before everything else # gets told to reconfig try: yield self.db.setup() except connector.DatabaseNotReadyError: # (message was already logged) _reactor.stop() return if hasattr(signal, "SIGHUP"): def sighup(*args): _reactor.callLater(0, self.reconfig) signal.signal(signal.SIGHUP, sighup) # call the parent method yield defer.maybeDeferred( lambda: service.MultiService.startService(self)) # give all services a chance to load the new configuration, rather than # the base configuration yield self.reconfigService(self.config)
def upgradeMaster(config, _noMonkey=False): if not _noMonkey: # pragma: no cover monkeypatches.patch_all() if not checkBasedir(config): defer.returnValue(1) return os.chdir(config['basedir']) configFile = base.getConfigFileWithFallback(config['basedir']) master_cfg = loadConfig(config, configFile) if not master_cfg: defer.returnValue(1) return upgradeFiles(config) try: yield upgradeDatabase(config, master_cfg) except Exception as e: print "UNEXPECTED ERROR: %s" % str(e) print "Buildmaster failed to upgrade." % str(e) defer.returnValue(1) return if not config['quiet']: print "upgrade complete" defer.returnValue(0)
def startService(self, _reactor=reactor): assert not self._already_started, "can only start the master once" self._already_started = True log.msg("Starting BuildMaster -- buildbot.version: %s" % buildbot.version) # Set umask if self.umask is not None: os.umask(self.umask) # first, apply all monkeypatches monkeypatches.patch_all() # we want to wait until the reactor is running, so we can call # reactor.stop() for fatal errors d = defer.Deferred() _reactor.callWhenRunning(d.callback, None) yield d try: # load the configuration file, treating errors as fatal try: self.config = config.MasterConfig.loadConfig(self.basedir, self.configFileName) except config.ConfigErrors, e: log.msg("Configuration Errors:") for msg in e.errors: log.msg(" " + msg) log.msg("Halting master.") _reactor.stop() return except: log.err(failure.Failure(), 'while starting BuildMaster') _reactor.stop() return # set up services that need access to the config before everything else # gets told to reconfig try: yield self.db.setup() except connector.DatabaseNotReadyError: # (message was already logged) _reactor.stop() return if hasattr(signal, "SIGHUP"): def sighup(*args): _reactor.callLater(0, self.reconfig) signal.signal(signal.SIGHUP, sighup) # call the parent method yield defer.maybeDeferred(lambda : service.MultiService.startService(self)) # give all services a chance to load the new configuration, rather than # the base configuration yield self.reconfigService(self.config)
def createDB(config, _noMonkey=False): # apply the db monkeypatches (and others - no harm) if not _noMonkey: # pragma: no cover monkeypatches.patch_all() # create a master with the default configuration, but with db_url # overridden master_cfg = config_module.MasterConfig() master_cfg.db['db_url'] = config['db'] master = BuildMaster(config['basedir']) master.config = master_cfg db = master.db yield db.setup(check_version=False, verbose=not config['quiet']) if not config['quiet']: print("creating database (%s)" % (master_cfg.db['db_url'],)) yield db.model.upgrade()
def createDB(config, _noMonkey=False): # apply the db monkeypatches (and others - no harm) if not _noMonkey: # pragma: no cover monkeypatches.patch_all() # create a master with the default configuration, but with db_url # overridden master_cfg = config_module.MasterConfig() master_cfg.db['db_url'] = config['db'] master = BuildMaster(config['basedir']) master.config = master_cfg db = connector.DBConnector(master, config['basedir']) yield db.setup(check_version=False, verbose=not config['quiet']) if not config['quiet']: print "creating database (%s)" % (master_cfg.db['db_url'],) yield db.model.upgrade()
def upgradeMaster(config, _noMonkey=False): if not _noMonkey: # pragma: no cover monkeypatches.patch_all() if not checkBasedir(config): defer.returnValue(1) return os.chdir(config['basedir']) try: configFile = base.getConfigFileFromTac(config['basedir']) except (SyntaxError, ImportError), e: print "Unable to load 'buildbot.tac' from '%s':" % config['basedir'] print e defer.returnValue(1) return
def upgradeMaster(config, _noMonkey=False): if not _noMonkey: # pragma: no cover monkeypatches.patch_all() if not base.checkBasedir(config): return defer.succeed(1) os.chdir(config["basedir"]) try: configFile = base.getConfigFileFromTac(config["basedir"]) except (SyntaxError, ImportError): print("Unable to load 'buildbot.tac' from '%s':" % config["basedir"], file=sys.stderr) e = traceback.format_exc() print(e, file=sys.stderr) return defer.succeed(1) master_cfg = base.loadConfig(config, configFile) if not master_cfg: return defer.succeed(1) return _upgradeMaster(config, master_cfg)
def upgradeMaster(config): from buildbot import config as config_module from buildbot import monkeypatches import traceback monkeypatches.patch_all() m = Maker(config) basedir = os.path.expanduser(config['basedir']) if not config['quiet']: print "checking master.cfg" try: master_cfg = config_module.MasterConfig.loadConfig( basedir, 'master.cfg') except config_module.ConfigErrors, e: print "Errors loading configuration:" for msg in e.errors: print " " + msg yield 1 return
def upgradeMaster(config, _noMonkey=False): if not _noMonkey: # pragma: no cover monkeypatches.patch_all() if not base.checkBasedir(config): return defer.succeed(1) os.chdir(config['basedir']) try: configFile = base.getConfigFileFromTac(config['basedir']) except (SyntaxError, ImportError): print("Unable to load 'buildbot.tac' from '{}':".format(config['basedir'], file=sys.stderr)) e = traceback.format_exc() print(e, file=sys.stderr) return defer.succeed(1) master_cfg = base.loadConfig(config, configFile) if not master_cfg: return defer.succeed(1) return _upgradeMaster(config, master_cfg)
def create_db(self): from buildbot.db import connector from buildbot.master import BuildMaster from buildbot import config as config_module from buildbot import monkeypatches monkeypatches.patch_all() # create a master with the default configuration, but with db_url # overridden master_cfg = config_module.MasterConfig() master_cfg.db['db_url'] = self.config['db'] master = BuildMaster(self.basedir) master.config = master_cfg db = connector.DBConnector(master, self.basedir) d = db.setup(check_version=False, verbose=not self.config['quiet']) if not self.config['quiet']: print "creating database (%s)" % (master_cfg.db['db_url'], ) d = db.model.upgrade() return d
def upgradeMaster(config, _noMonkey=False): if not _noMonkey: # pragma: no cover monkeypatches.patch_all() if not checkBasedir(config): defer.returnValue(1) return os.chdir(config['basedir']) master_cfg = loadConfig(config) if not master_cfg: defer.returnValue(1) return upgradeFiles(config) yield upgradeDatabase(config, master_cfg) if not config['quiet']: print "upgrade complete" defer.returnValue(0)
def upgradeMaster(config, _noMonkey=False): if not _noMonkey: # pragma: no cover monkeypatches.patch_all() if not checkBasedir(config): defer.returnValue(1) return os.chdir(config['basedir']) configFile = base.getConfigFileWithFallback(config['basedir']) master_cfg = loadConfig(config, configFile) if not master_cfg: defer.returnValue(1) return upgradeFiles(config) yield upgradeDatabase(config, master_cfg) if not config['quiet']: print "upgrade complete" defer.returnValue(0)
# This file is part of Buildbot. Buildbot is free software: you can # redistribute it and/or modify it under the terms of the GNU General Public # License as published by the Free Software Foundation, version 2. # # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more # details. # # You should have received a copy of the GNU General Public License along with # this program; if not, write to the Free Software Foundation, Inc., 51 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # # Copyright Buildbot Team Members # apply the same patches the buildmaster does when it starts from buildbot import monkeypatches monkeypatches.patch_all() # and some extras to aid debugging from buildbot.util import monkeypatches monkeypatches.add_debugging_monkeypatches()
def startService(self): assert not self._already_started, "can only start the master once" self._already_started = True log.msg("Starting BuildMaster -- buildbot.version: %s" % buildbot.version) # Set umask if self.umask is not None: os.umask(self.umask) # first, apply all monkeypatches monkeypatches.patch_all() # we want to wait until the reactor is running, so we can call # reactor.stop() for fatal errors d = defer.Deferred() self.reactor.callWhenRunning(d.callback, None) yield d startup_succeed = False try: # load the configuration file, treating errors as fatal try: # run the master.cfg in thread, so that it can use blocking # code self.config = yield threads.deferToThreadPool( self.reactor, self.reactor.getThreadPool(), self.config_loader.loadConfig) except config.ConfigErrors as e: log.msg("Configuration Errors:") for msg in e.errors: log.msg(" " + msg) log.msg("Halting master.") self.reactor.stop() return except Exception: log.err(failure.Failure(), 'while starting BuildMaster') self.reactor.stop() return # set up services that need access to the config before everything # else gets told to reconfig try: yield self.db.setup() except exceptions.DatabaseNotReadyError: # (message was already logged) self.reactor.stop() return self.mq.setup() if hasattr(signal, "SIGHUP"): def sighup(*args): eventually(self.reconfig) signal.signal(signal.SIGHUP, sighup) if hasattr(signal, "SIGUSR1"): def sigusr1(*args): self.reactor.callLater(0, self.botmaster.cleanShutdown) signal.signal(signal.SIGUSR1, sigusr1) # get the masterid so other services can use it in # startup/reconfig. This goes directly to the DB since the data # API isn't initialized yet, and anyway, this method is aware of # the DB API since it just called its setup function self.masterid = yield self.db.masters.findMasterId( name=self.name) # mark this master as stopped, in case it crashed before yield self.data.updates.masterStopped(name=self.name, masterid=self.masterid) # call the parent method yield service.AsyncMultiService.startService(self) # give all services a chance to load the new configuration, rather # than the base configuration yield self.reconfigServiceWithBuildbotConfig(self.config) # mark the master as active now that mq is running yield self.data.updates.masterActive( name=self.name, masterid=self.masterid) startup_succeed = True except Exception: f = failure.Failure() log.err(f, 'while starting BuildMaster') self.reactor.stop() finally: if startup_succeed: log.msg("BuildMaster is running") else: log.msg("BuildMaster startup failed") self._master_initialized = True
def startService(self): assert not self._already_started, "can only start the master once" self._already_started = True # ensure child services have been set up. Normally we would do this in serServiceParent, # but buildmaster is used in contexts we can't control. if self._services_d is not None: yield self._services_d self._services_d = None log.msg(f"Starting BuildMaster -- buildbot.version: {buildbot.version}") # Set umask if self.umask is not None: os.umask(self.umask) # first, apply all monkeypatches monkeypatches.patch_all() # we want to wait until the reactor is running, so we can call # reactor.stop() for fatal errors d = defer.Deferred() self.reactor.callWhenRunning(d.callback, None) yield d startup_succeed = False try: yield self.initLock.acquire() # load the configuration file, treating errors as fatal try: # run the master.cfg in thread, so that it can use blocking # code self.config = yield threads.deferToThreadPool( self.reactor, self.reactor.getThreadPool(), self.config_loader.loadConfig) except config.ConfigErrors as e: log.msg("Configuration Errors:") for msg in e.errors: log.msg(" " + msg) log.msg("Halting master.") self.reactor.stop() return except Exception: log.err(failure.Failure(), 'while starting BuildMaster') self.reactor.stop() return # set up services that need access to the config before everything # else gets told to reconfig try: yield self.db.setup() except exceptions.DatabaseNotReadyError: # (message was already logged) self.reactor.stop() return yield self.mq.setup() # the buildbot scripts send the SIGHUP signal to reconfig master if hasattr(signal, "SIGHUP"): def sighup(*args): eventually(self.reconfig) signal.signal(signal.SIGHUP, sighup) # the buildbot scripts send the SIGUSR1 signal to stop master if hasattr(signal, "SIGUSR1"): def sigusr1(*args): eventually(self.botmaster.cleanShutdown) signal.signal(signal.SIGUSR1, sigusr1) # get the masterid so other services can use it in # startup/reconfig. This goes directly to the DB since the data # API isn't initialized yet, and anyway, this method is aware of # the DB API since it just called its setup function self.masterid = yield self.db.masters.findMasterId( name=self.name) # mark this master as stopped, in case it crashed before yield self.data.updates.masterStopped(name=self.name, masterid=self.masterid) # call the parent method yield super().startService() # We make sure the housekeeping is done before configuring in order to cleanup # any remaining claimed schedulers or change sources from zombie # masters yield self.data.updates.expireMasters(forceHouseKeeping=True) # give all services a chance to load the new configuration, rather # than the base configuration yield self.reconfigServiceWithBuildbotConfig(self.config) # Mark the master as active now that mq is running yield self.data.updates.masterActive(name=self.name, masterid=self.masterid) # Start the heartbeat timer yield self.masterHeartbeatService.setServiceParent(self) # send the statistics to buildbot.net, without waiting self.sendBuildbotNetUsageData() startup_succeed = True except Exception: f = failure.Failure() log.err(f, 'while starting BuildMaster') self.reactor.stop() finally: if startup_succeed: log.msg("BuildMaster is running") else: log.msg("BuildMaster startup failed") yield self.initLock.release() self._master_initialized = True
def startService(self): assert not self._already_started, "can only start the master once" self._already_started = True log.msg("Starting BuildMaster -- buildbot.version: %s" % buildbot.version) # Set umask if self.umask is not None: os.umask(self.umask) # first, apply all monkeypatches monkeypatches.patch_all() # we want to wait until the reactor is running, so we can call # reactor.stop() for fatal errors d = defer.Deferred() self.reactor.callWhenRunning(d.callback, None) yield d startup_succeed = False try: # load the configuration file, treating errors as fatal try: # run the master.cfg in thread, so that it can use blocking # code self.config = yield threads.deferToThreadPool( self.reactor, self.reactor.getThreadPool(), self.config_loader.loadConfig) except config.ConfigErrors as e: log.msg("Configuration Errors:") for msg in e.errors: log.msg(" " + msg) log.msg("Halting master.") self.reactor.stop() return except Exception: log.err(failure.Failure(), 'while starting BuildMaster') self.reactor.stop() return # set up services that need access to the config before everything # else gets told to reconfig try: yield self.db.setup() except exceptions.DatabaseNotReadyError: # (message was already logged) self.reactor.stop() return self.mq.setup() if hasattr(signal, "SIGHUP"): def sighup(*args): eventually(self.reconfig) signal.signal(signal.SIGHUP, sighup) if hasattr(signal, "SIGUSR1"): def sigusr1(*args): self.reactor.callLater(0, self.botmaster.cleanShutdown) signal.signal(signal.SIGUSR1, sigusr1) # get the masterid so other services can use it in # startup/reconfig. This goes directly to the DB since the data # API isn't initialized yet, and anyway, this method is aware of # the DB API since it just called its setup function self.masterid = yield self.db.masters.findMasterId(name=self.name) # mark this master as stopped, in case it crashed before yield self.data.updates.masterStopped(name=self.name, masterid=self.masterid) # call the parent method yield service.AsyncMultiService.startService(self) # give all services a chance to load the new configuration, rather # than the base configuration yield self.reconfigServiceWithBuildbotConfig(self.config) # mark the master as active now that mq is running yield self.data.updates.masterActive(name=self.name, masterid=self.masterid) startup_succeed = True except Exception: f = failure.Failure() log.err(f, 'while starting BuildMaster') self.reactor.stop() finally: if startup_succeed: log.msg("BuildMaster is running") else: log.msg("BuildMaster startup failed") self._master_initialized = True
# # You should have received a copy of the GNU General Public License along with # this program; if not, write to the Free Software Foundation, Inc., 51 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # # Copyright Buildbot Team Members import sys import warnings from distutils.version import LooseVersion from buildbot import monkeypatches # apply the same patches the buildmaster does when it starts monkeypatches.patch_all(for_tests=True) # enable deprecation warnings warnings.filterwarnings('always', category=DeprecationWarning) if sys.version_info[:2] < (3, 2): # Setup logging unhandled messages to stderr. # Since Python 3.2 similar functionality implemented through # logging.lastResort handler. # Significant difference between this approach and Python 3.2 last resort # approach is that in the current approach only records with log level # equal or above to the root logger log level will be printed (WARNING by # default). For example, there still will be warnings about missing # handler for log if INFO or DEBUG records will be logged (but at least # WARNINGs and ERRORs will be printed). import logging
def cleanupDatabase(config, _noMonkey=False): # pragma: no cover # we separate the actual implementation to protect unit tests # from @in_reactor which stops the reactor if not _noMonkey: monkeypatches.patch_all() return _cleanupDatabase(config, _noMonkey=False)
from buildbot import monkeypatches from buildbot.test.util.warnings import assertProducesWarning # noqa pylint: disable=wrong-import-position from buildbot.test.util.warnings import assertProducesWarnings # noqa pylint: disable=wrong-import-position from buildbot.warnings import DeprecatedApiWarning # noqa pylint: disable=wrong-import-position # import mock so we bail out early if it's not installed try: import mock [mock] except ImportError as e: raise ImportError("\nBuildbot tests require the 'mock' module; " "try 'pip install mock'") from e # apply the same patches the buildmaster does when it starts monkeypatches.patch_all(for_tests=True) # enable deprecation warnings warnings.filterwarnings('always', category=DeprecationWarning) if parse_version(mock.__version__) < parse_version("0.8"): raise ImportError("\nBuildbot tests require mock version 0.8.0 or " "higher; try 'pip install -U mock'") [setuptools] # force use for pylint # This is where we load deprecated module-level APIs to ignore warning produced by importing them. # After the deprecated API has been removed, leave at least one instance of the import in a # commented state as reference. # with assertProducesWarnings(DeprecatedApiWarning,