def do_test_master(self): # create the master and set its config m = BuildMaster(self.basedir, self.configfile) m.config = config.MasterConfig.loadConfig( self.basedir, self.configfile) # update the DB yield m.db.setup(check_version=False) yield m.db.model.upgrade() # stub out m.db.setup since it was already called above m.db.setup = lambda : None # mock reactor.stop (which trial *really* doesn't # like test code to call!) mock_reactor = mock.Mock(spec=reactor) mock_reactor.callWhenRunning = reactor.callWhenRunning # start the service yield m.startService(_reactor=mock_reactor) self.failIf(mock_reactor.stop.called, "startService tried to stop the reactor; check logs") # stop the service yield m.stopService() # and shutdown the db threadpool, as is normally done at reactor stop m.db.pool.shutdown()
def do_test_master(self): # create the master and set its config m = BuildMaster(self.basedir, self.configfile) # update the DB yield m.db.setup(check_version=False) yield m.db.model.upgrade() # stub out m.db.setup since it was already called above m.db.setup = lambda: None # mock reactor.stop (which trial *really* doesn't # like test code to call!) mock_reactor = mock.Mock(spec=reactor) mock_reactor.callWhenRunning = reactor.callWhenRunning # start the service yield m.startService(_reactor=mock_reactor) self.failIf(mock_reactor.stop.called, "startService tried to stop the reactor; check logs") # hang out for a fraction of a second, to let startup processes run d = defer.Deferred() reactor.callLater(0.01, d.callback, None) yield d # stop the service yield m.stopService() # and shutdown the db threadpool, as is normally done at reactor stop m.db.pool.shutdown()
def upgradeDatabase(config, master_cfg): if not config['quiet']: print("upgrading database (%s)" % (stripUrlPassword(master_cfg.db['db_url']))) print("Warning: Stopping this process might cause data loss") def sighandler(signum, frame): msg = " ".join(""" WARNING: ignoring signal %s. This process should not be interrupted to avoid database corruption. If you really need to terminate it, use SIGKILL. """.split()) print(msg % signum) prev_handlers = {} try: for signame in ("SIGTERM", "SIGINT", "SIGQUIT", "SIGHUP", "SIGUSR1", "SIGUSR2", "SIGBREAK"): if hasattr(signal, signame): signum = getattr(signal, signame) prev_handlers[signum] = signal.signal(signum, sighandler) master = BuildMaster(config['basedir']) master.config = master_cfg master.db.disownServiceParent() db = connector.DBConnector(basedir=config['basedir']) db.setServiceParent(master) yield db.setup(check_version=False, verbose=not config['quiet']) yield db.model.upgrade() yield db.masters.setAllMastersActiveLongTimeAgo() finally: # restore previous signal handlers for signum, handler in prev_handlers.items(): signal.signal(signum, handler)
def do_test_master(self): # create the master and set its config m = BuildMaster(self.basedir, self.configfile) m.config = config.MasterConfig.loadConfig(self.basedir, self.configfile) # update the DB yield m.db.setup(check_version=False) yield m.db.model.upgrade() # stub out m.db.setup since it was already called above m.db.setup = lambda: None # mock reactor.stop (which trial *really* doesn't # like test code to call!) mock_reactor = mock.Mock(spec=reactor) mock_reactor.callWhenRunning = reactor.callWhenRunning # start the service yield m.startService(_reactor=mock_reactor) self.failIf(mock_reactor.stop.called, "startService tried to stop the reactor; check logs") # stop the service yield m.stopService() # and shutdown the db threadpool, as is normally done at reactor stop m.db.pool.shutdown()
def do_test_master(self): # create the master and set its config m = BuildMaster(self.basedir, self.configfile) # update the DB yield m.db.setup(check_version=False) yield m.db.model.upgrade() # stub out m.db.setup since it was already called above m.db.setup = lambda: None # mock reactor.stop (which trial *really* doesn't # like test code to call!) mock_reactor = mock.Mock(spec=reactor) mock_reactor.callWhenRunning = reactor.callWhenRunning mock_reactor.getThreadPool = reactor.getThreadPool mock_reactor.callFromThread = reactor.callFromThread # start the service yield m.startService(_reactor=mock_reactor) self.failIf(mock_reactor.stop.called, "startService tried to stop the reactor; check logs") # hang out for a fraction of a second, to let startup processes run d = defer.Deferred() reactor.callLater(0.01, d.callback, None) yield d # stop the service yield m.stopService() # and shutdown the db threadpool, as is normally done at reactor stop m.db.pool.shutdown()
def upgradeDatabase(config, master_cfg): if not config['quiet']: print("upgrading database (%s)" % (stripUrlPassword(master_cfg.db['db_url']))) print("Warning: Stopping this process might cause data loss") def sighandler(signum, frame): msg = " ".join(""" WARNING: ignoring signal %s. This process should not be interrupted to avoid database corruption. If you really need to terminate it, use SIGKILL. """.split()) print(msg % signum) for signame in ("SIGTERM", "SIGINT", "SIGQUIT", "SIGHUP", "SIGUSR1", "SIGUSR2", "SIGBREAK"): if hasattr(signal, signame): signal.signal(getattr(signal, signame), sighandler) master = BuildMaster(config['basedir']) master.config = master_cfg master.db.disownServiceParent() db = connector.DBConnector(basedir=config['basedir']) db.setServiceParent(master) yield db.setup(check_version=False, verbose=not config['quiet']) yield db.model.upgrade() yield db.masters.setAllMastersActiveLongTimeAgo()
def doCleanupDatabase(config, master_cfg): if not config['quiet']: print("cleaning database (%s)" % (master_cfg.db['db_url'])) master = BuildMaster(config['basedir']) master.config = master_cfg print(master.config.logCompressionMethod) db = master.db yield db.setup(check_version=False, verbose=not config['quiet']) res = yield db.logs.getLogs() i = 0 percent = 0 saved = 0 for log in res: saved += yield db.logs.compressLog(log['id']) i += 1 if not config['quiet'] and percent != i * 100 / len(res): percent = i * 100 / len(res) print(" {0}% {1} saved".format(percent, saved)) saved = 0 sys.stdout.flush() if master_cfg.db['db_url'].startswith("sqlite"): if not config['quiet']: print("executing sqlite vacuum function...") # sqlite vacuum function rebuild the whole database to claim # free disk space back def thd(engine): r = engine.execute("vacuum;") r.close() yield db.pool.do(thd)
def doCleanupDatabase(config, master_cfg): if not config["quiet"]: print("cleaning database (%s)" % (master_cfg.db["db_url"])) master = BuildMaster(config["basedir"]) master.config = master_cfg db = master.db yield db.setup(check_version=False, verbose=not config["quiet"]) res = yield db.logs.getLogs() i = 0 percent = 0 saved = 0 for log in res: saved += yield db.logs.compressLog(log["id"], force=config["force"]) i += 1 if not config["quiet"] and percent != i * 100 / len(res): percent = i * 100 / len(res) print(" {0}% {1} saved".format(percent, saved)) saved = 0 sys.stdout.flush() if master_cfg.db["db_url"].startswith("sqlite"): if not config["quiet"]: print("executing sqlite vacuum function...") # sqlite vacuum function rebuild the whole database to claim # free disk space back def thd(engine): r = engine.execute("vacuum;") r.close() yield db.pool.do(thd)
def getMaster(case, reactor, config_dict): """ Create a started ``BuildMaster`` with the given configuration. """ basedir = FilePath(case.mktemp()) basedir.createDirectory() config_dict['buildbotNetUsageData'] = None master = BuildMaster( basedir.path, reactor=reactor, config_loader=DictLoader(config_dict)) if 'db_url' not in config_dict: config_dict['db_url'] = 'sqlite://' # TODO: Allow BuildMaster to transparently upgrade the database, at least # for tests. master.config.db['db_url'] = config_dict['db_url'] yield master.db.setup(check_version=False) yield master.db.model.upgrade() master.db.setup = lambda: None yield master.startService() # and shutdown the db threadpool, as is normally done at reactor stop case.addCleanup(master.db.pool.shutdown) case.addCleanup(master.stopService) defer.returnValue(master)
def setupConfig(self, configFunc): """ Setup and start a master configured by the function configFunc defined in the test module. @type configFunc: string @param configFunc: name of a function without argument defined in the test module that returns a BuildmasterConfig object. """ self.basedir = os.path.abspath('basdir') self.setUpDirs(self.basedir) self.configfile = os.path.join(self.basedir, 'master.cfg') if self.proto == 'pb': proto = '{"pb": {"port": "tcp:0:interface=127.0.0.1"}}' elif self.proto == 'null': proto = '{"null": {}}' # We create a master.cfg, which loads the configuration from the # test module. Only the slave config is kept there, as it should not # be changed open(self.configfile, "w").write( textwrap.dedent(""" from buildbot.buildslave import BuildSlave from %s import %s c = BuildmasterConfig = %s() c['slaves'] = [BuildSlave("local1", "localpw")] c['protocols'] = %s """ % (self.__class__.__module__, configFunc, configFunc, proto))) # create the master and set its config m = BuildMaster(self.basedir, self.configfile) self.master = m # update the DB yield m.db.setup(check_version=False) yield m.db.model.upgrade() # stub out m.db.setup since it was already called above m.db.setup = lambda: None # mock reactor.stop (which trial *really* doesn't # like test code to call!) mock_reactor = mock.Mock(spec=reactor) mock_reactor.callWhenRunning = reactor.callWhenRunning # start the service yield m.startService(_reactor=mock_reactor) self.failIf(mock_reactor.stop.called, "startService tried to stop the reactor; check logs") if self.proto == 'pb': # We find out the slave port automatically slavePort = list(itervalues( m.pbmanager.dispatchers))[0].port.getHost().port # create a slave, and attach it to the master, it will be started, and stopped # along with the master s = BuildSlave("127.0.0.1", slavePort, "local1", "localpw", self.basedir, False, False) elif self.proto == 'null': s = LocalBuildSlave("local1", self.basedir, False) s.setServiceParent(m)
def setupConfig(self, configFunc): """ Setup and start a master configured by the function configFunc defined in the test module. @type configFunc: string @param configFunc: name of a function without argument defined in the test module that returns a BuildmasterConfig object. """ self.basedir = os.path.abspath('basdir') self.setUpDirs(self.basedir) self.configfile = os.path.join(self.basedir, 'master.cfg') if self.proto == 'pb': proto = '{"pb": {"port": "tcp:0:interface=127.0.0.1"}}' elif self.proto == 'null': proto = '{"null": {}}' # We create a master.cfg, which loads the configuration from the # test module. Only the slave config is kept there, as it should not # be changed open(self.configfile, "w").write(textwrap.dedent(""" from buildbot.buildslave import BuildSlave from %s import %s c = BuildmasterConfig = %s() c['slaves'] = [BuildSlave("local1", "localpw")] c['protocols'] = %s """ % (self.__class__.__module__, configFunc, configFunc, proto))) # create the master and set its config m = BuildMaster(self.basedir, self.configfile) self.master = m # update the DB yield m.db.setup(check_version=False) yield m.db.model.upgrade() # stub out m.db.setup since it was already called above m.db.setup = lambda: None # mock reactor.stop (which trial *really* doesn't # like test code to call!) mock_reactor = mock.Mock(spec=reactor) mock_reactor.callWhenRunning = reactor.callWhenRunning # start the service yield m.startService(_reactor=mock_reactor) self.failIf(mock_reactor.stop.called, "startService tried to stop the reactor; check logs") if self.proto == 'pb': # We find out the slave port automatically slavePort = list(itervalues(m.pbmanager.dispatchers))[0].port.getHost().port # create a slave, and attach it to the master, it will be started, and stopped # along with the master s = BuildSlave("127.0.0.1", slavePort, "local1", "localpw", self.basedir, False, False) elif self.proto == 'null': s = LocalBuildSlave("local1", self.basedir, False) s.setServiceParent(m)
def setupConfig(self, config_dict): """ Setup and start a master configured by the function configFunc defined in the test module. @type config_dict: dict @param configFunc: The BuildmasterConfig dictionary. """ self.basedir = os.path.abspath('basdir') self.setUpDirs(self.basedir) # mock reactor.stop (which trial *really* doesn't # like test code to call!) mock_reactor = mock.Mock(spec=reactor) mock_reactor.callWhenRunning = reactor.callWhenRunning mock_reactor.getThreadPool = reactor.getThreadPool mock_reactor.callFromThread = reactor.callFromThread workerclass = worker.Worker if self.proto == 'pb': proto = {"pb": {"port": "tcp:0:interface=127.0.0.1"}} elif self.proto == 'null': proto = {"null": {}} workerclass = worker.LocalWorker config_dict['workers'] = [workerclass("local1", "localpw")] config_dict['protocols'] = proto # create the master and set its config m = BuildMaster(self.basedir, reactor=mock_reactor, config_loader=DictLoader(config_dict)) self.master = m # update the DB yield m.db.setup(check_version=False) yield m.db.model.upgrade() # stub out m.db.setup since it was already called above m.db.setup = lambda: None # start the service yield m.startService() self.failIf(mock_reactor.stop.called, "startService tried to stop the reactor; check logs") if self.proto == 'pb': # We find out the worker port automatically workerPort = list(itervalues( m.pbmanager.dispatchers))[0].port.getHost().port # create a worker, and attach it to the master, it will be started, and stopped # along with the master self.w = BuildSlave("127.0.0.1", workerPort, "local1", "localpw", self.basedir, False, False) elif self.proto == 'null': self.w = None if self.w is not None: self.w.setServiceParent(m)
def getMaster(self, config_dict): """ Create a started ``BuildMaster`` with the given configuration. """ basedir = FilePath(self.mktemp()) basedir.createDirectory() master = BuildMaster( basedir.path, reactor=reactor, config_loader=DictLoader(config_dict)) master.config = master.config_loader.loadConfig() return master
def upgradeDatabase(config, master_cfg): if not config['quiet']: print "upgrading database (%s)" % (master_cfg.db['db_url']) master = BuildMaster(config['basedir']) master.config = master_cfg db = connector.DBConnector(master, basedir=config['basedir']) yield db.setup(check_version=False, verbose=not config['quiet']) yield db.model.upgrade()
def testSteps(self): m = BuildMaster(".") m.loadConfig(cfg1) b = m.botmaster.builders["builder1"] steps = b.buildFactory.steps self.failUnlessEqual(len(steps), 4) self.failUnlessExpectedShell(steps[0], command="echo yes") self.failUnlessExpectedShell(steps[1], defaults=False, command="old-style") self.failUnlessExpectedDarcs(steps[2], repourl="http://buildbot.net/repos/trunk") self.failUnlessExpectedShell(steps[3], defaults=False, command="echo old-style")
def check_master_cfg(self): """Check the buildmaster configuration, returning a deferred that fires with an approprate exit status (so 0=success).""" from buildbot.master import BuildMaster from twisted.python import log, failure master_cfg = os.path.join(self.basedir, "master.cfg") if not os.path.exists(master_cfg): if not self.quiet: print "No master.cfg found" return defer.succeed(1) # side-effects of loading the config file: # for each Builder defined in c['builders'], if the status directory # didn't already exist, it will be created, and the # $BUILDERNAME/builder pickle might be created (with a single # "builder created" event). # we put basedir in front of sys.path, because that's how the # buildmaster itself will run, and it is quite common to have the # buildmaster import helper classes from other .py files in its # basedir. if sys.path[0] != self.basedir: sys.path.insert(0, self.basedir) m = BuildMaster(self.basedir) # we need to route log.msg to stdout, so any problems can be seen # there. But if everything goes well, I'd rather not clutter stdout # with log messages. So instead we add a logObserver which gathers # messages and only displays them if something goes wrong. messages = [] log.addObserver(messages.append) try: # this will raise an exception if there's something wrong with # the config file. Note that this BuildMaster instance is never # started, so it won't actually do anything with the # configuration. return m.loadConfig(open(master_cfg, "r"), checkOnly=True) except: f = failure.Failure() if not self.quiet: print for m in messages: print "".join(m['message']) print f print print "An error was detected in the master.cfg file." print "Please correct the problem and run 'buildbot upgrade-master' again." print return 1 return 0
def upgradeDatabase(config, master_cfg): if not config['quiet']: print("upgrading database (%s)" % (master_cfg.db['db_url'])) master = BuildMaster(config['basedir']) master.config = master_cfg master.db.disownServiceParent() db = connector.DBConnector(basedir=config['basedir']) db.setServiceParent(master) yield db.setup(check_version=False, verbose=not config['quiet']) yield db.model.upgrade() yield db.masters.setAllMastersActiveLongTimeAgo()
def check_master_cfg(self): from buildbot.master import BuildMaster from twisted.python import log, failure master_cfg = os.path.join(self.basedir, "master.cfg") if not os.path.exists(master_cfg): if not self.quiet: print "No master.cfg found" return 1 # side-effects of loading the config file: # for each Builder defined in c['builders'], if the status directory # didn't already exist, it will be created, and the # $BUILDERNAME/builder pickle might be created (with a single # "builder created" event). # we put basedir in front of sys.path, because that's how the # buildmaster itself will run, and it is quite common to have the # buildmaster import helper classes from other .py files in its # basedir. if sys.path[0] != self.basedir: sys.path.insert(0, self.basedir) m = BuildMaster(self.basedir) # we need to route log.msg to stdout, so any problems can be seen # there. But if everything goes well, I'd rather not clutter stdout # with log messages. So instead we add a logObserver which gathers # messages and only displays them if something goes wrong. messages = [] log.addObserver(messages.append) try: # this will raise an exception if there's something wrong with # the config file. Note that this BuildMaster instance is never # started, so it won't actually do anything with the # configuration. m.loadConfig(open(master_cfg, "r")) except: f = failure.Failure() if not self.quiet: print for m in messages: print "".join(m['message']) print f print print "An error was detected in the master.cfg file." print "Please correct the problem and run 'buildbot upgrade-master' again." print return 1 return 0
def upgradeDatabase(config, master_cfg): if not config['quiet']: print("upgrading database (%s)" % (stripUrlPassword(master_cfg.db['db_url']))) print("Warning: Stopping this process might cause data loss") master = BuildMaster(config['basedir']) master.config = master_cfg master.db.disownServiceParent() db = connector.DBConnector(basedir=config['basedir']) db.setServiceParent(master) yield db.setup(check_version=False, verbose=not config['quiet']) yield db.model.upgrade() yield db.masters.setAllMastersActiveLongTimeAgo()
def createDB(config, _noMonkey=False): # apply the db monkeypatches (and others - no harm) if not _noMonkey: # pragma: no cover monkeypatches.patch_all() # create a master with the default configuration, but with db_url # overridden master_cfg = config_module.MasterConfig() master_cfg.db['db_url'] = config['db'] master = BuildMaster(config['basedir']) master.config = master_cfg db = master.db yield db.setup(check_version=False, verbose=not config['quiet']) if not config['quiet']: print("creating database (%s)" % (master_cfg.db['db_url'],)) yield db.model.upgrade()
def createDB(config, _noMonkey=False): # apply the db monkeypatches (and others - no harm) if not _noMonkey: # pragma: no cover monkeypatches.patch_all() # create a master with the default configuration, but with db_url # overridden master_cfg = config_module.MasterConfig() master_cfg.db['db_url'] = config['db'] master = BuildMaster(config['basedir']) master.config = master_cfg db = connector.DBConnector(master, config['basedir']) yield db.setup(check_version=False, verbose=not config['quiet']) if not config['quiet']: print "creating database (%s)" % (master_cfg.db['db_url'],) yield db.model.upgrade()
def create_db(self): from buildbot.db import connector from buildbot.master import BuildMaster db = connector.DBConnector(BuildMaster(self.basedir), self.config['db'], basedir=self.basedir) if not self.config['quiet']: print "creating database" d = db.model.upgrade() return d
def create_db(self): from buildbot.db import connector from buildbot.master import BuildMaster from buildbot import config as config_module # create a master with the default configuration, but with db_url # overridden master_cfg = config_module.MasterConfig() master_cfg.db['db_url'] = self.config['db'] master = BuildMaster(self.basedir) master.config = master_cfg db = connector.DBConnector(master, self.basedir) d = db.setup(check_version=False) if not self.config['quiet']: print "creating database (%s)" % (master_cfg.db['db_url'], ) d = db.model.upgrade() return d
def testStartService(self): os.mkdir("test_ss") self.master = m = BuildMaster("test_ss") # inhibit the usual read-config-on-startup behavior m.readConfig = True m.startService() d = m.loadConfig(startableEmptyCfg % 0) d.addCallback(self._testStartService_0) return d
def setUp(self): self.basedir = os.path.abspath('basdir') self.setUpDirs(self.basedir) self.configfile = os.path.join(self.basedir, 'master.cfg') # We create a master.cfg, which loads the configuration from the # test module. Only the slave config is kept there, as it should not # be changed open(self.configfile, "w").write(textwrap.dedent(""" from buildbot.buildslave import BuildSlave from %s import masterConfig c = BuildmasterConfig = masterConfig() c['slaves'] = [BuildSlave("local1", "localpw")] c['protocols'] = {"pb": {"port": "tcp:0:interface=127.0.0.1"}} """ % self.__class__.__module__)) # create the master and set its config m = BuildMaster(self.basedir, self.configfile) self.master = m # update the DB yield m.db.setup(check_version=False) yield m.db.model.upgrade() # stub out m.db.setup since it was already called above m.db.setup = lambda: None # mock reactor.stop (which trial *really* doesn't # like test code to call!) mock_reactor = mock.Mock(spec=reactor) mock_reactor.callWhenRunning = reactor.callWhenRunning # start the service yield m.startService(_reactor=mock_reactor) self.failIf(mock_reactor.stop.called, "startService tried to stop the reactor; check logs") # We find out the slave port automatically slavePort = m.pbmanager.dispatchers.values()[0].port.getHost().port # create a slave, and attach it to the master, it will be started, and stopped # along with the master s = BuildSlave("127.0.0.1", slavePort, "local1", "localpw", self.basedir, False, False) s.setServiceParent(m)
def testFindConfigFile(self): os.mkdir("test_cf") open(os.path.join("test_cf", "master.cfg"), "w").write(emptyCfg) slaveportCfg = emptyCfg + "c['slavePortnum'] = 9000\n" open(os.path.join("test_cf", "alternate.cfg"), "w").write(slaveportCfg) m = BuildMaster("test_cf") m.loadTheConfigFile() self.failUnlessEqual(m.slavePortnum, "tcp:9999") m = BuildMaster("test_cf", "alternate.cfg") m.loadTheConfigFile() self.failUnlessEqual(m.slavePortnum, "tcp:9000")
def getMaster(case, reactor, config_dict): """ Create a started ``BuildMaster`` with the given configuration. """ basedir = FilePath(case.mktemp()) basedir.createDirectory() master = BuildMaster(basedir.path, reactor=reactor, config_loader=DictLoader(config_dict)) if 'db_url' not in config_dict: config_dict['db_url'] = 'sqlite://' # TODO: Allow BuildMaster to transparently upgrade the database, at least for tests. master.config.db['db_url'] = config_dict['db_url'] yield master.db.setup(check_version=False) yield master.db.model.upgrade() master.db.setup = lambda: None yield master.startService() defer.returnValue(master)
def doCleanupDatabase(config, master_cfg): if not config['quiet']: print("cleaning database (%s)" % (master_cfg.db['db_url'])) master = BuildMaster(config['basedir']) master.config = master_cfg db = master.db yield db.setup(check_version=False, verbose=not config['quiet']) res = yield db.logs.getLogs() i = 0 percent = 0 saved = 0 for log in res: saved += yield db.logs.compressLog(log['id'], force=config['force']) i += 1 if not config['quiet'] and percent != i * 100 / len(res): percent = i * 100 / len(res) print(" {0}% {1} saved".format(percent, saved)) saved = 0 sys.stdout.flush() if master_cfg.db['db_url'].startswith("sqlite"): if not config['quiet']: print("executing sqlite vacuum function...") # sqlite vacuum function rebuild the whole database to claim # free disk space back def thd(engine): # In Python 3.6 and higher, sqlite3 no longer commits an # open transaction before DDL statements. # It is necessary to set the isolation_level to none # for auto-commit mode before doing a VACUUM. # See: https://bugs.python.org/issue28518 # Get the underlying sqlite connection from SQLAlchemy. sqlite_conn = engine.connection.connection # Set isolation_level to 'auto-commit mode' sqlite_conn.isolation_level = None sqlite_conn.execute("vacuum;").close() yield db.pool.do(thd)
def populate_database(config): master = BuildMaster(config['baseDir']) master.config = load_config(config, config['configFile']) db = connector.DBConnector(master, basedir=config['baseDir']) seed = int(time()) if config['seed']: seed = int(config['seed']) random.seed(seed) if not config['quiet']: print("Seed =", seed) yield db.setup(check_version=False, verbose=not config['quiet']) users = yield populate_user(db, int(config['users']), verbose=not config['quiet']) yield populate_build(db, int(config['builds']), master.config.builders, master.config.projects, users, verbose=not config['quiet'])
def _run_master(self, loaded_config): # create the master m = BuildMaster(self.basedir, self.configfile) # update the DB yield m.db.setup(check_version=False) yield m.db.model.upgrade() # stub out m.db.setup since it was already called above m.db.setup = lambda: None # mock reactor.stop (which trial *really* doesn't # like test code to call!) mock_reactor = mock.Mock(spec=reactor) mock_reactor.callWhenRunning = reactor.callWhenRunning mock_reactor.getThreadPool = reactor.getThreadPool mock_reactor.callFromThread = reactor.callFromThread # mock configuration loading @classmethod def loadConfig(cls, basedir, filename): return loaded_config with mock.patch('buildbot.config.MasterConfig.loadConfig', loadConfig): # start the service yield m.startService(_reactor=mock_reactor) self.failIf(mock_reactor.stop.called, "startService tried to stop the reactor; check logs") # hang out for a fraction of a second, to let startup processes run d = defer.Deferred() reactor.callLater(0.01, d.callback, None) yield d # stop the service yield m.stopService() # and shutdown the db threadpool, as is normally done at reactor stop m.db.pool.shutdown()
def upgradeMaster(config): m = Maker(config) if not config['quiet']: print "upgrading basedir" basedir = os.path.expanduser(config['basedir']) # TODO: check Makefile # TODO: check TAC file # check web files: index.html, default.css, robots.txt m.upgrade_public_html({ 'bg_gradient.jpg': util.sibpath(__file__, "../status/web/files/bg_gradient.jpg"), 'default.css': util.sibpath(__file__, "../status/web/files/default.css"), 'robots.txt': util.sibpath(__file__, "../status/web/files/robots.txt"), 'favicon.ico': util.sibpath(__file__, "../status/web/files/favicon.ico"), }) m.populate_if_missing(os.path.join(basedir, "master.cfg.sample"), util.sibpath(__file__, "sample.cfg"), overwrite=True) # if index.html exists, use it to override the root page tempalte m.move_if_present(os.path.join(basedir, "public_html/index.html"), os.path.join(basedir, "templates/root.html")) if not config['quiet']: print "checking master.cfg" wfd = defer.waitForDeferred( m.check_master_cfg(expected_db_url=config['db'])) yield wfd rc = wfd.getResult() if rc == 0: from buildbot.db import connector from buildbot.master import BuildMaster if not config['quiet']: print "upgrading database" db = connector.DBConnector(BuildMaster(config['basedir']), config['db'], basedir=config['basedir']) wfd = defer.waitForDeferred(db.model.upgrade()) yield wfd wfd.getResult() if not config['quiet']: print "upgrade complete" yield 0 else: yield rc
def __init__(self, config, reactor=None, source='TestMaster', log_handler=None, attach_on=tuple()): """Lightweight in-process BuildMaster Spins up a lightweight BuildMaster in the same process and can trigger builders defined in the configuration. The TestMaster only pays attention to the `workers`, `builders` and `schedulers` configuration keys, so it doesn't configure non-essential services like the reporters. It is used in the CLI interface to locally reproduce specific builds, but it is also suitable for general integration testing of the builders. Parameters ---------- config: MasterConfig reactor: twisted.reactor, default None source: str, default `TestMaster` Used for highligting the origin or the build properties. log_handler: Callable[[unseen_log_lines], None], default lambda _: None A callback to handle the logs produced by the builder's buildsteps. attach_on: List[Results], default [] If a build finishes with any of the listed states and it is executed withing a DockerLatentWorker then start an interactive shell session in the container. Use it with caution, because it blocks the event loop. """ assert isinstance(config, MasterConfig) assert all(result in ALL_RESULTS for result in attach_on) self.config = config self.attach_on = set(attach_on) loader = EagerLoader(config, source=source) if reactor is None: from twisted.internet import reactor self._source = source self._master = BuildMaster('.', reactor=reactor, config_loader=loader) self._log_handler = log_handler or (lambda _: None) # state variable updated by the event handlers below self._buildset = None self._buildset_id = None self._log_offset = 0
def check_master_cfg(self, expected_db_url=None): """Check the buildmaster configuration, returning a deferred that fires with an approprate exit status (so 0=success).""" from buildbot.master import BuildMaster from twisted.python import log master_cfg = os.path.join(self.basedir, "master.cfg") if not os.path.exists(master_cfg): if not self.quiet: print "No master.cfg found" return defer.succeed(1) # side-effects of loading the config file: # for each Builder defined in c['builders'], if the status directory # didn't already exist, it will be created, and the # $BUILDERNAME/builder pickle might be created (with a single # "builder created" event). # we put basedir in front of sys.path, because that's how the # buildmaster itself will run, and it is quite common to have the # buildmaster import helper classes from other .py files in its # basedir. if sys.path[0] != self.basedir: sys.path.insert(0, self.basedir) m = BuildMaster(self.basedir) # we need to route log.msg to stdout, so any problems can be seen # there. But if everything goes well, I'd rather not clutter stdout # with log messages. So instead we add a logObserver which gathers # messages and only displays them if something goes wrong. messages = [] log.addObserver(messages.append) # this will errback if there's something wrong with the config file. # Note that this BuildMaster instance is never started, so it won't # actually do anything with the configuration. d = defer.maybeDeferred(lambda : m.loadConfig(open(master_cfg, "r"), checkOnly=True)) def check_db_url(config): if (expected_db_url and config.get('db_url', 'sqlite:///state.sqlite') != expected_db_url): raise ValueError("c['db_url'] in the config file ('%s') does" " not match '%s'; please edit the configuration" " file before upgrading." % (config['db_url'], expected_db_url)) d.addCallback(check_db_url) def cb(_): return 0 def eb(f): if not self.quiet: print for m in messages: print "".join(m['message']) f.printTraceback() print print "An error was detected in the master.cfg file." print "Please correct the problem and run 'buildbot upgrade-master' again." print return 1 d.addCallbacks(cb, eb) return d
def setupConfig(self, configFunc): """ Setup and start a master configured by the function configFunc defined in the test module. @type configFunc: string @param configFunc: name of a function without argument defined in the test module that returns a BuildmasterConfig object. """ self.basedir = os.path.abspath("basdir") self.setUpDirs(self.basedir) self.configfile = os.path.join(self.basedir, "master.cfg") workerclass = "BuildWorker" if self.proto == "pb": proto = '{"pb": {"port": "tcp:0:interface=127.0.0.1"}}' elif self.proto == "null": proto = '{"null": {}}' workerclass = "LocalBuildWorker" # We create a master.cfg, which loads the configuration from the # test module. Only the worker config is kept there, as it should not # be changed open(self.configfile, "w").write( textwrap.dedent( """ from buildbot.plugins import buildworker from {module} import {configFunc} c = BuildmasterConfig = {configFunc}() c['workers'] = [buildworker.{workerclass}("local1", "localpw")] c['protocols'] = {proto} """ ).format(module=self.__class__.__module__, configFunc=configFunc, proto=proto, workerclass=workerclass) ) # create the master and set its config m = BuildMaster(self.basedir, self.configfile) self.master = m # update the DB yield m.db.setup(check_version=False) yield m.db.model.upgrade() # stub out m.db.setup since it was already called above m.db.setup = lambda: None # mock reactor.stop (which trial *really* doesn't # like test code to call!) mock_reactor = mock.Mock(spec=reactor) mock_reactor.callWhenRunning = reactor.callWhenRunning # start the service yield m.startService(_reactor=mock_reactor) self.failIf(mock_reactor.stop.called, "startService tried to stop the reactor; check logs") if self.proto == "pb": # We find out the worker port automatically workerPort = list(itervalues(m.pbmanager.dispatchers))[0].port.getHost().port # create a worker, and attach it to the master, it will be started, and stopped # along with the master s = BuildWorker("127.0.0.1", workerPort, "local1", "localpw", self.basedir, False, False) elif self.proto == "null": s = None if s is not None: s.setServiceParent(m)
class ConfigTest(unittest.TestCase): def setUp(self): # this class generates several deprecation warnings, which the user # doesn't need to see. warnings.simplefilter('ignore', exceptions.DeprecationWarning) self.buildmaster = BuildMaster(".") def failUnlessListsEquivalent(self, list1, list2): l1 = list1[:] l1.sort() l2 = list2[:] l2.sort() self.failUnlessEqual(l1, l2) def servers(self, s, types): # perform a recursive search of s.services, looking for instances of # twisted.application.internet.TCPServer, then extract their .args # values to find the TCP ports they want to listen on for child in s: if service.IServiceCollection.providedBy(child): for gc in self.servers(child, types): yield gc if isinstance(child, types): yield child def TCPports(self, s): return list(self.servers(s, internet.TCPServer)) def UNIXports(self, s): return list(self.servers(s, internet.UNIXServer)) def TCPclients(self, s): return list(self.servers(s, internet.TCPClient)) def checkPorts(self, svc, expected): """Verify that the TCPServer and UNIXServer children of the given service have the expected portnum/pathname and factory classes. As a side-effect, return a list of servers in the same order as the 'expected' list. This can be used to verify properties of the factories contained therein.""" expTCP = [e for e in expected if type(e[0]) == int] expUNIX = [e for e in expected if type(e[0]) == str] haveTCP = [(p.args[0], p.args[1].__class__) for p in self.TCPports(svc)] haveUNIX = [(p.args[0], p.args[1].__class__) for p in self.UNIXports(svc)] self.failUnlessListsEquivalent(expTCP, haveTCP) self.failUnlessListsEquivalent(expUNIX, haveUNIX) ret = [] for e in expected: for have in self.TCPports(svc) + self.UNIXports(svc): if have.args[0] == e[0]: ret.append(have) continue assert (len(ret) == len(expected)) return ret def testEmpty(self): self.failUnlessRaises(KeyError, self.buildmaster.loadConfig, "") def testSimple(self): # covers slavePortnum, base checker passwords master = self.buildmaster master.loadChanges() master.loadConfig(emptyCfg) # note: this doesn't actually start listening, because the app # hasn't been started running self.failUnlessEqual(master.slavePortnum, "tcp:9999") self.checkPorts(master, [(9999, pb.PBServerFactory)]) self.failUnlessEqual(list(master.change_svc), []) self.failUnlessEqual(master.botmaster.builders, {}) self.failUnlessEqual(master.checker.users, {"change": "changepw"}) self.failUnlessEqual(master.projectName, "dummy project") self.failUnlessEqual(master.projectURL, "http://dummy.example.com") self.failUnlessEqual(master.buildbotURL, "http://dummy.example.com/buildbot") def testSlavePortnum(self): master = self.buildmaster master.loadChanges() master.loadConfig(emptyCfg) self.failUnlessEqual(master.slavePortnum, "tcp:9999") ports = self.checkPorts(master, [(9999, pb.PBServerFactory)]) p = ports[0] master.loadConfig(emptyCfg) self.failUnlessEqual(master.slavePortnum, "tcp:9999") ports = self.checkPorts(master, [(9999, pb.PBServerFactory)]) self.failUnlessIdentical(p, ports[0], "the slave port was changed even " + \ "though the configuration was not") master.loadConfig(emptyCfg + "c['slavePortnum'] = 9000\n") self.failUnlessEqual(master.slavePortnum, "tcp:9000") ports = self.checkPorts(master, [(9000, pb.PBServerFactory)]) self.failIf(p is ports[0], "slave port was unchanged but configuration was changed") def testSlaves(self): master = self.buildmaster master.loadChanges() master.loadConfig(emptyCfg) self.failUnlessEqual(master.botmaster.builders, {}) self.failUnlessEqual(master.checker.users, {"change": "changepw"}) # 'botsCfg' is testing backwards compatibility, for 0.7.5 config # files that have not yet been updated to 0.7.6 . This compatibility # (and this test) is scheduled for removal in 0.8.0 . botsCfg = (emptyCfg + "c['bots'] = [('bot1', 'pw1'), ('bot2', 'pw2')]\n") master.loadConfig(botsCfg) self.failUnlessEqual(master.checker.users, { "change": "changepw", "bot1": "pw1", "bot2": "pw2" }) master.loadConfig(botsCfg) self.failUnlessEqual(master.checker.users, { "change": "changepw", "bot1": "pw1", "bot2": "pw2" }) master.loadConfig(emptyCfg) self.failUnlessEqual(master.checker.users, {"change": "changepw"}) slavesCfg = (emptyCfg + "from buildbot.buildslave import BuildSlave\n" "c['slaves'] = [BuildSlave('bot1','pw1'), " "BuildSlave('bot2','pw2')]\n") master.loadConfig(slavesCfg) self.failUnlessEqual(master.checker.users, { "change": "changepw", "bot1": "pw1", "bot2": "pw2" }) def testChangeSource(self): master = self.buildmaster master.loadChanges() master.loadConfig(emptyCfg) self.failUnlessEqual(list(master.change_svc), []) sourcesCfg = emptyCfg + \ """ from buildbot.changes.pb import PBChangeSource c['change_source'] = PBChangeSource() """ d = master.loadConfig(sourcesCfg) def _check1(res): self.failUnlessEqual(len(list(self.buildmaster.change_svc)), 1) s1 = list(self.buildmaster.change_svc)[0] self.failUnless(isinstance(s1, PBChangeSource)) self.failUnlessEqual(s1, list(self.buildmaster.change_svc)[0]) self.failUnless(s1.parent) # verify that unchanged sources are not interrupted d1 = self.buildmaster.loadConfig(sourcesCfg) def _check2(res): self.failUnlessEqual(len(list(self.buildmaster.change_svc)), 1) s2 = list(self.buildmaster.change_svc)[0] self.failUnlessIdentical(s1, s2) self.failUnless(s1.parent) d1.addCallback(_check2) return d1 d.addCallback(_check1) # make sure we can get rid of the sources too d.addCallback(lambda res: self.buildmaster.loadConfig(emptyCfg)) def _check3(res): self.failUnlessEqual(list(self.buildmaster.change_svc), []) d.addCallback(_check3) return d def testChangeSources(self): # make sure we can accept a list master = self.buildmaster master.loadChanges() master.loadConfig(emptyCfg) self.failUnlessEqual(list(master.change_svc), []) sourcesCfg = emptyCfg + \ """ from buildbot.changes.pb import PBChangeSource from buildbot.changes.mail import SyncmailMaildirSource c['change_source'] = [PBChangeSource(), SyncmailMaildirSource('.'), ] """ d = master.loadConfig(sourcesCfg) def _check1(res): self.failUnlessEqual(len(list(self.buildmaster.change_svc)), 2) s1, s2 = list(self.buildmaster.change_svc) if isinstance(s2, PBChangeSource): s1, s2 = s2, s1 self.failUnless(isinstance(s1, PBChangeSource)) self.failUnless(s1.parent) self.failUnless(isinstance(s2, SyncmailMaildirSource)) self.failUnless(s2.parent) d.addCallback(_check1) return d def testSources(self): # test backwards compatibility. c['sources'] is deprecated. master = self.buildmaster master.loadChanges() master.loadConfig(emptyCfg) self.failUnlessEqual(list(master.change_svc), []) sourcesCfg = emptyCfg + \ """ from buildbot.changes.pb import PBChangeSource c['sources'] = [PBChangeSource()] """ d = master.loadConfig(sourcesCfg) def _check1(res): self.failUnlessEqual(len(list(self.buildmaster.change_svc)), 1) s1 = list(self.buildmaster.change_svc)[0] self.failUnless(isinstance(s1, PBChangeSource)) self.failUnless(s1.parent) d.addCallback(_check1) return d def shouldBeFailure(self, res, *expected): self.failUnless(isinstance(res, failure.Failure), "we expected this to fail, not produce %s" % (res, )) res.trap(*expected) return None # all is good def testSchedulerErrors(self): master = self.buildmaster master.loadChanges() master.loadConfig(emptyCfg) self.failUnlessEqual(master.allSchedulers(), []) def _shouldBeFailure(res, hint=None): self.shouldBeFailure(res, AssertionError, ValueError) if hint: self.failUnless(str(res).find(hint) != -1) def _loadConfig(res, newcfg): return self.buildmaster.loadConfig(newcfg) d = defer.succeed(None) # c['schedulers'] must be a list badcfg = schedulersCfg + \ """ c['schedulers'] = Scheduler('full', None, 60, ['builder1']) """ d.addCallback(_loadConfig, badcfg) d.addBoth(_shouldBeFailure, "c['schedulers'] must be a list of Scheduler instances") # c['schedulers'] must be a list of IScheduler objects badcfg = schedulersCfg + \ """ c['schedulers'] = ['oops', 'problem'] """ d.addCallback(_loadConfig, badcfg) d.addBoth(_shouldBeFailure, "c['schedulers'] must be a list of Scheduler instances") # c['schedulers'] must point at real builders badcfg = schedulersCfg + \ """ c['schedulers'] = [Scheduler('full', None, 60, ['builder-bogus'])] """ d.addCallback(_loadConfig, badcfg) d.addBoth(_shouldBeFailure, "uses unknown builder") # builderNames= must be a list badcfg = schedulersCfg + \ """ c['schedulers'] = [Scheduler('full', None, 60, 'builder1')] """ d.addCallback(_loadConfig, badcfg) d.addBoth(_shouldBeFailure, "must be a list of Builder description names") # builderNames= must be a list of strings, not dicts badcfg = schedulersCfg + \ """ c['schedulers'] = [Scheduler('full', None, 60, [b1])] """ d.addCallback(_loadConfig, badcfg) d.addBoth(_shouldBeFailure, "must be a list of Builder description names") # builderNames= must be a list of strings, not a dict badcfg = schedulersCfg + \ """ c['schedulers'] = [Scheduler('full', None, 60, b1)] """ d.addCallback(_loadConfig, badcfg) d.addBoth(_shouldBeFailure, "must be a list of Builder description names") # each Scheduler must have a unique name badcfg = schedulersCfg + \ """ c['schedulers'] = [Scheduler('dup', None, 60, []), Scheduler('dup', None, 60, [])] """ d.addCallback(_loadConfig, badcfg) d.addBoth(_shouldBeFailure, "Schedulers must have unique names") return d def testSchedulers(self): master = self.buildmaster master.loadChanges() master.loadConfig(emptyCfg) self.failUnlessEqual(master.allSchedulers(), []) d = self.buildmaster.loadConfig(schedulersCfg) d.addCallback(self._testSchedulers_1) return d def _testSchedulers_1(self, res): sch = self.buildmaster.allSchedulers() self.failUnlessEqual(len(sch), 1) s = sch[0] self.failUnless(isinstance(s, scheduler.Scheduler)) self.failUnlessEqual(s.name, "full") self.failUnlessEqual(s.branch, None) self.failUnlessEqual(s.treeStableTimer, 60) self.failUnlessEqual(s.builderNames, ['builder1']) newcfg = schedulersCfg + \ """ s1 = Scheduler('full', None, 60, ['builder1']) c['schedulers'] = [s1, Dependent('downstream', s1, ['builder1'])] """ d = self.buildmaster.loadConfig(newcfg) d.addCallback(self._testSchedulers_2, newcfg) return d def _testSchedulers_2(self, res, newcfg): sch = self.buildmaster.allSchedulers() self.failUnlessEqual(len(sch), 2) s = sch[0] self.failUnless(isinstance(s, scheduler.Scheduler)) s = sch[1] self.failUnless(isinstance(s, scheduler.Dependent)) self.failUnlessEqual(s.name, "downstream") self.failUnlessEqual(s.builderNames, ['builder1']) # reloading the same config file should leave the schedulers in place d = self.buildmaster.loadConfig(newcfg) d.addCallback(self._testSchedulers_3, sch) return d def _testSchedulers_3(self, res, sch1): sch2 = self.buildmaster.allSchedulers() self.failUnlessEqual(len(sch2), 2) sch1.sort() sch2.sort() self.failUnlessEqual(sch1, sch2) self.failUnlessIdentical(sch1[0], sch2[0]) self.failUnlessIdentical(sch1[1], sch2[1]) self.failUnlessIdentical(sch1[0].parent, self.buildmaster) self.failUnlessIdentical(sch1[1].parent, self.buildmaster) def testBuilders(self): master = self.buildmaster master.loadConfig(emptyCfg) self.failUnlessEqual(master.botmaster.builders, {}) master.loadConfig(buildersCfg) self.failUnlessEqual(master.botmaster.builderNames, ["builder1"]) self.failUnlessEqual(master.botmaster.builders.keys(), ["builder1"]) b = master.botmaster.builders["builder1"] self.failUnless(isinstance(b, Builder)) self.failUnlessEqual(b.name, "builder1") self.failUnlessEqual(b.slavenames, ["bot1"]) self.failUnlessEqual(b.builddir, "workdir") f1 = b.buildFactory self.failUnless(isinstance(f1, BasicBuildFactory)) steps = f1.steps self.failUnlessEqual(len(steps), 3) self.failUnlessEqual(steps[0], (CVS, { 'cvsroot': 'cvsroot', 'cvsmodule': 'cvsmodule', 'mode': 'clobber' })) self.failUnlessEqual(steps[1], (Compile, {'command': 'make all'})) self.failUnlessEqual(steps[2], (Test, {'command': 'make check'})) # make sure a reload of the same data doesn't interrupt the Builder master.loadConfig(buildersCfg) self.failUnlessEqual(master.botmaster.builderNames, ["builder1"]) self.failUnlessEqual(master.botmaster.builders.keys(), ["builder1"]) b2 = master.botmaster.builders["builder1"] self.failUnlessIdentical(b, b2) # TODO: test that the BuilderStatus object doesn't change #statusbag2 = master.client_svc.statusbags["builder1"] #self.failUnlessIdentical(statusbag, statusbag2) # but changing something should result in a new Builder master.loadConfig(buildersCfg2) self.failUnlessEqual(master.botmaster.builderNames, ["builder1"]) self.failUnlessEqual(master.botmaster.builders.keys(), ["builder1"]) b3 = master.botmaster.builders["builder1"] self.failIf(b is b3) # the statusbag remains the same TODO #statusbag3 = master.client_svc.statusbags["builder1"] #self.failUnlessIdentical(statusbag, statusbag3) # adding new builder master.loadConfig(buildersCfg3) self.failUnlessEqual(master.botmaster.builderNames, ["builder1", "builder2"]) self.failUnlessListsEquivalent(master.botmaster.builders.keys(), ["builder1", "builder2"]) b4 = master.botmaster.builders["builder1"] self.failUnlessIdentical(b3, b4) # changing first builder should leave it at the same place in the list master.loadConfig(buildersCfg4) self.failUnlessEqual(master.botmaster.builderNames, ["builder1", "builder2"]) self.failUnlessListsEquivalent(master.botmaster.builders.keys(), ["builder1", "builder2"]) b5 = master.botmaster.builders["builder1"] self.failIf(b4 is b5) master.loadConfig(buildersCfg5) self.failUnlessEqual(master.botmaster.builderNames, ["builder1", "builder2"]) self.failUnlessListsEquivalent(master.botmaster.builders.keys(), ["builder1", "builder2"]) b5 = master.botmaster.builders["builder1"] # and removing it should make the Builder go away master.loadConfig(emptyCfg) self.failUnlessEqual(master.botmaster.builderNames, []) self.failUnlessEqual(master.botmaster.builders, {}) #self.failUnlessEqual(master.client_svc.statusbags, {}) # TODO def testWithProperties(self): master = self.buildmaster master.loadConfig(wpCfg1) self.failUnlessEqual(master.botmaster.builderNames, ["builder1"]) self.failUnlessEqual(master.botmaster.builders.keys(), ["builder1"]) b1 = master.botmaster.builders["builder1"] # reloading the same config should leave the builder unchanged master.loadConfig(wpCfg1) b2 = master.botmaster.builders["builder1"] self.failUnlessIdentical(b1, b2) # but changing the parameters of the WithProperties should change it master.loadConfig(wpCfg2) b3 = master.botmaster.builders["builder1"] self.failIf(b1 is b3) # again, reloading same config should leave the builder unchanged master.loadConfig(wpCfg2) b4 = master.botmaster.builders["builder1"] self.failUnlessIdentical(b3, b4) def checkIRC(self, m, expected): ircs = {} for irc in self.servers(m, words.IRC): ircs[irc.host] = (irc.nick, irc.channels) self.failUnlessEqual(ircs, expected) def testIRC(self): if not words: raise unittest.SkipTest("Twisted Words package is not installed") master = self.buildmaster master.loadChanges() d = master.loadConfig(emptyCfg) e1 = {} d.addCallback(lambda res: self.checkIRC(master, e1)) d.addCallback(lambda res: master.loadConfig(ircCfg1)) e2 = {'irc.us.freenode.net': ('buildbot', ['twisted'])} d.addCallback(lambda res: self.checkIRC(master, e2)) d.addCallback(lambda res: master.loadConfig(ircCfg2)) e3 = { 'irc.us.freenode.net': ('buildbot', ['twisted']), 'irc.example.com': ('otherbot', ['chan1', 'chan2']) } d.addCallback(lambda res: self.checkIRC(master, e3)) d.addCallback(lambda res: master.loadConfig(ircCfg3)) e4 = {'irc.us.freenode.net': ('buildbot', ['knotted'])} d.addCallback(lambda res: self.checkIRC(master, e4)) d.addCallback(lambda res: master.loadConfig(ircCfg1)) e5 = {'irc.us.freenode.net': ('buildbot', ['twisted'])} d.addCallback(lambda res: self.checkIRC(master, e5)) return d def testWebPortnum(self): master = self.buildmaster master.loadChanges() d = master.loadConfig(webCfg1) def _check1(res): ports = self.checkPorts(self.buildmaster, [(9999, pb.PBServerFactory), (9980, Site)]) p = ports[1] self.p = p # nothing should be changed d.addCallback(_check1) d.addCallback(lambda res: self.buildmaster.loadConfig(webCfg1)) def _check2(res): ports = self.checkPorts(self.buildmaster, [(9999, pb.PBServerFactory), (9980, Site)]) self.failUnlessIdentical( self.p, ports[1], "web port was changed even though " "configuration was not") # WebStatus is no longer a ComparableMixin, so it will be # rebuilt on each reconfig #d.addCallback(_check2) d.addCallback(lambda res: self.buildmaster.loadConfig(webCfg2)) # changes port to 9981 def _check3(p): ports = self.checkPorts(self.buildmaster, [(9999, pb.PBServerFactory), (9981, Site)]) self.failIf( self.p is ports[1], "configuration was changed but web port was unchanged") d.addCallback(_check3) d.addCallback(lambda res: self.buildmaster.loadConfig(webCfg3)) # make 9981 on only localhost def _check4(p): ports = self.checkPorts(self.buildmaster, [(9999, pb.PBServerFactory), (9981, Site)]) self.failUnlessEqual(ports[1].kwargs['interface'], "127.0.0.1") d.addCallback(_check4) d.addCallback(lambda res: self.buildmaster.loadConfig(emptyCfg)) d.addCallback(lambda res: self.checkPorts(self.buildmaster, [( 9999, pb.PBServerFactory)])) return d def testWebPathname(self): master = self.buildmaster master.loadChanges() d = master.loadConfig(webNameCfg1) def _check1(res): self.checkPorts(self.buildmaster, [(9999, pb.PBServerFactory), ('~/.twistd-web-pb', pb.PBServerFactory)]) unixports = self.UNIXports(self.buildmaster) self.f = f = unixports[0].args[1] self.failUnless(isinstance(f.root, ResourcePublisher)) d.addCallback(_check1) d.addCallback(lambda res: self.buildmaster.loadConfig(webNameCfg1)) # nothing should be changed def _check2(res): self.checkPorts(self.buildmaster, [(9999, pb.PBServerFactory), ('~/.twistd-web-pb', pb.PBServerFactory)]) newf = self.UNIXports(self.buildmaster)[0].args[1] self.failUnlessIdentical( self.f, newf, "web factory was changed even though " "configuration was not") # WebStatus is no longer a ComparableMixin, so it will be # rebuilt on each reconfig #d.addCallback(_check2) d.addCallback(lambda res: self.buildmaster.loadConfig(webNameCfg2)) def _check3(res): self.checkPorts(self.buildmaster, [(9999, pb.PBServerFactory), ('./bar.socket', pb.PBServerFactory)]) newf = self.UNIXports(self.buildmaster)[0].args[1], self.failIf( self.f is newf, "web factory was unchanged but " "configuration was changed") d.addCallback(_check3) d.addCallback(lambda res: self.buildmaster.loadConfig(emptyCfg)) d.addCallback(lambda res: self.checkPorts(self.buildmaster, [( 9999, pb.PBServerFactory)])) return d def testDebugPassword(self): master = self.buildmaster master.loadConfig(debugPasswordCfg) self.failUnlessEqual(master.checker.users, { "change": "changepw", "debug": "sekrit" }) master.loadConfig(debugPasswordCfg) self.failUnlessEqual(master.checker.users, { "change": "changepw", "debug": "sekrit" }) master.loadConfig(emptyCfg) self.failUnlessEqual(master.checker.users, {"change": "changepw"}) def testLocks(self): master = self.buildmaster botmaster = master.botmaster # make sure that c['interlocks'] is rejected properly self.failUnlessRaises(KeyError, master.loadConfig, interlockCfgBad) # and that duplicate-named Locks are caught self.failUnlessRaises(ValueError, master.loadConfig, lockCfgBad1) self.failUnlessRaises(ValueError, master.loadConfig, lockCfgBad2) self.failUnlessRaises(ValueError, master.loadConfig, lockCfgBad3) # create a Builder that uses Locks master.loadConfig(lockCfg1a) b1 = master.botmaster.builders["builder1"] self.failUnlessEqual(len(b1.locks), 2) # reloading the same config should not change the Builder master.loadConfig(lockCfg1a) self.failUnlessIdentical(b1, master.botmaster.builders["builder1"]) # but changing the set of locks used should change it master.loadConfig(lockCfg1b) self.failIfIdentical(b1, master.botmaster.builders["builder1"]) b1 = master.botmaster.builders["builder1"] self.failUnlessEqual(len(b1.locks), 1) # similar test with step-scoped locks master.loadConfig(lockCfg2a) b1 = master.botmaster.builders["builder1"] # reloading the same config should not change the Builder master.loadConfig(lockCfg2a) self.failUnlessIdentical(b1, master.botmaster.builders["builder1"]) # but changing the set of locks used should change it master.loadConfig(lockCfg2b) self.failIfIdentical(b1, master.botmaster.builders["builder1"]) b1 = master.botmaster.builders["builder1"] # remove the locks entirely master.loadConfig(lockCfg2c) self.failIfIdentical(b1, master.botmaster.builders["builder1"]) def testNoChangeHorizon(self): master = self.buildmaster master.loadChanges() sourcesCfg = emptyCfg + \ """ from buildbot.changes.pb import PBChangeSource c['change_source'] = PBChangeSource() """ d = master.loadConfig(sourcesCfg) def _check1(res): self.failUnlessEqual(len(list(self.buildmaster.change_svc)), 1) self.failUnlessEqual(self.buildmaster.change_svc.changeHorizon, 0) d.addCallback(_check1) return d def testChangeHorizon(self): master = self.buildmaster master.loadChanges() sourcesCfg = emptyCfg + \ """ from buildbot.changes.pb import PBChangeSource c['change_source'] = PBChangeSource() c['changeHorizon'] = 5 """ d = master.loadConfig(sourcesCfg) def _check1(res): self.failUnlessEqual(len(list(self.buildmaster.change_svc)), 1) self.failUnlessEqual(self.buildmaster.change_svc.changeHorizon, 5) d.addCallback(_check1) return d
def setUp(self): # this class generates several deprecation warnings, which the user # doesn't need to see. warnings.simplefilter('ignore', exceptions.DeprecationWarning) self.buildmaster = BuildMaster(".")
def __init__(self, config_dict): self.config_dict = config_dict def loadConfig(self): return MasterConfig.loadFromDict(self.config_dict, '<dict>') @defer.inlineCallbacks def getMaster(case, reactor, config_dict): """ Create a started ``BuildMaster`` with the given configuration. """ basedir = FilePath(case.mktemp()) basedir.createDirectory() config_dict['buildbotNetUsageData'] = None master = BuildMaster( basedir.path, reactor=reactor, config_loader=DictLoader(config_dict)) if 'db_url' not in config_dict: config_dict['db_url'] = 'sqlite://' # TODO: Allow BuildMaster to transparently upgrade the database, at least # for tests. master.config.db['db_url'] = config_dict['db_url'] yield master.db.setup(check_version=False) yield master.db.model.upgrade() master.db.setup = lambda: None yield master.startService() case.addCleanup(master.stopService) return master
'favicon.ico': util.sibpath(__file__, "../status/web/files/favicon.ico"), }) m.populate_if_missing(os.path.join(basedir, "master.cfg.sample"), util.sibpath(__file__, "sample.cfg"), overwrite=True) # if index.html exists, use it to override the root page tempalte m.move_if_present(os.path.join(basedir, "public_html/index.html"), os.path.join(basedir, "templates/root.html")) from buildbot.db import connector from buildbot.master import BuildMaster if not config['quiet']: print "upgrading database (%s)" % (master_cfg.db['db_url']) master = BuildMaster(config['basedir']) master.config = master_cfg db = connector.DBConnector(master, basedir=config['basedir']) wfd = defer.waitForDeferred( db.setup(check_version=False, verbose=not config['quiet'])) yield wfd wfd.getResult() wfd = defer.waitForDeferred(db.model.upgrade()) yield wfd wfd.getResult() if not config['quiet']: print "upgrade complete" yield 0
class ConfigTest(unittest.TestCase): def setUp(self): # this class generates several deprecation warnings, which the user # doesn't need to see. warnings.simplefilter('ignore', exceptions.DeprecationWarning) self.buildmaster = BuildMaster(".") def failUnlessListsEquivalent(self, list1, list2): l1 = list1[:] l1.sort() l2 = list2[:] l2.sort() self.failUnlessEqual(l1, l2) def servers(self, s, types): # perform a recursive search of s.services, looking for instances of # twisted.application.internet.TCPServer, then extract their .args # values to find the TCP ports they want to listen on for child in s: if service.IServiceCollection.providedBy(child): for gc in self.servers(child, types): yield gc if isinstance(child, types): yield child def TCPports(self, s): return list(self.servers(s, internet.TCPServer)) def UNIXports(self, s): return list(self.servers(s, internet.UNIXServer)) def TCPclients(self, s): return list(self.servers(s, internet.TCPClient)) def checkPorts(self, svc, expected): """Verify that the TCPServer and UNIXServer children of the given service have the expected portnum/pathname and factory classes. As a side-effect, return a list of servers in the same order as the 'expected' list. This can be used to verify properties of the factories contained therein.""" expTCP = [e for e in expected if type(e[0]) == int] expUNIX = [e for e in expected if type(e[0]) == str] haveTCP = [(p.args[0], p.args[1].__class__) for p in self.TCPports(svc)] haveUNIX = [(p.args[0], p.args[1].__class__) for p in self.UNIXports(svc)] self.failUnlessListsEquivalent(expTCP, haveTCP) self.failUnlessListsEquivalent(expUNIX, haveUNIX) ret = [] for e in expected: for have in self.TCPports(svc) + self.UNIXports(svc): if have.args[0] == e[0]: ret.append(have) continue assert(len(ret) == len(expected)) return ret def testEmpty(self): self.failUnlessRaises(KeyError, self.buildmaster.loadConfig, "") def testSimple(self): # covers slavePortnum, base checker passwords master = self.buildmaster master.loadChanges() master.loadConfig(emptyCfg) # note: this doesn't actually start listening, because the app # hasn't been started running self.failUnlessEqual(master.slavePortnum, "tcp:9999") self.checkPorts(master, [(9999, pb.PBServerFactory)]) self.failUnlessEqual(list(master.change_svc), []) self.failUnlessEqual(master.botmaster.builders, {}) self.failUnlessEqual(master.checker.users, {"change": "changepw"}) self.failUnlessEqual(master.projectName, "dummy project") self.failUnlessEqual(master.projectURL, "http://dummy.example.com") self.failUnlessEqual(master.buildbotURL, "http://dummy.example.com/buildbot") def testSlavePortnum(self): master = self.buildmaster master.loadChanges() master.loadConfig(emptyCfg) self.failUnlessEqual(master.slavePortnum, "tcp:9999") ports = self.checkPorts(master, [(9999, pb.PBServerFactory)]) p = ports[0] master.loadConfig(emptyCfg) self.failUnlessEqual(master.slavePortnum, "tcp:9999") ports = self.checkPorts(master, [(9999, pb.PBServerFactory)]) self.failUnlessIdentical(p, ports[0], "the slave port was changed even " + \ "though the configuration was not") master.loadConfig(emptyCfg + "c['slavePortnum'] = 9000\n") self.failUnlessEqual(master.slavePortnum, "tcp:9000") ports = self.checkPorts(master, [(9000, pb.PBServerFactory)]) self.failIf(p is ports[0], "slave port was unchanged but configuration was changed") def testSlaves(self): master = self.buildmaster master.loadChanges() master.loadConfig(emptyCfg) self.failUnlessEqual(master.botmaster.builders, {}) self.failUnlessEqual(master.checker.users, {"change": "changepw"}) # 'botsCfg' is testing backwards compatibility, for 0.7.5 config # files that have not yet been updated to 0.7.6 . This compatibility # (and this test) is scheduled for removal in 0.8.0 . botsCfg = (emptyCfg + "c['bots'] = [('bot1', 'pw1'), ('bot2', 'pw2')]\n") master.loadConfig(botsCfg) self.failUnlessEqual(master.checker.users, {"change": "changepw", "bot1": "pw1", "bot2": "pw2"}) master.loadConfig(botsCfg) self.failUnlessEqual(master.checker.users, {"change": "changepw", "bot1": "pw1", "bot2": "pw2"}) master.loadConfig(emptyCfg) self.failUnlessEqual(master.checker.users, {"change": "changepw"}) slavesCfg = (emptyCfg + "from buildbot.buildslave import BuildSlave\n" "c['slaves'] = [BuildSlave('bot1','pw1'), " "BuildSlave('bot2','pw2')]\n") master.loadConfig(slavesCfg) self.failUnlessEqual(master.checker.users, {"change": "changepw", "bot1": "pw1", "bot2": "pw2"}) def testChangeSource(self): master = self.buildmaster master.loadChanges() master.loadConfig(emptyCfg) self.failUnlessEqual(list(master.change_svc), []) sourcesCfg = emptyCfg + \ """ from buildbot.changes.pb import PBChangeSource c['change_source'] = PBChangeSource() """ d = master.loadConfig(sourcesCfg) def _check1(res): self.failUnlessEqual(len(list(self.buildmaster.change_svc)), 1) s1 = list(self.buildmaster.change_svc)[0] self.failUnless(isinstance(s1, PBChangeSource)) self.failUnlessEqual(s1, list(self.buildmaster.change_svc)[0]) self.failUnless(s1.parent) # verify that unchanged sources are not interrupted d1 = self.buildmaster.loadConfig(sourcesCfg) def _check2(res): self.failUnlessEqual(len(list(self.buildmaster.change_svc)), 1) s2 = list(self.buildmaster.change_svc)[0] self.failUnlessIdentical(s1, s2) self.failUnless(s1.parent) d1.addCallback(_check2) return d1 d.addCallback(_check1) # make sure we can get rid of the sources too d.addCallback(lambda res: self.buildmaster.loadConfig(emptyCfg)) def _check3(res): self.failUnlessEqual(list(self.buildmaster.change_svc), []) d.addCallback(_check3) return d def testChangeSources(self): # make sure we can accept a list master = self.buildmaster master.loadChanges() master.loadConfig(emptyCfg) self.failUnlessEqual(list(master.change_svc), []) sourcesCfg = emptyCfg + \ """ from buildbot.changes.pb import PBChangeSource from buildbot.changes.mail import SyncmailMaildirSource c['change_source'] = [PBChangeSource(), SyncmailMaildirSource('.'), ] """ d = master.loadConfig(sourcesCfg) def _check1(res): self.failUnlessEqual(len(list(self.buildmaster.change_svc)), 2) s1,s2 = list(self.buildmaster.change_svc) if isinstance(s2, PBChangeSource): s1,s2 = s2,s1 self.failUnless(isinstance(s1, PBChangeSource)) self.failUnless(s1.parent) self.failUnless(isinstance(s2, SyncmailMaildirSource)) self.failUnless(s2.parent) d.addCallback(_check1) return d def testSources(self): # test backwards compatibility. c['sources'] is deprecated. master = self.buildmaster master.loadChanges() master.loadConfig(emptyCfg) self.failUnlessEqual(list(master.change_svc), []) sourcesCfg = emptyCfg + \ """ from buildbot.changes.pb import PBChangeSource c['sources'] = [PBChangeSource()] """ d = master.loadConfig(sourcesCfg) def _check1(res): self.failUnlessEqual(len(list(self.buildmaster.change_svc)), 1) s1 = list(self.buildmaster.change_svc)[0] self.failUnless(isinstance(s1, PBChangeSource)) self.failUnless(s1.parent) d.addCallback(_check1) return d def shouldBeFailure(self, res, *expected): self.failUnless(isinstance(res, failure.Failure), "we expected this to fail, not produce %s" % (res,)) res.trap(*expected) return None # all is good def testSchedulerErrors(self): master = self.buildmaster master.loadChanges() master.loadConfig(emptyCfg) self.failUnlessEqual(master.allSchedulers(), []) def _shouldBeFailure(res, hint=None): self.shouldBeFailure(res, AssertionError, ValueError) if hint: self.failUnless(str(res).find(hint) != -1) def _loadConfig(res, newcfg): return self.buildmaster.loadConfig(newcfg) d = defer.succeed(None) # c['schedulers'] must be a list badcfg = schedulersCfg + \ """ c['schedulers'] = Scheduler('full', None, 60, ['builder1']) """ d.addCallback(_loadConfig, badcfg) d.addBoth(_shouldBeFailure, "c['schedulers'] must be a list of Scheduler instances") # c['schedulers'] must be a list of IScheduler objects badcfg = schedulersCfg + \ """ c['schedulers'] = ['oops', 'problem'] """ d.addCallback(_loadConfig, badcfg) d.addBoth(_shouldBeFailure, "c['schedulers'] must be a list of Scheduler instances") # c['schedulers'] must point at real builders badcfg = schedulersCfg + \ """ c['schedulers'] = [Scheduler('full', None, 60, ['builder-bogus'])] """ d.addCallback(_loadConfig, badcfg) d.addBoth(_shouldBeFailure, "uses unknown builder") # builderNames= must be a list badcfg = schedulersCfg + \ """ c['schedulers'] = [Scheduler('full', None, 60, 'builder1')] """ d.addCallback(_loadConfig, badcfg) d.addBoth(_shouldBeFailure, "must be a list of Builder description names") # builderNames= must be a list of strings, not dicts badcfg = schedulersCfg + \ """ c['schedulers'] = [Scheduler('full', None, 60, [b1])] """ d.addCallback(_loadConfig, badcfg) d.addBoth(_shouldBeFailure, "must be a list of Builder description names") # builderNames= must be a list of strings, not a dict badcfg = schedulersCfg + \ """ c['schedulers'] = [Scheduler('full', None, 60, b1)] """ d.addCallback(_loadConfig, badcfg) d.addBoth(_shouldBeFailure, "must be a list of Builder description names") # each Scheduler must have a unique name badcfg = schedulersCfg + \ """ c['schedulers'] = [Scheduler('dup', None, 60, []), Scheduler('dup', None, 60, [])] """ d.addCallback(_loadConfig, badcfg) d.addBoth(_shouldBeFailure, "Schedulers must have unique names") return d def testSchedulers(self): master = self.buildmaster master.loadChanges() master.loadConfig(emptyCfg) self.failUnlessEqual(master.allSchedulers(), []) d = self.buildmaster.loadConfig(schedulersCfg) d.addCallback(self._testSchedulers_1) return d def _testSchedulers_1(self, res): sch = self.buildmaster.allSchedulers() self.failUnlessEqual(len(sch), 1) s = sch[0] self.failUnless(isinstance(s, scheduler.Scheduler)) self.failUnlessEqual(s.name, "full") self.failUnlessEqual(s.branch, None) self.failUnlessEqual(s.treeStableTimer, 60) self.failUnlessEqual(s.builderNames, ['builder1']) newcfg = schedulersCfg + \ """ s1 = Scheduler('full', None, 60, ['builder1']) c['schedulers'] = [s1, Dependent('downstream', s1, ['builder1'])] """ d = self.buildmaster.loadConfig(newcfg) d.addCallback(self._testSchedulers_2, newcfg) return d def _testSchedulers_2(self, res, newcfg): sch = self.buildmaster.allSchedulers() self.failUnlessEqual(len(sch), 2) s = sch[0] self.failUnless(isinstance(s, scheduler.Scheduler)) s = sch[1] self.failUnless(isinstance(s, scheduler.Dependent)) self.failUnlessEqual(s.name, "downstream") self.failUnlessEqual(s.builderNames, ['builder1']) # reloading the same config file should leave the schedulers in place d = self.buildmaster.loadConfig(newcfg) d.addCallback(self._testSchedulers_3, sch) return d def _testSchedulers_3(self, res, sch1): sch2 = self.buildmaster.allSchedulers() self.failUnlessEqual(len(sch2), 2) sch1.sort() sch2.sort() self.failUnlessEqual(sch1, sch2) self.failUnlessIdentical(sch1[0], sch2[0]) self.failUnlessIdentical(sch1[1], sch2[1]) self.failUnlessIdentical(sch1[0].parent, self.buildmaster) self.failUnlessIdentical(sch1[1].parent, self.buildmaster) def testBuilders(self): master = self.buildmaster master.loadConfig(emptyCfg) self.failUnlessEqual(master.botmaster.builders, {}) master.loadConfig(buildersCfg) self.failUnlessEqual(master.botmaster.builderNames, ["builder1"]) self.failUnlessEqual(master.botmaster.builders.keys(), ["builder1"]) b = master.botmaster.builders["builder1"] self.failUnless(isinstance(b, Builder)) self.failUnlessEqual(b.name, "builder1") self.failUnlessEqual(b.slavenames, ["bot1"]) self.failUnlessEqual(b.builddir, "workdir") f1 = b.buildFactory self.failUnless(isinstance(f1, BasicBuildFactory)) steps = f1.steps self.failUnlessEqual(len(steps), 3) self.failUnlessEqual(steps[0], (CVS, {'cvsroot': 'cvsroot', 'cvsmodule': 'cvsmodule', 'mode': 'clobber'})) self.failUnlessEqual(steps[1], (Compile, {'command': 'make all'})) self.failUnlessEqual(steps[2], (Test, {'command': 'make check'})) # make sure a reload of the same data doesn't interrupt the Builder master.loadConfig(buildersCfg) self.failUnlessEqual(master.botmaster.builderNames, ["builder1"]) self.failUnlessEqual(master.botmaster.builders.keys(), ["builder1"]) b2 = master.botmaster.builders["builder1"] self.failUnlessIdentical(b, b2) # TODO: test that the BuilderStatus object doesn't change #statusbag2 = master.client_svc.statusbags["builder1"] #self.failUnlessIdentical(statusbag, statusbag2) # but changing something should result in a new Builder master.loadConfig(buildersCfg2) self.failUnlessEqual(master.botmaster.builderNames, ["builder1"]) self.failUnlessEqual(master.botmaster.builders.keys(), ["builder1"]) b3 = master.botmaster.builders["builder1"] self.failIf(b is b3) # the statusbag remains the same TODO #statusbag3 = master.client_svc.statusbags["builder1"] #self.failUnlessIdentical(statusbag, statusbag3) # adding new builder master.loadConfig(buildersCfg3) self.failUnlessEqual(master.botmaster.builderNames, ["builder1", "builder2"]) self.failUnlessListsEquivalent(master.botmaster.builders.keys(), ["builder1", "builder2"]) b4 = master.botmaster.builders["builder1"] self.failUnlessIdentical(b3, b4) # changing first builder should leave it at the same place in the list master.loadConfig(buildersCfg4) self.failUnlessEqual(master.botmaster.builderNames, ["builder1", "builder2"]) self.failUnlessListsEquivalent(master.botmaster.builders.keys(), ["builder1", "builder2"]) b5 = master.botmaster.builders["builder1"] self.failIf(b4 is b5) master.loadConfig(buildersCfg5) self.failUnlessEqual(master.botmaster.builderNames, ["builder1", "builder2"]) self.failUnlessListsEquivalent(master.botmaster.builders.keys(), ["builder1", "builder2"]) b5 = master.botmaster.builders["builder1"] # and removing it should make the Builder go away master.loadConfig(emptyCfg) self.failUnlessEqual(master.botmaster.builderNames, []) self.failUnlessEqual(master.botmaster.builders, {}) #self.failUnlessEqual(master.client_svc.statusbags, {}) # TODO def testWithProperties(self): master = self.buildmaster master.loadConfig(wpCfg1) self.failUnlessEqual(master.botmaster.builderNames, ["builder1"]) self.failUnlessEqual(master.botmaster.builders.keys(), ["builder1"]) b1 = master.botmaster.builders["builder1"] # reloading the same config should leave the builder unchanged master.loadConfig(wpCfg1) b2 = master.botmaster.builders["builder1"] self.failUnlessIdentical(b1, b2) # but changing the parameters of the WithProperties should change it master.loadConfig(wpCfg2) b3 = master.botmaster.builders["builder1"] self.failIf(b1 is b3) # again, reloading same config should leave the builder unchanged master.loadConfig(wpCfg2) b4 = master.botmaster.builders["builder1"] self.failUnlessIdentical(b3, b4) def checkIRC(self, m, expected): ircs = {} for irc in self.servers(m, words.IRC): ircs[irc.host] = (irc.nick, irc.channels) self.failUnlessEqual(ircs, expected) def testIRC(self): if not words: raise unittest.SkipTest("Twisted Words package is not installed") master = self.buildmaster master.loadChanges() d = master.loadConfig(emptyCfg) e1 = {} d.addCallback(lambda res: self.checkIRC(master, e1)) d.addCallback(lambda res: master.loadConfig(ircCfg1)) e2 = {'irc.us.freenode.net': ('buildbot', ['twisted'])} d.addCallback(lambda res: self.checkIRC(master, e2)) d.addCallback(lambda res: master.loadConfig(ircCfg2)) e3 = {'irc.us.freenode.net': ('buildbot', ['twisted']), 'irc.example.com': ('otherbot', ['chan1', 'chan2'])} d.addCallback(lambda res: self.checkIRC(master, e3)) d.addCallback(lambda res: master.loadConfig(ircCfg3)) e4 = {'irc.us.freenode.net': ('buildbot', ['knotted'])} d.addCallback(lambda res: self.checkIRC(master, e4)) d.addCallback(lambda res: master.loadConfig(ircCfg1)) e5 = {'irc.us.freenode.net': ('buildbot', ['twisted'])} d.addCallback(lambda res: self.checkIRC(master, e5)) return d def testWebPortnum(self): master = self.buildmaster master.loadChanges() d = master.loadConfig(webCfg1) def _check1(res): ports = self.checkPorts(self.buildmaster, [(9999, pb.PBServerFactory), (9980, Site)]) p = ports[1] self.p = p # nothing should be changed d.addCallback(_check1) d.addCallback(lambda res: self.buildmaster.loadConfig(webCfg1)) def _check2(res): ports = self.checkPorts(self.buildmaster, [(9999, pb.PBServerFactory), (9980, Site)]) self.failUnlessIdentical(self.p, ports[1], "web port was changed even though " "configuration was not") # WebStatus is no longer a ComparableMixin, so it will be # rebuilt on each reconfig #d.addCallback(_check2) d.addCallback(lambda res: self.buildmaster.loadConfig(webCfg2)) # changes port to 9981 def _check3(p): ports = self.checkPorts(self.buildmaster, [(9999, pb.PBServerFactory), (9981, Site)]) self.failIf(self.p is ports[1], "configuration was changed but web port was unchanged") d.addCallback(_check3) d.addCallback(lambda res: self.buildmaster.loadConfig(webCfg3)) # make 9981 on only localhost def _check4(p): ports = self.checkPorts(self.buildmaster, [(9999, pb.PBServerFactory), (9981, Site)]) self.failUnlessEqual(ports[1].kwargs['interface'], "127.0.0.1") d.addCallback(_check4) d.addCallback(lambda res: self.buildmaster.loadConfig(emptyCfg)) d.addCallback(lambda res: self.checkPorts(self.buildmaster, [(9999, pb.PBServerFactory)])) return d def testWebPathname(self): master = self.buildmaster master.loadChanges() d = master.loadConfig(webNameCfg1) def _check1(res): self.checkPorts(self.buildmaster, [(9999, pb.PBServerFactory), ('~/.twistd-web-pb', pb.PBServerFactory)]) unixports = self.UNIXports(self.buildmaster) self.f = f = unixports[0].args[1] self.failUnless(isinstance(f.root, ResourcePublisher)) d.addCallback(_check1) d.addCallback(lambda res: self.buildmaster.loadConfig(webNameCfg1)) # nothing should be changed def _check2(res): self.checkPorts(self.buildmaster, [(9999, pb.PBServerFactory), ('~/.twistd-web-pb', pb.PBServerFactory)]) newf = self.UNIXports(self.buildmaster)[0].args[1] self.failUnlessIdentical(self.f, newf, "web factory was changed even though " "configuration was not") # WebStatus is no longer a ComparableMixin, so it will be # rebuilt on each reconfig #d.addCallback(_check2) d.addCallback(lambda res: self.buildmaster.loadConfig(webNameCfg2)) def _check3(res): self.checkPorts(self.buildmaster, [(9999, pb.PBServerFactory), ('./bar.socket', pb.PBServerFactory)]) newf = self.UNIXports(self.buildmaster)[0].args[1], self.failIf(self.f is newf, "web factory was unchanged but " "configuration was changed") d.addCallback(_check3) d.addCallback(lambda res: self.buildmaster.loadConfig(emptyCfg)) d.addCallback(lambda res: self.checkPorts(self.buildmaster, [(9999, pb.PBServerFactory)])) return d def testDebugPassword(self): master = self.buildmaster master.loadConfig(debugPasswordCfg) self.failUnlessEqual(master.checker.users, {"change": "changepw", "debug": "sekrit"}) master.loadConfig(debugPasswordCfg) self.failUnlessEqual(master.checker.users, {"change": "changepw", "debug": "sekrit"}) master.loadConfig(emptyCfg) self.failUnlessEqual(master.checker.users, {"change": "changepw"}) def testLocks(self): master = self.buildmaster botmaster = master.botmaster # make sure that c['interlocks'] is rejected properly self.failUnlessRaises(KeyError, master.loadConfig, interlockCfgBad) # and that duplicate-named Locks are caught self.failUnlessRaises(ValueError, master.loadConfig, lockCfgBad1) self.failUnlessRaises(ValueError, master.loadConfig, lockCfgBad2) self.failUnlessRaises(ValueError, master.loadConfig, lockCfgBad3) # create a Builder that uses Locks master.loadConfig(lockCfg1a) b1 = master.botmaster.builders["builder1"] self.failUnlessEqual(len(b1.locks), 2) # reloading the same config should not change the Builder master.loadConfig(lockCfg1a) self.failUnlessIdentical(b1, master.botmaster.builders["builder1"]) # but changing the set of locks used should change it master.loadConfig(lockCfg1b) self.failIfIdentical(b1, master.botmaster.builders["builder1"]) b1 = master.botmaster.builders["builder1"] self.failUnlessEqual(len(b1.locks), 1) # similar test with step-scoped locks master.loadConfig(lockCfg2a) b1 = master.botmaster.builders["builder1"] # reloading the same config should not change the Builder master.loadConfig(lockCfg2a) self.failUnlessIdentical(b1, master.botmaster.builders["builder1"]) # but changing the set of locks used should change it master.loadConfig(lockCfg2b) self.failIfIdentical(b1, master.botmaster.builders["builder1"]) b1 = master.botmaster.builders["builder1"] # remove the locks entirely master.loadConfig(lockCfg2c) self.failIfIdentical(b1, master.botmaster.builders["builder1"]) def testNoChangeHorizon(self): master = self.buildmaster master.loadChanges() sourcesCfg = emptyCfg + \ """ from buildbot.changes.pb import PBChangeSource c['change_source'] = PBChangeSource() """ d = master.loadConfig(sourcesCfg) def _check1(res): self.failUnlessEqual(len(list(self.buildmaster.change_svc)), 1) self.failUnlessEqual(self.buildmaster.change_svc.changeHorizon, 0) d.addCallback(_check1) return d def testChangeHorizon(self): master = self.buildmaster master.loadChanges() sourcesCfg = emptyCfg + \ """ from buildbot.changes.pb import PBChangeSource c['change_source'] = PBChangeSource() c['changeHorizon'] = 5 """ d = master.loadConfig(sourcesCfg) def _check1(res): self.failUnlessEqual(len(list(self.buildmaster.change_svc)), 1) self.failUnlessEqual(self.buildmaster.change_svc.changeHorizon, 5) d.addCallback(_check1) return d
basedir = os.path.abspath(os.path.dirname(__file__)) configfile = 'master.cfg' # Default umask for server umask = None # note: this line is matched against to check that this is a buildmaster # directory; do not edit it. application = service.Application('buildmaster') import sys from twisted.python.log import ILogObserver, FileLogObserver application.setComponent(ILogObserver, FileLogObserver(sys.stdout).emit) m = BuildMaster(basedir, configfile, umask) m.setServiceParent(application) # and slave on the same process! buildmaster_host = 'localhost' port = 19989 slavename = 'example-slave' passwd = 'pass' keepalive = 600 usepty = 0 umask = None maxdelay = 300 allow_shutdown = None slavedir = os.path.join(basedir, "slave") if not os.path.exists(slavedir):
def setupConfig(self, config_dict, startWorker=True): """ Setup and start a master configured by the function configFunc defined in the test module. @type config_dict: dict @param configFunc: The BuildmasterConfig dictionary. """ self.basedir = os.path.abspath('basdir') self.setUpDirs(self.basedir) self.addCleanup(self.tearDownDirs) # mock reactor.stop (which trial *really* doesn't # like test code to call!) stop = mock.create_autospec(reactor.stop) self.patch(reactor, 'stop', stop) if startWorker: if self.proto == 'pb': proto = {"pb": {"port": "tcp:0:interface=127.0.0.1"}} workerclass = worker.Worker elif self.proto == 'null': proto = {"null": {}} workerclass = worker.LocalWorker config_dict['workers'] = [workerclass("local1", "localpw")] config_dict['protocols'] = proto # create the master and set its config m = BuildMaster(self.basedir, reactor=reactor, config_loader=DictLoader(config_dict)) self.master = m # update the DB yield m.db.setup(check_version=False) yield m.db.model.upgrade() # stub out m.db.setup since it was already called above m.db.setup = lambda: None # start the service yield m.startService() self.failIf(stop.called, "startService tried to stop the reactor; check logs") # and shutdown the db threadpool, as is normally done at reactor stop self.addCleanup(m.db.pool.shutdown) self.addCleanup(m.stopService) if not startWorker: return if self.proto == 'pb': # We find out the worker port automatically workerPort = list(itervalues(m.pbmanager.dispatchers))[ 0].port.getHost().port # create a worker, and attach it to the master, it will be started, and stopped # along with the master self.w = BuildSlave( "127.0.0.1", workerPort, "local1", "localpw", self.basedir, False, False) elif self.proto == 'null': self.w = None if self.w is not None: self.w.startService() self.addCleanup(self.w.stopService) @defer.inlineCallbacks def dump(): if not self._passed: dump = StringIO.StringIO() print("FAILED! dumping build db for debug", file=dump) builds = yield self.master.data.get(("builds",)) for build in builds: yield self.printBuild(build, dump, withLogs=True) raise self.failureException(dump.getvalue()) self.addCleanup(dump)
def setupConfig(self, config_dict, startWorker=True): """ Setup and start a master configured by the function configFunc defined in the test module. @type config_dict: dict @param configFunc: The BuildmasterConfig dictionary. """ self.basedir = os.path.abspath('basdir') self.setUpDirs(self.basedir) self.addCleanup(self.tearDownDirs) # mock reactor.stop (which trial *really* doesn't # like test code to call!) stop = mock.create_autospec(reactor.stop) self.patch(reactor, 'stop', stop) if startWorker: if self.proto == 'pb': proto = {"pb": {"port": "tcp:0:interface=127.0.0.1"}} workerclass = worker.Worker elif self.proto == 'null': proto = {"null": {}} workerclass = worker.LocalWorker config_dict['workers'] = [workerclass("local1", "localpw")] config_dict['protocols'] = proto # create the master and set its config m = BuildMaster(self.basedir, reactor=reactor, config_loader=DictLoader(config_dict)) self.master = m # update the DB yield m.db.setup(check_version=False) yield m.db.model.upgrade() # stub out m.db.setup since it was already called above m.db.setup = lambda: None # start the service yield m.startService() self.failIf(stop.called, "startService tried to stop the reactor; check logs") # and shutdown the db threadpool, as is normally done at reactor stop self.addCleanup(m.db.pool.shutdown) self.addCleanup(m.stopService) if not startWorker: return if self.proto == 'pb': # We find out the worker port automatically workerPort = list(itervalues( m.pbmanager.dispatchers))[0].port.getHost().port # create a worker, and attach it to the master, it will be started, and stopped # along with the master self.w = BuildSlave("127.0.0.1", workerPort, "local1", "localpw", self.basedir, False, False) elif self.proto == 'null': self.w = None if self.w is not None: self.w.startService() self.addCleanup(self.w.stopService) @defer.inlineCallbacks def dump(): if not self._passed: dump = StringIO.StringIO() print("FAILED! dumping build db for debug", file=dump) builds = yield self.master.data.get(("builds", )) for build in builds: yield self.printBuild(build, dump, withLogs=True) raise self.failureException(dump.getvalue()) self.addCleanup(dump)
def get_master(basedir): cfg = cm.MasterConfig.loadConfig(basedir, 'master.cfg') master = BuildMaster(basedir) master.config = cfg return master