Esempio n. 1
0
class BuildMaster(config.ReconfigurableServiceMixin, service.MultiService):

    # frequency with which to reclaim running builds; this should be set to
    # something fairly long, to avoid undue database load
    RECLAIM_BUILD_INTERVAL = 10*60

    # multiplier on RECLAIM_BUILD_INTERVAL at which a build is considered
    # unclaimed; this should be at least 2 to avoid false positives
    UNCLAIMED_BUILD_FACTOR = 6

    # if this quantity of unclaimed build requests are present in the table,
    # then something is probably wrong!  The master will log a WARNING on every
    # database poll operation.
    WARNING_UNCLAIMED_COUNT = 10000

    def __init__(self, basedir, configFileName="master.cfg"):
        service.MultiService.__init__(self)
        self.setName("buildmaster")

        self.basedir = basedir
        assert os.path.isdir(self.basedir)
        self.configFileName = configFileName

        # set up child services
        self.create_child_services()

        # loop for polling the db
        self.db_loop = None

        # configuration / reconfiguration handling
        self.config = config.MasterConfig()
        self.reconfig_active = False
        self.reconfig_requested = False
        self.reconfig_notifier = None

        # this stores parameters used in the tac file, and is accessed by the
        # WebStatus to duplicate those values.
        self.log_rotation = LogRotation()

        # subscription points
        self._change_subs = \
                subscription.SubscriptionPoint("changes")
        self._new_buildrequest_subs = \
                subscription.SubscriptionPoint("buildrequest_additions")
        self._new_buildset_subs = \
                subscription.SubscriptionPoint("buildset_additions")
        self._complete_buildset_subs = \
                subscription.SubscriptionPoint("buildset_completion")

        # local cache for this master's object ID
        self._object_id = None


    def create_child_services(self):
        # note that these are order-dependent.  If you get the order wrong,
        # you'll know it, as the master will fail to start.

        self.metrics = metrics.MetricLogObserver()
        self.metrics.setServiceParent(self)

        self.caches = cache.CacheManager()
        self.caches.setServiceParent(self)

        self.pbmanager = buildbot.pbmanager.PBManager()
        self.pbmanager.setServiceParent(self)

        self.change_svc = ChangeManager(self)
        self.change_svc.setServiceParent(self)

        self.botmaster = BotMaster(self)
        self.botmaster.setServiceParent(self)

        self.scheduler_manager = SchedulerManager(self)
        self.scheduler_manager.setServiceParent(self)

        self.user_manager = UserManagerManager(self)
        self.user_manager.setServiceParent(self)

        self.db = connector.DBConnector(self, self.basedir)
        self.db.setServiceParent(self)

        self.debug = debug.DebugServices(self)
        self.debug.setServiceParent(self)

        self.status = Status(self)
        self.status.setServiceParent(self)

    # setup and reconfig handling

    _already_started = False
    @defer.deferredGenerator
    def startService(self, _reactor=reactor):
        assert not self._already_started, "can only start the master once"
        self._already_started = True

        log.msg("Starting BuildMaster -- buildbot.version: %s" %
                buildbot.version)

        # first, apply all monkeypatches
        monkeypatches.patch_all()

        # we want to wait until the reactor is running, so we can call
        # reactor.stop() for fatal errors
        d = defer.Deferred()
        _reactor.callWhenRunning(d.callback, None)
        wfd = defer.waitForDeferred(d)
        yield wfd
        wfd.getResult()

        try:
            # load the configuration file, treating errors as fatal
            try:
                self.config = config.MasterConfig.loadConfig(self.basedir,
                                                        self.configFileName)
            except config.ConfigErrors, e:
                log.msg("Configuration Errors:")
                for msg in e.errors:
                    log.msg("  " + msg)
                log.msg("Halting master.")
                _reactor.stop()
                return
            except:
                log.err(failure.Failure(), 'while starting BuildMaster')
                _reactor.stop()
                return

            # set up services that need access to the config before everything else
            # gets told to reconfig
            try:
                wfd = defer.waitForDeferred(
                        self.db.setup())
                yield wfd
                wfd.getResult()
            except connector.DatabaseNotReadyError:
                # (message was already logged)
                _reactor.stop()
                return

            if hasattr(signal, "SIGHUP"):
                def sighup(*args):
                    _reactor.callLater(0, self.reconfig)
                signal.signal(signal.SIGHUP, sighup)

            # call the parent method
            wfd = defer.waitForDeferred(
                defer.maybeDeferred(lambda :
                    service.MultiService.startService(self)))
            yield wfd
            wfd.getResult()

            # give all services a chance to load the new configuration, rather than
            # the base configuration
            wfd = defer.waitForDeferred(
                self.reconfigService(self.config))
            yield wfd
            wfd.getResult()
        except:
Esempio n. 2
0
class BuildMaster(service.ReconfigurableServiceMixin, service.MasterService,
                  WorkerAPICompatMixin):

    # frequency with which to reclaim running builds; this should be set to
    # something fairly long, to avoid undue database load
    RECLAIM_BUILD_INTERVAL = 10 * 60

    # multiplier on RECLAIM_BUILD_INTERVAL at which a build is considered
    # unclaimed; this should be at least 2 to avoid false positives
    UNCLAIMED_BUILD_FACTOR = 6

    def __init__(self, basedir, configFileName=None, umask=None, reactor=None, config_loader=None):
        service.AsyncMultiService.__init__(self)

        if reactor is None:
            from twisted.internet import reactor
        self.reactor = reactor

        self.setName("buildmaster")

        self.umask = umask

        self.basedir = basedir
        if basedir is not None:  # None is used in tests
            assert os.path.isdir(self.basedir)

        if config_loader is not None and configFileName is not None:
            raise config.ConfigErrors([
                "Can't specify both `config_loader` and `configFilename`.",
            ])
        elif config_loader is None:
            if configFileName is None:
                configFileName = 'master.cfg'
            config_loader = config.FileLoader(self.basedir, configFileName)
        self.config_loader = config_loader
        self.configFileName = configFileName

        # flag so we don't try to do fancy things before the master is ready
        self._master_initialized = False

        # set up child services
        self.create_child_services()

        # db configured values
        self.configured_db_url = None

        # configuration / reconfiguration handling
        self.config = config.MasterConfig()
        self.reconfig_active = False
        self.reconfig_requested = False
        self.reconfig_notifier = None

        # this stores parameters used in the tac file, and is accessed by the
        # WebStatus to duplicate those values.
        self.log_rotation = LogRotation()

        # local cache for this master's object ID
        self._object_id = None

        # Check environment is sensible
        check_functional_environment(self.config)

        # figure out local hostname
        try:
            self.hostname = os.uname()[1]  # only on unix
        except AttributeError:
            self.hostname = socket.getfqdn()

        # public attributes
        self.name = ("%s:%s" % (self.hostname,
                                os.path.abspath(self.basedir or '.')))
        self.name = self.name.decode('ascii', 'replace')
        self.masterid = None

    def create_child_services(self):
        # note that these are order-dependent.  If you get the order wrong,
        # you'll know it, as the master will fail to start.

        self.metrics = metrics.MetricLogObserver()
        self.metrics.setServiceParent(self)

        self.caches = cache.CacheManager()
        self.caches.setServiceParent(self)

        self.pbmanager = buildbot.pbmanager.PBManager()
        self.pbmanager.setServiceParent(self)

        self.workers = workermanager.WorkerManager(self)
        self.workers.setServiceParent(self)

        self.change_svc = ChangeManager()
        self.change_svc.setServiceParent(self)

        self.botmaster = BotMaster()
        self.botmaster.setServiceParent(self)

        self.scheduler_manager = SchedulerManager()
        self.scheduler_manager.setServiceParent(self)

        self.user_manager = UserManagerManager(self)
        self.user_manager.setServiceParent(self)

        self.db = dbconnector.DBConnector(self.basedir)
        self.db.setServiceParent(self)

        self.wamp = wampconnector.WampConnector()
        self.wamp.setServiceParent(self)

        self.mq = mqconnector.MQConnector()
        self.mq.setServiceParent(self)

        self.data = dataconnector.DataConnector()
        self.data.setServiceParent(self)

        self.www = wwwservice.WWWService()
        self.www.setServiceParent(self)

        self.debug = debug.DebugServices()
        self.debug.setServiceParent(self)

        self.status = Status()
        self.status.setServiceParent(self)

        self.service_manager = service.BuildbotServiceManager()
        self.service_manager.setServiceParent(self)
        self.service_manager.reconfig_priority = 1000

        self.masterHouskeepingTimer = 0

        @defer.inlineCallbacks
        def heartbeat():
            if self.masterid is not None:
                yield self.data.updates.masterActive(name=self.name,
                                                     masterid=self.masterid)
            # force housekeeping once a day
            yield self.data.updates.expireMasters((self.masterHouskeepingTimer % (24 * 60)) == 0)
            self.masterHouskeepingTimer += 1
        self.masterHeartbeatService = internet.TimerService(60, heartbeat)
        # we do setServiceParent only when the master is configured
        # master should advertise itself only at that time

    # setup and reconfig handling

    _already_started = False

    @defer.inlineCallbacks
    def startService(self):
        assert not self._already_started, "can only start the master once"
        self._already_started = True

        log.msg("Starting BuildMaster -- buildbot.version: %s" %
                buildbot.version)

        # Set umask
        if self.umask is not None:
            os.umask(self.umask)

        # first, apply all monkeypatches
        monkeypatches.patch_all()

        # we want to wait until the reactor is running, so we can call
        # reactor.stop() for fatal errors
        d = defer.Deferred()
        self.reactor.callWhenRunning(d.callback, None)
        yield d

        startup_succeed = False
        try:
            # load the configuration file, treating errors as fatal
            try:
                # run the master.cfg in thread, so that it can use blocking
                # code
                self.config = yield threads.deferToThreadPool(
                    self.reactor, self.reactor.getThreadPool(),
                    self.config_loader.loadConfig)

            except config.ConfigErrors as e:
                log.msg("Configuration Errors:")
                for msg in e.errors:
                    log.msg("  " + msg)
                log.msg("Halting master.")
                self.reactor.stop()
                return
            except Exception:
                log.err(failure.Failure(), 'while starting BuildMaster')
                self.reactor.stop()
                return

            # set up services that need access to the config before everything
            # else gets told to reconfig
            try:
                yield self.db.setup()
            except exceptions.DatabaseNotReadyError:
                # (message was already logged)
                self.reactor.stop()
                return

            self.mq.setup()

            if hasattr(signal, "SIGHUP"):
                def sighup(*args):
                    eventually(self.reconfig)
                signal.signal(signal.SIGHUP, sighup)

            if hasattr(signal, "SIGUSR1"):
                def sigusr1(*args):
                    self.reactor.callLater(0, self.botmaster.cleanShutdown)
                signal.signal(signal.SIGUSR1, sigusr1)

            # get the masterid so other services can use it in
            # startup/reconfig.  This goes directly to the DB since the data
            # API isn't initialized yet, and anyway, this method is aware of
            # the DB API since it just called its setup function
            self.masterid = yield self.db.masters.findMasterId(
                name=self.name)

            # mark this master as stopped, in case it crashed before
            yield self.data.updates.masterStopped(name=self.name,
                                                  masterid=self.masterid)

            # call the parent method
            yield service.AsyncMultiService.startService(self)

            # give all services a chance to load the new configuration, rather
            # than the base configuration
            yield self.reconfigServiceWithBuildbotConfig(self.config)

            # mark the master as active now that mq is running
            yield self.data.updates.masterActive(
                name=self.name,
                masterid=self.masterid)

            startup_succeed = True
        except Exception:
            f = failure.Failure()
            log.err(f, 'while starting BuildMaster')
            self.reactor.stop()

        finally:
            if startup_succeed:
                log.msg("BuildMaster is running")
            else:
                log.msg("BuildMaster startup failed")

            self._master_initialized = True

    @defer.inlineCallbacks
    def stopService(self):
        if self.masterid is not None:
            yield self.data.updates.masterStopped(
                name=self.name, masterid=self.masterid)
        if self.running:
            yield service.AsyncMultiService.stopService(self)

        log.msg("BuildMaster is stopped")
        self._master_initialized = False

    def reconfig(self):
        # this method wraps doConfig, ensuring it is only ever called once at
        # a time, and alerting the user if the reconfig takes too long
        if self.reconfig_active:
            log.msg("reconfig already active; will reconfig again after")
            self.reconfig_requested = True
            return

        self.reconfig_active = self.reactor.seconds()
        metrics.MetricCountEvent.log("loaded_config", 1)

        # notify every 10 seconds that the reconfig is still going on, although
        # reconfigs should not take that long!
        self.reconfig_notifier = task.LoopingCall(lambda:
                                                  log.msg("reconfig is ongoing for %d s" %
                                                          (self.reactor.seconds() - self.reconfig_active)))
        self.reconfig_notifier.start(10, now=False)

        timer = metrics.Timer("BuildMaster.reconfig")
        timer.start()

        d = self.doReconfig()

        @d.addBoth
        def cleanup(res):
            timer.stop()
            self.reconfig_notifier.stop()
            self.reconfig_notifier = None
            self.reconfig_active = False
            if self.reconfig_requested:
                self.reconfig_requested = False
                self.reconfig()
            return res

        @d.addCallback
        def advertiseItself(res):
            if self.masterHeartbeatService.parent is None:
                self.masterHeartbeatService.setServiceParent(self)

        d.addErrback(log.err, 'while reconfiguring')

        return d  # for tests

    @defer.inlineCallbacks
    def doReconfig(self):
        log.msg("beginning configuration update")
        changes_made = False
        failed = False
        try:
            # Run the master.cfg in thread, so that it cas use blocking code
            new_config = yield threads.deferToThreadPool(
                self.reactor, self.reactor.getThreadPool(),
                self.config_loader.loadConfig)
            changes_made = True
            self.config = new_config

            yield self.reconfigServiceWithBuildbotConfig(new_config)

        except config.ConfigErrors as e:
            for msg in e.errors:
                log.msg(msg)
            failed = True

        except Exception:
            log.err(failure.Failure(), 'during reconfig:')
            failed = True

        if failed:
            if changes_made:
                log.msg("WARNING: reconfig partially applied; master "
                        "may malfunction")
            else:
                log.msg("reconfig aborted without making any changes")
        else:
            log.msg("configuration update complete")

    def reconfigServiceWithBuildbotConfig(self, new_config):
        if self.configured_db_url is None:
            self.configured_db_url = new_config.db['db_url']
        elif (self.configured_db_url != new_config.db['db_url']):
            config.error(
                "Cannot change c['db']['db_url'] after the master has started",
            )

        if self.config.mq['type'] != new_config.mq['type']:
            raise config.ConfigErrors([
                "Cannot change c['mq']['type'] after the master has started",
            ])

        return service.ReconfigurableServiceMixin.reconfigServiceWithBuildbotConfig(self,
                                                                                    new_config)

    # informational methods
    def allSchedulers(self):
        return list(self.scheduler_manager)

    def getStatus(self):
        """
        @rtype: L{buildbot.status.builder.Status}
        """
        return self.status

    # triggering methods

    @defer.inlineCallbacks
    def addChange(self, who=None, files=None, comments=None, **kwargs):
        # deprecated in 0.9.0; will be removed in 1.0.0
        log.msg("WARNING: change source is using deprecated "
                "self.master.addChange method; this method will disappear in "
                "Buildbot-1.0.0")
        # handle positional arguments
        kwargs['who'] = who
        kwargs['files'] = files
        kwargs['comments'] = comments

        def handle_deprec(oldname, newname):
            if oldname not in kwargs:
                return
            old = kwargs.pop(oldname)
            if old is not None:
                if kwargs.get(newname) is None:
                    log.msg("WARNING: change source is using deprecated "
                            "addChange parameter '%s'" % oldname)
                    return old
                raise TypeError("Cannot provide '%s' and '%s' to addChange"
                                % (oldname, newname))
            return kwargs.get(newname)

        kwargs['author'] = handle_deprec("who", "author")
        kwargs['when_timestamp'] = handle_deprec("when", "when_timestamp")

        # timestamp must be an epoch timestamp now
        if isinstance(kwargs.get('when_timestamp'), datetime.datetime):
            kwargs['when_timestamp'] = datetime2epoch(kwargs['when_timestamp'])

        # unicodify stuff
        for k in ('comments', 'author', 'revision', 'branch', 'category',
                  'revlink', 'repository', 'codebase', 'project'):
            if k in kwargs:
                kwargs[k] = ascii2unicode(kwargs[k])
        if kwargs.get('files'):
            kwargs['files'] = [ascii2unicode(f)
                               for f in kwargs['files']]
        if kwargs.get('properties'):
            kwargs['properties'] = dict((ascii2unicode(k), v)
                                        for k, v in iteritems(kwargs['properties']))

        # pass the converted call on to the data API
        changeid = yield self.data.updates.addChange(**kwargs)

        # and turn that changeid into a change object, since that's what
        # callers expected (and why this method was deprecated)
        chdict = yield self.db.changes.getChange(changeid)
        change = yield changes.Change.fromChdict(self, chdict)
        defer.returnValue(change)

    @defer.inlineCallbacks
    def addBuildset(self, scheduler, **kwargs):
        log.msg("WARNING: master.addBuildset is deprecated; "
                "use the data API update method")
        bsid, brids = yield self.data.updates.addBuildset(
            scheduler=scheduler, **kwargs)
        defer.returnValue((bsid, brids))

    # state maintenance (private)
    def getObjectId(self):
        """
        Return the object id for this master, for associating state with the
        master.

        @returns: ID, via Deferred
        """
        # try to get the cached value
        if self._object_id is not None:
            return defer.succeed(self._object_id)

        # failing that, get it from the DB; multiple calls to this function
        # at the same time will not hurt

        d = self.db.state.getObjectId(self.name,
                                      "buildbot.master.BuildMaster")

        @d.addCallback
        def keep(id):
            self._object_id = id
            return id
        return d

    def _getState(self, name, default=None):
        "private wrapper around C{self.db.state.getState}"
        d = self.getObjectId()

        @d.addCallback
        def get(objectid):
            return self.db.state.getState(objectid, name, default)
        return d

    def _setState(self, name, value):
        "private wrapper around C{self.db.state.setState}"
        d = self.getObjectId()

        @d.addCallback
        def set(objectid):
            return self.db.state.setState(objectid, name, value)
        return d
Esempio n. 3
0
class BuildMaster(service.ReconfigurableServiceMixin, service.MasterService,
                  WorkerAPICompatMixin):

    # frequency with which to reclaim running builds; this should be set to
    # something fairly long, to avoid undue database load
    RECLAIM_BUILD_INTERVAL = 10 * 60

    # multiplier on RECLAIM_BUILD_INTERVAL at which a build is considered
    # unclaimed; this should be at least 2 to avoid false positives
    UNCLAIMED_BUILD_FACTOR = 6

    def __init__(self,
                 basedir,
                 configFileName=None,
                 umask=None,
                 reactor=None,
                 config_loader=None):
        service.AsyncMultiService.__init__(self)

        if reactor is None:
            from twisted.internet import reactor
        self.reactor = reactor

        self.setName("buildmaster")

        self.umask = umask

        self.basedir = basedir
        if basedir is not None:  # None is used in tests
            assert os.path.isdir(self.basedir)

        if config_loader is not None and configFileName is not None:
            raise config.ConfigErrors([
                "Can't specify both `config_loader` and `configFilename`.",
            ])
        elif config_loader is None:
            if configFileName is None:
                configFileName = 'master.cfg'
            config_loader = config.FileLoader(self.basedir, configFileName)
        self.config_loader = config_loader
        self.configFileName = configFileName

        # flag so we don't try to do fancy things before the master is ready
        self._master_initialized = False

        # set up child services
        self.create_child_services()

        # db configured values
        self.configured_db_url = None

        # configuration / reconfiguration handling
        self.config = config.MasterConfig()
        self.reconfig_active = False
        self.reconfig_requested = False
        self.reconfig_notifier = None

        # this stores parameters used in the tac file, and is accessed by the
        # WebStatus to duplicate those values.
        self.log_rotation = LogRotation()

        # local cache for this master's object ID
        self._object_id = None

        # Check environment is sensible
        check_functional_environment(self.config)

        # figure out local hostname
        try:
            self.hostname = os.uname()[1]  # only on unix
        except AttributeError:
            self.hostname = socket.getfqdn()

        # public attributes
        self.name = ("%s:%s" %
                     (self.hostname, os.path.abspath(self.basedir or '.')))
        self.name = self.name.decode('ascii', 'replace')
        self.masterid = None

    def create_child_services(self):
        # note that these are order-dependent.  If you get the order wrong,
        # you'll know it, as the master will fail to start.

        self.metrics = metrics.MetricLogObserver()
        self.metrics.setServiceParent(self)

        self.caches = cache.CacheManager()
        self.caches.setServiceParent(self)

        self.pbmanager = buildbot.pbmanager.PBManager()
        self.pbmanager.setServiceParent(self)

        self.workers = workermanager.WorkerManager(self)
        self.workers.setServiceParent(self)

        self.change_svc = ChangeManager()
        self.change_svc.setServiceParent(self)

        self.botmaster = BotMaster()
        self.botmaster.setServiceParent(self)

        self.scheduler_manager = SchedulerManager()
        self.scheduler_manager.setServiceParent(self)

        self.user_manager = UserManagerManager(self)
        self.user_manager.setServiceParent(self)

        self.db = dbconnector.DBConnector(self.basedir)
        self.db.setServiceParent(self)

        self.wamp = wampconnector.WampConnector()
        self.wamp.setServiceParent(self)

        self.mq = mqconnector.MQConnector()
        self.mq.setServiceParent(self)

        self.data = dataconnector.DataConnector()
        self.data.setServiceParent(self)

        self.www = wwwservice.WWWService()
        self.www.setServiceParent(self)

        self.debug = debug.DebugServices()
        self.debug.setServiceParent(self)

        self.status = Status()
        self.status.setServiceParent(self)

        self.service_manager = service.BuildbotServiceManager()
        self.service_manager.setServiceParent(self)
        self.service_manager.reconfig_priority = 1000

        self.masterHouskeepingTimer = 0

        @defer.inlineCallbacks
        def heartbeat():
            if self.masterid is not None:
                yield self.data.updates.masterActive(name=self.name,
                                                     masterid=self.masterid)
            # force housekeeping once a day
            yield self.data.updates.expireMasters(
                (self.masterHouskeepingTimer % (24 * 60)) == 0)
            self.masterHouskeepingTimer += 1

        self.masterHeartbeatService = internet.TimerService(60, heartbeat)
        # we do setServiceParent only when the master is configured
        # master should advertise itself only at that time

    # setup and reconfig handling

    _already_started = False

    @defer.inlineCallbacks
    def startService(self):
        assert not self._already_started, "can only start the master once"
        self._already_started = True

        log.msg("Starting BuildMaster -- buildbot.version: %s" %
                buildbot.version)

        # Set umask
        if self.umask is not None:
            os.umask(self.umask)

        # first, apply all monkeypatches
        monkeypatches.patch_all()

        # we want to wait until the reactor is running, so we can call
        # reactor.stop() for fatal errors
        d = defer.Deferred()
        self.reactor.callWhenRunning(d.callback, None)
        yield d

        startup_succeed = False
        try:
            # load the configuration file, treating errors as fatal
            try:
                # run the master.cfg in thread, so that it can use blocking
                # code
                self.config = yield threads.deferToThreadPool(
                    self.reactor, self.reactor.getThreadPool(),
                    self.config_loader.loadConfig)

            except config.ConfigErrors as e:
                log.msg("Configuration Errors:")
                for msg in e.errors:
                    log.msg("  " + msg)
                log.msg("Halting master.")
                self.reactor.stop()
                return
            except Exception:
                log.err(failure.Failure(), 'while starting BuildMaster')
                self.reactor.stop()
                return

            # set up services that need access to the config before everything
            # else gets told to reconfig
            try:
                yield self.db.setup()
            except exceptions.DatabaseNotReadyError:
                # (message was already logged)
                self.reactor.stop()
                return

            self.mq.setup()

            if hasattr(signal, "SIGHUP"):

                def sighup(*args):
                    eventually(self.reconfig)

                signal.signal(signal.SIGHUP, sighup)

            if hasattr(signal, "SIGUSR1"):

                def sigusr1(*args):
                    self.reactor.callLater(0, self.botmaster.cleanShutdown)

                signal.signal(signal.SIGUSR1, sigusr1)

            # get the masterid so other services can use it in
            # startup/reconfig.  This goes directly to the DB since the data
            # API isn't initialized yet, and anyway, this method is aware of
            # the DB API since it just called its setup function
            self.masterid = yield self.db.masters.findMasterId(name=self.name)

            # mark this master as stopped, in case it crashed before
            yield self.data.updates.masterStopped(name=self.name,
                                                  masterid=self.masterid)

            # call the parent method
            yield service.AsyncMultiService.startService(self)

            # give all services a chance to load the new configuration, rather
            # than the base configuration
            yield self.reconfigServiceWithBuildbotConfig(self.config)

            # mark the master as active now that mq is running
            yield self.data.updates.masterActive(name=self.name,
                                                 masterid=self.masterid)

            startup_succeed = True
        except Exception:
            f = failure.Failure()
            log.err(f, 'while starting BuildMaster')
            self.reactor.stop()

        finally:
            if startup_succeed:
                log.msg("BuildMaster is running")
            else:
                log.msg("BuildMaster startup failed")

            self._master_initialized = True

    @defer.inlineCallbacks
    def stopService(self):
        if self.masterid is not None:
            yield self.data.updates.masterStopped(name=self.name,
                                                  masterid=self.masterid)
        if self.running:
            yield service.AsyncMultiService.stopService(self)

        log.msg("BuildMaster is stopped")
        self._master_initialized = False

    def reconfig(self):
        # this method wraps doConfig, ensuring it is only ever called once at
        # a time, and alerting the user if the reconfig takes too long
        if self.reconfig_active:
            log.msg("reconfig already active; will reconfig again after")
            self.reconfig_requested = True
            return

        self.reconfig_active = self.reactor.seconds()
        metrics.MetricCountEvent.log("loaded_config", 1)

        # notify every 10 seconds that the reconfig is still going on, although
        # reconfigs should not take that long!
        self.reconfig_notifier = task.LoopingCall(
            lambda: log.msg("reconfig is ongoing for %d s" %
                            (self.reactor.seconds() - self.reconfig_active)))
        self.reconfig_notifier.start(10, now=False)

        timer = metrics.Timer("BuildMaster.reconfig")
        timer.start()

        d = self.doReconfig()

        @d.addBoth
        def cleanup(res):
            timer.stop()
            self.reconfig_notifier.stop()
            self.reconfig_notifier = None
            self.reconfig_active = False
            if self.reconfig_requested:
                self.reconfig_requested = False
                self.reconfig()
            return res

        @d.addCallback
        def advertiseItself(res):
            if self.masterHeartbeatService.parent is None:
                self.masterHeartbeatService.setServiceParent(self)

        d.addErrback(log.err, 'while reconfiguring')

        return d  # for tests

    @defer.inlineCallbacks
    def doReconfig(self):
        log.msg("beginning configuration update")
        changes_made = False
        failed = False
        try:
            # Run the master.cfg in thread, so that it cas use blocking code
            new_config = yield threads.deferToThreadPool(
                self.reactor, self.reactor.getThreadPool(),
                self.config_loader.loadConfig)
            changes_made = True
            self.config = new_config

            yield self.reconfigServiceWithBuildbotConfig(new_config)

        except config.ConfigErrors as e:
            for msg in e.errors:
                log.msg(msg)
            failed = True

        except Exception:
            log.err(failure.Failure(), 'during reconfig:')
            failed = True

        if failed:
            if changes_made:
                log.msg("WARNING: reconfig partially applied; master "
                        "may malfunction")
            else:
                log.msg("reconfig aborted without making any changes")
        else:
            log.msg("configuration update complete")

    def reconfigServiceWithBuildbotConfig(self, new_config):
        if self.configured_db_url is None:
            self.configured_db_url = new_config.db['db_url']
        elif (self.configured_db_url != new_config.db['db_url']):
            config.error(
                "Cannot change c['db']['db_url'] after the master has started",
            )

        if self.config.mq['type'] != new_config.mq['type']:
            raise config.ConfigErrors([
                "Cannot change c['mq']['type'] after the master has started",
            ])

        return service.ReconfigurableServiceMixin.reconfigServiceWithBuildbotConfig(
            self, new_config)

    # informational methods
    def allSchedulers(self):
        return list(self.scheduler_manager)

    def getStatus(self):
        """
        @rtype: L{buildbot.status.builder.Status}
        """
        return self.status

    # triggering methods

    @defer.inlineCallbacks
    def addChange(self, who=None, files=None, comments=None, **kwargs):
        # deprecated in 0.9.0; will be removed in 1.0.0
        log.msg("WARNING: change source is using deprecated "
                "self.master.addChange method; this method will disappear in "
                "Buildbot-1.0.0")
        # handle positional arguments
        kwargs['who'] = who
        kwargs['files'] = files
        kwargs['comments'] = comments

        def handle_deprec(oldname, newname):
            if oldname not in kwargs:
                return
            old = kwargs.pop(oldname)
            if old is not None:
                if kwargs.get(newname) is None:
                    log.msg("WARNING: change source is using deprecated "
                            "addChange parameter '%s'" % oldname)
                    return old
                raise TypeError("Cannot provide '%s' and '%s' to addChange" %
                                (oldname, newname))
            return kwargs.get(newname)

        kwargs['author'] = handle_deprec("who", "author")
        kwargs['when_timestamp'] = handle_deprec("when", "when_timestamp")

        # timestamp must be an epoch timestamp now
        if isinstance(kwargs.get('when_timestamp'), datetime.datetime):
            kwargs['when_timestamp'] = datetime2epoch(kwargs['when_timestamp'])

        # unicodify stuff
        for k in ('comments', 'author', 'revision', 'branch', 'category',
                  'revlink', 'repository', 'codebase', 'project'):
            if k in kwargs:
                kwargs[k] = ascii2unicode(kwargs[k])
        if kwargs.get('files'):
            kwargs['files'] = [ascii2unicode(f) for f in kwargs['files']]
        if kwargs.get('properties'):
            kwargs['properties'] = dict(
                (ascii2unicode(k), v)
                for k, v in iteritems(kwargs['properties']))

        # pass the converted call on to the data API
        changeid = yield self.data.updates.addChange(**kwargs)

        # and turn that changeid into a change object, since that's what
        # callers expected (and why this method was deprecated)
        chdict = yield self.db.changes.getChange(changeid)
        change = yield changes.Change.fromChdict(self, chdict)
        defer.returnValue(change)

    @defer.inlineCallbacks
    def addBuildset(self, scheduler, **kwargs):
        log.msg("WARNING: master.addBuildset is deprecated; "
                "use the data API update method")
        bsid, brids = yield self.data.updates.addBuildset(scheduler=scheduler,
                                                          **kwargs)
        defer.returnValue((bsid, brids))

    # state maintenance (private)
    def getObjectId(self):
        """
        Return the object id for this master, for associating state with the
        master.

        @returns: ID, via Deferred
        """
        # try to get the cached value
        if self._object_id is not None:
            return defer.succeed(self._object_id)

        # failing that, get it from the DB; multiple calls to this function
        # at the same time will not hurt

        d = self.db.state.getObjectId(self.name, "buildbot.master.BuildMaster")

        @d.addCallback
        def keep(id):
            self._object_id = id
            return id

        return d

    def _getState(self, name, default=None):
        "private wrapper around C{self.db.state.getState}"
        d = self.getObjectId()

        @d.addCallback
        def get(objectid):
            return self.db.state.getState(objectid, name, default)

        return d

    def _setState(self, name, value):
        "private wrapper around C{self.db.state.setState}"
        d = self.getObjectId()

        @d.addCallback
        def set(objectid):
            return self.db.state.setState(objectid, name, value)

        return d
Esempio n. 4
0
class BuildMaster(config.ReconfigurableServiceMixin,
                  service.AsyncMultiService):

    # frequency with which to reclaim running builds; this should be set to
    # something fairly long, to avoid undue database load
    RECLAIM_BUILD_INTERVAL = 10 * 60

    # multiplier on RECLAIM_BUILD_INTERVAL at which a build is considered
    # unclaimed; this should be at least 2 to avoid false positives
    UNCLAIMED_BUILD_FACTOR = 6

    def __init__(self, basedir, configFileName="master.cfg", umask=None):
        service.AsyncMultiService.__init__(self)
        self.setName("buildmaster")

        self.umask = umask

        self.basedir = basedir
        if basedir is not None:  # None is used in tests
            assert os.path.isdir(self.basedir)
        self.configFileName = configFileName

        # flag so we don't try to do fancy things before the master is ready
        self._master_initialized = False

        # set up child services
        self.create_child_services()

        # db configured values
        self.configured_db_url = None

        # configuration / reconfiguration handling
        self.config = config.MasterConfig()
        self.reconfig_active = False
        self.reconfig_requested = False
        self.reconfig_notifier = None

        # this stores parameters used in the tac file, and is accessed by the
        # WebStatus to duplicate those values.
        self.log_rotation = LogRotation()

        # local cache for this master's object ID
        self._object_id = None

        # Check environment is sensible
        check_functional_environment(self.config)

        # figure out local hostname
        try:
            self.hostname = os.uname()[1]  # only on unix
        except AttributeError:
            self.hostname = socket.getfqdn()

        # public attributes
        self.name = ("%s:%s" %
                     (self.hostname, os.path.abspath(self.basedir or '.')))
        self.name = self.name.decode('ascii', 'replace')
        self.masterid = None

    def create_child_services(self):
        # note that these are order-dependent.  If you get the order wrong,
        # you'll know it, as the master will fail to start.

        self.metrics = metrics.MetricLogObserver()
        self.metrics.setServiceParent(self)

        self.caches = cache.CacheManager()
        self.caches.setServiceParent(self)

        self.pbmanager = buildbot.pbmanager.PBManager()
        self.pbmanager.setServiceParent(self)

        self.buildslaves = bslavemanager.BuildslaveManager(self)
        self.buildslaves.setServiceParent(self)

        self.change_svc = ChangeManager(self)
        self.change_svc.setServiceParent(self)

        self.botmaster = BotMaster(self)
        self.botmaster.setServiceParent(self)

        self.scheduler_manager = SchedulerManager(self)
        self.scheduler_manager.setServiceParent(self)

        self.user_manager = UserManagerManager(self)
        self.user_manager.setServiceParent(self)

        self.db = dbconnector.DBConnector(self, self.basedir)
        self.db.setServiceParent(self)

        self.mq = mqconnector.MQConnector(self)
        self.mq.setServiceParent(self)

        self.data = dataconnector.DataConnector(self)
        self.data.setServiceParent(self)

        self.www = wwwservice.WWWService(self)
        self.www.setServiceParent(self)

        self.debug = debug.DebugServices(self)
        self.debug.setServiceParent(self)

        self.status = Status(self)
        self.status.setServiceParent(self)

        @defer.inlineCallbacks
        def heartbeat():
            if self.masterid is not None:
                yield self.data.updates.masterActive(name=self.name,
                                                     masterid=self.masterid)
            yield self.data.updates.expireMasters()

        self.masterHeartbeatService = internet.TimerService(60, heartbeat)

    # setup and reconfig handling

    _already_started = False

    @defer.inlineCallbacks
    def startService(self, _reactor=reactor):
        assert not self._already_started, "can only start the master once"
        self._already_started = True

        log.msg("Starting BuildMaster -- buildbot.version: %s" %
                buildbot.version)

        # Set umask
        if self.umask is not None:
            os.umask(self.umask)

        # first, apply all monkeypatches
        monkeypatches.patch_all()

        # we want to wait until the reactor is running, so we can call
        # reactor.stop() for fatal errors
        d = defer.Deferred()
        _reactor.callWhenRunning(d.callback, None)
        yield d

        try:
            # load the configuration file, treating errors as fatal
            try:
                self.config = config.MasterConfig.loadConfig(
                    self.basedir, self.configFileName)

            except config.ConfigErrors, e:
                log.msg("Configuration Errors:")
                for msg in e.errors:
                    log.msg("  " + msg)
                log.msg("Halting master.")
                _reactor.stop()
                return
            except:
                log.err(failure.Failure(), 'while starting BuildMaster')
                _reactor.stop()
                return

            # set up services that need access to the config before everything
            # else gets told to reconfig
            try:
                yield self.db.setup()
            except exceptions.DatabaseNotReadyError:
                # (message was already logged)
                _reactor.stop()
                return

            self.mq.setup()

            if hasattr(signal, "SIGHUP"):

                def sighup(*args):
                    eventually(self.reconfig)

                signal.signal(signal.SIGHUP, sighup)

            if hasattr(signal, "SIGUSR1"):

                def sigusr1(*args):
                    _reactor.callLater(0, self.botmaster.cleanShutdown)

                signal.signal(signal.SIGUSR1, sigusr1)

            # get the masterid so other services can use it in
            # startup/reconfig.  This goes directly to the DB since the data
            # API isn't initialized yet, and anyway, this method is aware of
            # the DB API since it just called its setup function
            self.masterid = yield self.db.masters.findMasterId(name=self.name)

            yield self.doMasterHouseKeeping(self.masterid)

            # call the parent method
            yield service.AsyncMultiService.startService(self)

            # give all services a chance to load the new configuration, rather
            # than the base configuration
            yield self.reconfigService(self.config)

            # mark the master as active now that mq is running
            yield self.data.updates.masterActive(name=self.name,
                                                 masterid=self.masterid)
        except:
Esempio n. 5
0
class BuildMaster(config.ReconfigurableServiceMixin, service.AsyncMultiService):

    # frequency with which to reclaim running builds; this should be set to
    # something fairly long, to avoid undue database load
    RECLAIM_BUILD_INTERVAL = 10 * 60

    # multiplier on RECLAIM_BUILD_INTERVAL at which a build is considered
    # unclaimed; this should be at least 2 to avoid false positives
    UNCLAIMED_BUILD_FACTOR = 6

    def __init__(self, basedir, configFileName="master.cfg", umask=None):
        service.AsyncMultiService.__init__(self)
        self.setName("buildmaster")

        self.umask = umask

        self.basedir = basedir
        if basedir is not None:  # None is used in tests
            assert os.path.isdir(self.basedir)
        self.configFileName = configFileName

        # flag so we don't try to do fancy things before the master is ready
        self._master_initialized = False

        # set up child services
        self.create_child_services()

        # db configured values
        self.configured_db_url = None

        # configuration / reconfiguration handling
        self.config = config.MasterConfig()
        self.reconfig_active = False
        self.reconfig_requested = False
        self.reconfig_notifier = None

        # this stores parameters used in the tac file, and is accessed by the
        # WebStatus to duplicate those values.
        self.log_rotation = LogRotation()

        # local cache for this master's object ID
        self._object_id = None

        # Check environment is sensible
        check_functional_environment(self.config)

        # figure out local hostname
        try:
            self.hostname = os.uname()[1]  # only on unix
        except AttributeError:
            self.hostname = socket.getfqdn()

        # public attributes
        self.name = ("%s:%s" % (self.hostname,
                                os.path.abspath(self.basedir or '.')))
        self.name = self.name.decode('ascii', 'replace')
        self.masterid = None

    def create_child_services(self):
        # note that these are order-dependent.  If you get the order wrong,
        # you'll know it, as the master will fail to start.

        self.metrics = metrics.MetricLogObserver()
        self.metrics.setServiceParent(self)

        self.caches = cache.CacheManager()
        self.caches.setServiceParent(self)

        self.pbmanager = buildbot.pbmanager.PBManager()
        self.pbmanager.setServiceParent(self)

        self.buildslaves = bslavemanager.BuildslaveManager(self)
        self.buildslaves.setServiceParent(self)

        self.change_svc = ChangeManager(self)
        self.change_svc.setServiceParent(self)

        self.botmaster = BotMaster(self)
        self.botmaster.setServiceParent(self)

        self.scheduler_manager = SchedulerManager(self)
        self.scheduler_manager.setServiceParent(self)

        self.user_manager = UserManagerManager(self)
        self.user_manager.setServiceParent(self)

        self.db = dbconnector.DBConnector(self, self.basedir)
        self.db.setServiceParent(self)

        self.mq = mqconnector.MQConnector(self)
        self.mq.setServiceParent(self)

        self.data = dataconnector.DataConnector(self)
        self.data.setServiceParent(self)

        self.www = wwwservice.WWWService(self)
        self.www.setServiceParent(self)

        self.debug = debug.DebugServices(self)
        self.debug.setServiceParent(self)

        self.status = Status(self)
        self.status.setServiceParent(self)

        @defer.inlineCallbacks
        def heartbeat():
            if self.masterid is not None:
                yield self.data.updates.masterActive(name=self.name,
                                                     masterid=self.masterid)
            yield self.data.updates.expireMasters()
        self.masterHeartbeatService = internet.TimerService(60, heartbeat)

    # setup and reconfig handling

    _already_started = False

    @defer.inlineCallbacks
    def startService(self, _reactor=reactor):
        assert not self._already_started, "can only start the master once"
        self._already_started = True

        log.msg("Starting BuildMaster -- buildbot.version: %s" %
                buildbot.version)

        # Set umask
        if self.umask is not None:
            os.umask(self.umask)

        # first, apply all monkeypatches
        monkeypatches.patch_all()

        # we want to wait until the reactor is running, so we can call
        # reactor.stop() for fatal errors
        d = defer.Deferred()
        _reactor.callWhenRunning(d.callback, None)
        yield d

        try:
            # load the configuration file, treating errors as fatal
            try:
                self.config = config.MasterConfig.loadConfig(self.basedir,
                                                             self.configFileName)

            except config.ConfigErrors, e:
                log.msg("Configuration Errors:")
                for msg in e.errors:
                    log.msg("  " + msg)
                log.msg("Halting master.")
                _reactor.stop()
                return
            except:
                log.err(failure.Failure(), 'while starting BuildMaster')
                _reactor.stop()
                return

            # set up services that need access to the config before everything
            # else gets told to reconfig
            try:
                yield self.db.setup()
            except exceptions.DatabaseNotReadyError:
                # (message was already logged)
                _reactor.stop()
                return

            self.mq.setup()

            if hasattr(signal, "SIGHUP"):
                def sighup(*args):
                    eventually(self.reconfig)
                signal.signal(signal.SIGHUP, sighup)

            if hasattr(signal, "SIGUSR1"):
                def sigusr1(*args):
                    _reactor.callLater(0, self.botmaster.cleanShutdown)
                signal.signal(signal.SIGUSR1, sigusr1)

            # get the masterid so other services can use it in
            # startup/reconfig.  This goes directly to the DB since the data
            # API isn't initialized yet, and anyway, this method is aware of
            # the DB API since it just called its setup function
            self.masterid = yield self.db.masters.findMasterId(
                name=self.name)

            yield self.doMasterHouseKeeping(self.masterid)

            # call the parent method
            yield service.AsyncMultiService.startService(self)

            # give all services a chance to load the new configuration, rather
            # than the base configuration
            yield self.reconfigService(self.config)

            # mark the master as active now that mq is running
            yield self.data.updates.masterActive(
                name=self.name,
                masterid=self.masterid)
        except:
Esempio n. 6
0
class BuildMaster(service.ReconfigurableServiceMixin, service.MasterService,
                  WorkerAPICompatMixin):

    # multiplier on RECLAIM_BUILD_INTERVAL at which a build is considered
    # unclaimed; this should be at least 2 to avoid false positives
    UNCLAIMED_BUILD_FACTOR = 6

    def __init__(self, basedir, configFileName=None, umask=None, reactor=None, config_loader=None):
        service.AsyncMultiService.__init__(self)

        if reactor is None:
            from twisted.internet import reactor
        self.reactor = reactor

        self.setName("buildmaster")

        self.umask = umask

        self.basedir = basedir
        if basedir is not None:  # None is used in tests
            assert os.path.isdir(self.basedir)

        if config_loader is not None and configFileName is not None:
            raise config.ConfigErrors([
                "Can't specify both `config_loader` and `configFilename`.",
            ])
        elif config_loader is None:
            if configFileName is None:
                configFileName = 'master.cfg'
            config_loader = config.FileLoader(self.basedir, configFileName)
        self.config_loader = config_loader
        self.configFileName = configFileName

        # flag so we don't try to do fancy things before the master is ready
        self._master_initialized = False
        self.initLock = defer.DeferredLock()

        # set up child services
        self.create_child_services()

        # db configured values
        self.configured_db_url = None

        # configuration / reconfiguration handling
        self.config = config.MasterConfig()
        self.reconfig_active = False
        self.reconfig_requested = False
        self.reconfig_notifier = None

        # this stores parameters used in the tac file, and is accessed by the
        # WebStatus to duplicate those values.
        self.log_rotation = LogRotation()

        # local cache for this master's object ID
        self._object_id = None

        # Check environment is sensible
        check_functional_environment(self.config)

        # figure out local hostname
        try:
            self.hostname = os.uname()[1]  # only on unix
        except AttributeError:
            self.hostname = socket.getfqdn()

        # public attributes
        self.name = ("%s:%s" % (self.hostname,
                                os.path.abspath(self.basedir or '.')))
        if isinstance(self.name, bytes):
            self.name = self.name.decode('ascii', 'replace')
        self.masterid = None

    def create_child_services(self):
        # note that these are order-dependent.  If you get the order wrong,
        # you'll know it, as the master will fail to start.

        self.metrics = metrics.MetricLogObserver()
        self.metrics.setServiceParent(self)

        self.caches = cache.CacheManager()
        self.caches.setServiceParent(self)

        self.pbmanager = buildbot.pbmanager.PBManager()
        self.pbmanager.setServiceParent(self)

        self.workers = workermanager.WorkerManager(self)
        self.workers.setServiceParent(self)

        self.change_svc = ChangeManager()
        self.change_svc.setServiceParent(self)

        self.botmaster = BotMaster()
        self.botmaster.setServiceParent(self)

        self.scheduler_manager = SchedulerManager()
        self.scheduler_manager.setServiceParent(self)

        self.user_manager = UserManagerManager(self)
        self.user_manager.setServiceParent(self)

        self.db = dbconnector.DBConnector(self.basedir)
        self.db.setServiceParent(self)

        self.wamp = wampconnector.WampConnector()
        self.wamp.setServiceParent(self)

        self.mq = mqconnector.MQConnector()
        self.mq.setServiceParent(self)

        self.data = dataconnector.DataConnector()
        self.data.setServiceParent(self)

        self.www = wwwservice.WWWService()
        self.www.setServiceParent(self)

        self.debug = debug.DebugServices()
        self.debug.setServiceParent(self)

        self.status = Status()
        self.status.setServiceParent(self)

        self.secrets_manager = SecretManager()
        self.secrets_manager.setServiceParent(self)
        self.secrets_manager.reconfig_priority = 2000

        self.service_manager = service.BuildbotServiceManager()
        self.service_manager.setServiceParent(self)
        self.service_manager.reconfig_priority = 1000

        self.masterHouskeepingTimer = 0

        @defer.inlineCallbacks
        def heartbeat():
            if self.masterid is not None:
                yield self.data.updates.masterActive(name=self.name,
                                                     masterid=self.masterid)
            yield self.data.updates.expireMasters()
        self.masterHeartbeatService = internet.TimerService(60, heartbeat)
        # we do setServiceParent only when the master is configured
        # master should advertise itself only at that time

    # setup and reconfig handling

    _already_started = False

    @defer.inlineCallbacks
    def startService(self):
        assert not self._already_started, "can only start the master once"
        self._already_started = True

        log.msg("Starting BuildMaster -- buildbot.version: %s" %
                buildbot.version)

        # Set umask
        if self.umask is not None:
            os.umask(self.umask)

        # first, apply all monkeypatches
        monkeypatches.patch_all()

        # we want to wait until the reactor is running, so we can call
        # reactor.stop() for fatal errors
        d = defer.Deferred()
        self.reactor.callWhenRunning(d.callback, None)
        yield d

        startup_succeed = False
        try:
            yield self.initLock.acquire()
            # load the configuration file, treating errors as fatal
            try:
                # run the master.cfg in thread, so that it can use blocking
                # code
                self.config = yield threads.deferToThreadPool(
                    self.reactor, self.reactor.getThreadPool(),
                    self.config_loader.loadConfig)

            except config.ConfigErrors as e:
                log.msg("Configuration Errors:")
                for msg in e.errors:
                    log.msg("  " + msg)
                log.msg("Halting master.")
                self.reactor.stop()
                return
            except Exception:
                log.err(failure.Failure(), 'while starting BuildMaster')
                self.reactor.stop()
                return

            # set up services that need access to the config before everything
            # else gets told to reconfig
            try:
                yield self.db.setup()
            except exceptions.DatabaseNotReadyError:
                # (message was already logged)
                self.reactor.stop()
                return

            self.mq.setup()

            if hasattr(signal, "SIGHUP"):
                def sighup(*args):
                    eventually(self.reconfig)
                signal.signal(signal.SIGHUP, sighup)

            if hasattr(signal, "SIGUSR1"):
                def sigusr1(*args):
                    eventually(self.botmaster.cleanShutdown)
                signal.signal(signal.SIGUSR1, sigusr1)

            # get the masterid so other services can use it in
            # startup/reconfig.  This goes directly to the DB since the data
            # API isn't initialized yet, and anyway, this method is aware of
            # the DB API since it just called its setup function
            self.masterid = yield self.db.masters.findMasterId(
                name=self.name)

            # mark this master as stopped, in case it crashed before
            yield self.data.updates.masterStopped(name=self.name,
                                                  masterid=self.masterid)

            # call the parent method
            yield service.AsyncMultiService.startService(self)

            # We make sure the housekeeping is done before configuring in order to cleanup
            # any remaining claimed schedulers or change sources from zombie
            # masters
            yield self.data.updates.expireMasters(forceHouseKeeping=True)

            # give all services a chance to load the new configuration, rather
            # than the base configuration
            yield self.reconfigServiceWithBuildbotConfig(self.config)

            # Mark the master as active now that mq is running
            yield self.data.updates.masterActive(name=self.name,
                                                 masterid=self.masterid)

            # Start the heartbeat timer
            yield self.masterHeartbeatService.setServiceParent(self)

            # send the statistics to buildbot.net, without waiting
            self.sendBuildbotNetUsageData()
            startup_succeed = True
        except Exception:
            f = failure.Failure()
            log.err(f, 'while starting BuildMaster')
            self.reactor.stop()

        finally:
            if startup_succeed:
                log.msg("BuildMaster is running")
            else:
                log.msg("BuildMaster startup failed")

            yield self.initLock.release()
            self._master_initialized = True

    def sendBuildbotNetUsageData(self):
        if "TRIAL_PYTHONPATH" in os.environ and self.config.buildbotNetUsageData is not None:
            raise RuntimeError(
                "Should not enable buildbotNetUsageData in trial tests!")
        sendBuildbotNetUsageData(self)

    @defer.inlineCallbacks
    def stopService(self):
        try:
            yield self.initLock.acquire()
            if self.masterid is not None:
                yield self.data.updates.masterStopped(
                    name=self.name, masterid=self.masterid)
            if self.running:
                yield self.botmaster.cleanShutdown(
                    quickMode=True, stopReactor=False)
                yield service.AsyncMultiService.stopService(self)

            log.msg("BuildMaster is stopped")
            self._master_initialized = False
        finally:
            yield self.initLock.release()

    def reconfig(self):
        # this method wraps doConfig, ensuring it is only ever called once at
        # a time, and alerting the user if the reconfig takes too long
        if self.reconfig_active:
            log.msg("reconfig already active; will reconfig again after")
            self.reconfig_requested = True
            return

        self.reconfig_active = self.reactor.seconds()
        metrics.MetricCountEvent.log("loaded_config", 1)

        # notify every 10 seconds that the reconfig is still going on, although
        # reconfigs should not take that long!
        self.reconfig_notifier = task.LoopingCall(lambda:
                                                  log.msg("reconfig is ongoing for %d s" %
                                                          (self.reactor.seconds() - self.reconfig_active)))
        self.reconfig_notifier.start(10, now=False)

        timer = metrics.Timer("BuildMaster.reconfig")
        timer.start()

        d = self.doReconfig()

        @d.addBoth
        def cleanup(res):
            timer.stop()
            self.reconfig_notifier.stop()
            self.reconfig_notifier = None
            self.reconfig_active = False
            if self.reconfig_requested:
                self.reconfig_requested = False
                self.reconfig()
            return res

        d.addErrback(log.err, 'while reconfiguring')

        return d  # for tests

    @defer.inlineCallbacks
    def doReconfig(self):
        log.msg("beginning configuration update")
        changes_made = False
        failed = False
        try:
            yield self.initLock.acquire()
            # Run the master.cfg in thread, so that it can use blocking code
            new_config = yield threads.deferToThreadPool(
                self.reactor, self.reactor.getThreadPool(),
                self.config_loader.loadConfig)
            changes_made = True
            self.config = new_config

            yield self.reconfigServiceWithBuildbotConfig(new_config)

        except config.ConfigErrors as e:
            for msg in e.errors:
                log.msg(msg)
            failed = True

        except Exception:
            log.err(failure.Failure(), 'during reconfig:')
            failed = True

        finally:
            yield self.initLock.release()

        if failed:
            if changes_made:
                log.msg("WARNING: reconfig partially applied; master "
                        "may malfunction")
            else:
                log.msg("reconfig aborted without making any changes")
        else:
            log.msg("configuration update complete")

    def reconfigServiceWithBuildbotConfig(self, new_config):
        if self.configured_db_url is None:
            self.configured_db_url = new_config.db['db_url']
        elif (self.configured_db_url != new_config.db['db_url']):
            config.error(
                "Cannot change c['db']['db_url'] after the master has started",
            )

        if self.config.mq['type'] != new_config.mq['type']:
            raise config.ConfigErrors([
                "Cannot change c['mq']['type'] after the master has started",
            ])

        return service.ReconfigurableServiceMixin.reconfigServiceWithBuildbotConfig(self,
                                                                                    new_config)

    # informational methods
    def allSchedulers(self):
        return list(self.scheduler_manager)

    def getStatus(self):
        """
        @rtype: L{buildbot.status.builder.Status}
        """
        return self.status

    # state maintenance (private)
    def getObjectId(self):
        """
        Return the object id for this master, for associating state with the
        master.

        @returns: ID, via Deferred
        """
        # try to get the cached value
        if self._object_id is not None:
            return defer.succeed(self._object_id)

        # failing that, get it from the DB; multiple calls to this function
        # at the same time will not hurt

        d = self.db.state.getObjectId(self.name,
                                      "buildbot.master.BuildMaster")

        @d.addCallback
        def keep(id):
            self._object_id = id
            return id
        return d

    def _getState(self, name, default=None):
        "private wrapper around C{self.db.state.getState}"
        d = self.getObjectId()

        @d.addCallback
        def get(objectid):
            return self.db.state.getState(objectid, name, default)
        return d

    def _setState(self, name, value):
        "private wrapper around C{self.db.state.setState}"
        d = self.getObjectId()

        @d.addCallback
        def set(objectid):
            return self.db.state.setState(objectid, name, value)
        return d
Esempio n. 7
0
class BuildMaster(service.ReconfigurableServiceMixin, service.MasterService,
                  WorkerAPICompatMixin):

    # multiplier on RECLAIM_BUILD_INTERVAL at which a build is considered
    # unclaimed; this should be at least 2 to avoid false positives
    UNCLAIMED_BUILD_FACTOR = 6

    def __init__(self,
                 basedir,
                 configFileName=None,
                 umask=None,
                 reactor=None,
                 config_loader=None):
        service.AsyncMultiService.__init__(self)

        if reactor is None:
            from twisted.internet import reactor
        self.reactor = reactor

        self.setName("buildmaster")

        self.umask = umask

        self.basedir = basedir
        if basedir is not None:  # None is used in tests
            assert os.path.isdir(self.basedir)

        if config_loader is not None and configFileName is not None:
            raise config.ConfigErrors([
                "Can't specify both `config_loader` and `configFilename`.",
            ])
        elif config_loader is None:
            if configFileName is None:
                configFileName = 'master.cfg'
            config_loader = config.FileLoader(self.basedir, configFileName)
        self.config_loader = config_loader
        self.configFileName = configFileName

        # flag so we don't try to do fancy things before the master is ready
        self._master_initialized = False
        self.initLock = defer.DeferredLock()

        # set up child services
        self.create_child_services()

        # db configured values
        self.configured_db_url = None

        # configuration / reconfiguration handling
        self.config = config.MasterConfig()
        self.reconfig_active = False
        self.reconfig_requested = False
        self.reconfig_notifier = None

        # this stores parameters used in the tac file, and is accessed by the
        # WebStatus to duplicate those values.
        self.log_rotation = LogRotation()

        # local cache for this master's object ID
        self._object_id = None

        # Check environment is sensible
        check_functional_environment(self.config)

        # figure out local hostname
        try:
            self.hostname = os.uname()[1]  # only on unix
        except AttributeError:
            self.hostname = socket.getfqdn()

        # public attributes
        self.name = ("%s:%s" %
                     (self.hostname, os.path.abspath(self.basedir or '.')))
        if isinstance(self.name, bytes):
            self.name = self.name.decode('ascii', 'replace')
        self.masterid = None

    def create_child_services(self):
        # note that these are order-dependent.  If you get the order wrong,
        # you'll know it, as the master will fail to start.

        self.metrics = metrics.MetricLogObserver()
        self.metrics.setServiceParent(self)

        self.caches = cache.CacheManager()
        self.caches.setServiceParent(self)

        self.pbmanager = buildbot.pbmanager.PBManager()
        self.pbmanager.setServiceParent(self)

        self.workers = workermanager.WorkerManager(self)
        self.workers.setServiceParent(self)

        self.change_svc = ChangeManager()
        self.change_svc.setServiceParent(self)

        self.botmaster = BotMaster()
        self.botmaster.setServiceParent(self)

        self.scheduler_manager = SchedulerManager()
        self.scheduler_manager.setServiceParent(self)

        self.user_manager = UserManagerManager(self)
        self.user_manager.setServiceParent(self)

        self.db = dbconnector.DBConnector(self.basedir)
        self.db.setServiceParent(self)

        self.wamp = wampconnector.WampConnector()
        self.wamp.setServiceParent(self)

        self.mq = mqconnector.MQConnector()
        self.mq.setServiceParent(self)

        self.data = dataconnector.DataConnector()
        self.data.setServiceParent(self)

        self.www = wwwservice.WWWService()
        self.www.setServiceParent(self)

        self.debug = debug.DebugServices()
        self.debug.setServiceParent(self)

        self.status = Status()
        self.status.setServiceParent(self)

        self.secrets_manager = SecretManager()
        self.secrets_manager.setServiceParent(self)
        self.secrets_manager.reconfig_priority = 2000

        self.service_manager = service.BuildbotServiceManager()
        self.service_manager.setServiceParent(self)
        self.service_manager.reconfig_priority = 1000

        self.masterHouskeepingTimer = 0

        @defer.inlineCallbacks
        def heartbeat():
            if self.masterid is not None:
                yield self.data.updates.masterActive(name=self.name,
                                                     masterid=self.masterid)
            yield self.data.updates.expireMasters()

        self.masterHeartbeatService = internet.TimerService(60, heartbeat)
        self.masterHeartbeatService.clock = self.reactor
        # we do setServiceParent only when the master is configured
        # master should advertise itself only at that time

    # setup and reconfig handling

    _already_started = False

    @defer.inlineCallbacks
    def startService(self):
        assert not self._already_started, "can only start the master once"
        self._already_started = True

        log.msg("Starting BuildMaster -- buildbot.version: %s" %
                buildbot.version)

        # Set umask
        if self.umask is not None:
            os.umask(self.umask)

        # first, apply all monkeypatches
        monkeypatches.patch_all()

        # we want to wait until the reactor is running, so we can call
        # reactor.stop() for fatal errors
        d = defer.Deferred()
        self.reactor.callWhenRunning(d.callback, None)
        yield d

        startup_succeed = False
        try:
            yield self.initLock.acquire()
            # load the configuration file, treating errors as fatal
            try:
                # run the master.cfg in thread, so that it can use blocking
                # code
                self.config = yield threads.deferToThreadPool(
                    self.reactor, self.reactor.getThreadPool(),
                    self.config_loader.loadConfig)

            except config.ConfigErrors as e:
                log.msg("Configuration Errors:")
                for msg in e.errors:
                    log.msg("  " + msg)
                log.msg("Halting master.")
                self.reactor.stop()
                return
            except Exception:
                log.err(failure.Failure(), 'while starting BuildMaster')
                self.reactor.stop()
                return

            # set up services that need access to the config before everything
            # else gets told to reconfig
            try:
                yield self.db.setup()
            except exceptions.DatabaseNotReadyError:
                # (message was already logged)
                self.reactor.stop()
                return

            self.mq.setup()

            if hasattr(signal, "SIGHUP"):

                def sighup(*args):
                    eventually(self.reconfig)

                signal.signal(signal.SIGHUP, sighup)

            if hasattr(signal, "SIGUSR1"):

                def sigusr1(*args):
                    eventually(self.botmaster.cleanShutdown)

                signal.signal(signal.SIGUSR1, sigusr1)

            # get the masterid so other services can use it in
            # startup/reconfig.  This goes directly to the DB since the data
            # API isn't initialized yet, and anyway, this method is aware of
            # the DB API since it just called its setup function
            self.masterid = yield self.db.masters.findMasterId(name=self.name)

            # mark this master as stopped, in case it crashed before
            yield self.data.updates.masterStopped(name=self.name,
                                                  masterid=self.masterid)

            # call the parent method
            yield service.AsyncMultiService.startService(self)

            # We make sure the housekeeping is done before configuring in order to cleanup
            # any remaining claimed schedulers or change sources from zombie
            # masters
            yield self.data.updates.expireMasters(forceHouseKeeping=True)

            # give all services a chance to load the new configuration, rather
            # than the base configuration
            yield self.reconfigServiceWithBuildbotConfig(self.config)

            # Mark the master as active now that mq is running
            yield self.data.updates.masterActive(name=self.name,
                                                 masterid=self.masterid)

            # Start the heartbeat timer
            yield self.masterHeartbeatService.setServiceParent(self)

            # send the statistics to buildbot.net, without waiting
            self.sendBuildbotNetUsageData()
            startup_succeed = True
        except Exception:
            f = failure.Failure()
            log.err(f, 'while starting BuildMaster')
            self.reactor.stop()

        finally:
            if startup_succeed:
                log.msg("BuildMaster is running")
            else:
                log.msg("BuildMaster startup failed")

            yield self.initLock.release()
            self._master_initialized = True

    def sendBuildbotNetUsageData(self):
        if "TRIAL_PYTHONPATH" in os.environ and self.config.buildbotNetUsageData is not None:
            raise RuntimeError(
                "Should not enable buildbotNetUsageData in trial tests!")
        sendBuildbotNetUsageData(self)

    @defer.inlineCallbacks
    def stopService(self):
        try:
            yield self.initLock.acquire()
            if self.masterid is not None:
                yield self.data.updates.masterStopped(name=self.name,
                                                      masterid=self.masterid)
            if self.running:
                yield self.botmaster.cleanShutdown(quickMode=True,
                                                   stopReactor=False)
                yield service.AsyncMultiService.stopService(self)

            log.msg("BuildMaster is stopped")
            self._master_initialized = False
        finally:
            yield self.initLock.release()

    def reconfig(self):
        # this method wraps doConfig, ensuring it is only ever called once at
        # a time, and alerting the user if the reconfig takes too long
        if self.reconfig_active:
            log.msg("reconfig already active; will reconfig again after")
            self.reconfig_requested = True
            return

        self.reconfig_active = self.reactor.seconds()
        metrics.MetricCountEvent.log("loaded_config", 1)

        # notify every 10 seconds that the reconfig is still going on, although
        # reconfigs should not take that long!
        self.reconfig_notifier = task.LoopingCall(
            lambda: log.msg("reconfig is ongoing for %d s" %
                            (self.reactor.seconds() - self.reconfig_active)))
        self.reconfig_notifier.start(10, now=False)

        timer = metrics.Timer("BuildMaster.reconfig")
        timer.start()

        d = self.doReconfig()

        @d.addBoth
        def cleanup(res):
            timer.stop()
            self.reconfig_notifier.stop()
            self.reconfig_notifier = None
            self.reconfig_active = False
            if self.reconfig_requested:
                self.reconfig_requested = False
                self.reconfig()
            return res

        d.addErrback(log.err, 'while reconfiguring')

        return d  # for tests

    @defer.inlineCallbacks
    def doReconfig(self):
        log.msg("beginning configuration update")
        changes_made = False
        failed = False
        try:
            yield self.initLock.acquire()
            # Run the master.cfg in thread, so that it can use blocking code
            new_config = yield threads.deferToThreadPool(
                self.reactor, self.reactor.getThreadPool(),
                self.config_loader.loadConfig)
            changes_made = True
            self.config = new_config

            yield self.reconfigServiceWithBuildbotConfig(new_config)

        except config.ConfigErrors as e:
            for msg in e.errors:
                log.msg(msg)
            failed = True

        except Exception:
            log.err(failure.Failure(), 'during reconfig:')
            failed = True

        finally:
            yield self.initLock.release()

        if failed:
            if changes_made:
                log.msg("WARNING: reconfig partially applied; master "
                        "may malfunction")
            else:
                log.msg("reconfig aborted without making any changes")
        else:
            log.msg("configuration update complete")

    def reconfigServiceWithBuildbotConfig(self, new_config):
        if self.configured_db_url is None:
            self.configured_db_url = new_config.db['db_url']
        elif (self.configured_db_url != new_config.db['db_url']):
            config.error(
                "Cannot change c['db']['db_url'] after the master has started",
            )

        if self.config.mq['type'] != new_config.mq['type']:
            raise config.ConfigErrors([
                "Cannot change c['mq']['type'] after the master has started",
            ])

        return service.ReconfigurableServiceMixin.reconfigServiceWithBuildbotConfig(
            self, new_config)

    # informational methods
    def allSchedulers(self):
        return list(self.scheduler_manager)

    def getStatus(self):
        """
        @rtype: L{buildbot.status.builder.Status}
        """
        return self.status

    # state maintenance (private)
    def getObjectId(self):
        """
        Return the object id for this master, for associating state with the
        master.

        @returns: ID, via Deferred
        """
        # try to get the cached value
        if self._object_id is not None:
            return defer.succeed(self._object_id)

        # failing that, get it from the DB; multiple calls to this function
        # at the same time will not hurt

        d = self.db.state.getObjectId(self.name, "buildbot.master.BuildMaster")

        @d.addCallback
        def keep(id):
            self._object_id = id
            return id

        return d

    def _getState(self, name, default=None):
        "private wrapper around C{self.db.state.getState}"
        d = self.getObjectId()

        @d.addCallback
        def get(objectid):
            return self.db.state.getState(objectid, name, default)

        return d

    def _setState(self, name, value):
        "private wrapper around C{self.db.state.setState}"
        d = self.getObjectId()

        @d.addCallback
        def set(objectid):
            return self.db.state.setState(objectid, name, value)

        return d