Example #1
0
    def testBasic(self):
        s = transfer.StringDownload("Hello World", "hello.txt")
        s.build = Mock()
        s.build.getProperties.return_value = Properties()
        s.build.getSlaveCommandVersion.return_value = 1

        s._step_status = Mock()
        s.buildslave = Mock()
        s.remote = Mock()

        s.start()

        for c in s.remote.method_calls:
            name, command, args = c
            commandName = command[3]
            kwargs = command[-1]
            if commandName == 'downloadFile':
                self.assertEquals(kwargs['slavedest'], 'hello.txt')
                reader = kwargs['reader']
                data = reader.remote_read(100)
                self.assertEquals(data, "Hello World")
                break
        else:
            self.assert_(False, "No downloadFile command found")
Example #2
0
    def trigger(self, ssid, set_props=None):
        """Trigger this scheduler with the given sourcestamp ID. Returns a
        deferred that will fire when the buildset is finished."""
        # properties for this buildset are composed of our own properties,
        # potentially overridden by anything from the triggering build
        props = Properties()
        props.updateFromProperties(self.properties)
        if set_props:
            props.updateFromProperties(set_props)

        # note that this does not use the buildset subscriptions mechanism, as
        # the duration of interest to the caller is bounded by the lifetime of
        # this process.
        d = self.addBuildsetForSourceStamp(reason=self.reason,
                                           ssid=ssid,
                                           properties=props)

        def setup_waiter(bsid):
            self._waiters[bsid] = d = defer.Deferred()
            self._updateWaiters()
            return d

        d.addCallback(setup_waiter)
        return d
Example #3
0
    def __init__(self, name, password, max_builds=None,
                 notify_on_missing=[], missing_timeout=3600,
                 properties={}):
        """
        @param name: botname this machine will supply when it connects
        @param password: password this machine will supply when
                         it connects
        @param max_builds: maximum number of simultaneous builds that will
                           be run concurrently on this buildslave (the
                           default is None for no limit)
        @param properties: properties that will be applied to builds run on
                           this slave
        @type properties: dictionary
        """
        service.MultiService.__init__(self)
        self.slavename = name
        self.password = password
        self.botmaster = None # no buildmaster yet
        self.slave_status = SlaveStatus(name)
        self.slave = None # a RemoteReference to the Bot, when connected
        self.slave_commands = None
        self.slavebuilders = {}
        self.max_builds = max_builds

        self.properties = Properties()
        self.properties.update(properties, "BuildSlave")
        self.properties.setProperty("slavename", name, "BuildSlave")

        self.lastMessageReceived = 0
        if isinstance(notify_on_missing, str):
            notify_on_missing = [notify_on_missing]
        self.notify_on_missing = notify_on_missing
        for i in notify_on_missing:
            assert isinstance(i, str)
        self.missing_timeout = missing_timeout
        self.missing_timer = None
Example #4
0
    def __init__(self, name, builderNames, properties,
                 codebases = DefaultCodebases):
        """
        Initialize a Scheduler.

        @param name: name of this scheduler (used as a key for state)
        @type name: unicode

        @param builderNames: list of builders this scheduler may start
        @type builderNames: list of unicode

        @param properties: properties to add to builds triggered by this
        scheduler
        @type properties: dictionary

        @param codebases: codebases that are necessary to process the changes
        @type codebases: dict with following struct:
            key: '<codebase>'
            value: {'repository':'<repo>', 'branch':'<br>', 'revision:'<rev>'}

        @param consumeChanges: true if this scheduler wishes to be informed
        about the addition of new changes.  Defaults to False.  This should
        be passed explicitly from subclasses to indicate their interest in
        consuming changes.
        @type consumeChanges: boolean
        """
        service.MultiService.__init__(self)
        self.name = name
        "name of this scheduler; used to identify replacements on reconfig"

        ok = True
        if not isinstance(builderNames, (list, tuple)):
            ok = False
        else:
            for b in builderNames:
                if not isinstance(b, basestring):
                    ok = False
        if not ok:
            config.error(
                "The builderNames argument to a scheduler must be a list "
                  "of Builder names.")

        self.builderNames = builderNames
        "list of builder names to start in each buildset"

        self.properties = Properties()
        "properties that are contributed to each buildset"
        self.properties.update(properties, "Scheduler")
        self.properties.setProperty("scheduler", name, "Scheduler")

        self.objectid = None

        self.master = None

        # Set the codebases that are necessary to process the changes
        # These codebases will always result in a sourcestamp with or without changes
        if codebases is not None:
            if not isinstance(codebases, dict):
                config.error("Codebases must be a dict of dicts")
            for codebase, codebase_attrs in codebases.iteritems():
                if not isinstance(codebase_attrs, dict):
                    config.error("Codebases must be a dict of dicts")
                if (codebases != BaseScheduler.DefaultCodebases and
                   'repository' not in codebase_attrs):
                    config.error("The key 'repository' is mandatory in codebases")
        else:
            config.error("Codebases cannot be None")

        self.codebases = codebases
        
        # internal variables
        self._change_subscription = None
        self._change_consumption_lock = defer.DeferredLock()
Example #5
0
    def command_FORCE(self, args):
        # FIXME: NEED TO THINK ABOUT!
        errReply = "try '%s'" % (self.command_FORCE.usage)
        args = self.splitArgs(args)
        if not args:
            raise UsageError(errReply)
        what = args.pop(0)
        if what != "build":
            raise UsageError(errReply)
        opts = ForceOptions()
        opts.parseOptions(args)

        builderName = opts['builder']
        builder = yield self.getBuilder(buildername=builderName)
        branch = opts['branch']
        revision = opts['revision']
        codebase = opts['codebase']
        project = opts['project']
        reason = opts['reason']
        props = opts['props']

        if builderName is None:
            raise UsageError("you must provide a Builder, " + errReply)

        # keep weird stuff out of the branch, revision, and properties args.
        branch_validate = self.master.config.validation['branch']
        revision_validate = self.master.config.validation['revision']
        pname_validate = self.master.config.validation['property_name']
        pval_validate = self.master.config.validation['property_value']
        if branch and not branch_validate.match(branch):
            log.msg("bad branch '%s'" % branch)
            self.send("sorry, bad branch '%s'" % branch)
            return
        if revision and not revision_validate.match(revision):
            log.msg("bad revision '%s'" % revision)
            self.send("sorry, bad revision '%s'" % revision)
            return

        properties = Properties()
        properties.master = self.master

        if props:
            # split props into name:value dict
            pdict = {}
            propertylist = props.split(",")
            for prop in propertylist:
                splitproperty = prop.split("=", 1)
                pdict[splitproperty[0]] = splitproperty[1]

            # set properties
            for prop in pdict:
                pname = prop
                pvalue = pdict[prop]
                if not pname_validate.match(pname) \
                        or not pval_validate.match(pvalue):
                    log.msg("bad property name='%s', value='%s'" %
                            (pname, pvalue))
                    self.send("sorry, bad property name='%s', value='%s'" %
                              (pname, pvalue))
                    return
                properties.setProperty(pname, pvalue, "Force Build chat")

        reason = "forced: by %s: %s" % (self.describeUser(), reason)
        try:
            yield self.master.data.updates.addBuildset(
                builderids=[builder['builderid']],
                # For now, we just use
                # this as the id.
                scheduler="status.words",
                sourcestamps=[{
                    'codebase': codebase,
                    'branch': branch,
                    'revision': revision,
                    'project': project,
                    'repository': "null"
                }],
                reason=reason,
                properties=properties.asDict(),
                waited_for=False)
        except AssertionError as e:
            self.send("I can't: " + str(e))
Example #6
0
    def checkConfig(self,
                    name,
                    password,
                    max_builds=None,
                    notify_on_missing=None,
                    missing_timeout=None,
                    properties=None,
                    defaultProperties=None,
                    locks=None,
                    keepalive_interval=DEFAULT_KEEPALIVE_INTERVAL,
                    machine_name=None):
        """
        @param name: botname this machine will supply when it connects
        @param password: password this machine will supply when
                         it connects
        @param max_builds: maximum number of simultaneous builds that will
                           be run concurrently on this worker (the
                           default is None for no limit)
        @param properties: properties that will be applied to builds run on
                           this worker
        @type properties: dictionary
        @param defaultProperties: properties that will be applied to builds
                                  run on this worker only if the property
                                  has not been set by another source
        @type defaultProperties: dictionary
        @param locks: A list of locks that must be acquired before this worker
                      can be used
        @type locks: dictionary
        @param machine_name: The name of the machine to associate with the
                             worker.
        """
        self.name = name = bytes2unicode(name)
        self.machine_name = machine_name

        self.password = password

        # protocol registration
        self.registration = None

        self._graceful = False
        self._paused = False

        # these are set when the service is started
        self.manager = None
        self.workerid = None

        self.worker_status = WorkerStatus(name)
        self.worker_commands = None
        self.workerforbuilders = {}
        self.max_builds = max_builds
        self.access = []
        if locks:
            self.access = locks
        self.lock_subscriptions = []

        self.properties = Properties()
        self.properties.update(properties or {}, "Worker")
        self.properties.setProperty("workername", name, "Worker")
        self.defaultProperties = Properties()
        self.defaultProperties.update(defaultProperties or {}, "Worker")

        if self.machine_name is not None:
            self.properties.setProperty('machine_name', self.machine_name,
                                        'Worker')
        self.machine = None

        self.lastMessageReceived = 0

        if notify_on_missing is None:
            notify_on_missing = []
        if isinstance(notify_on_missing, str):
            notify_on_missing = [notify_on_missing]
        self.notify_on_missing = notify_on_missing
        for i in notify_on_missing:
            if not isinstance(i, str):
                config.error('notify_on_missing arg %r is not a string' %
                             (i, ))

        self.missing_timeout = missing_timeout
        self.missing_timer = None

        # a protocol connection, if we're currently connected
        self.conn = None

        # during disconnection self.conn will be set to None before all disconnection notifications
        # are delivered. During that period _pending_disconnection_delivery_notifier will be set to
        # a notifier and allows interested users to wait until all disconnection notifications are
        # delivered.
        self._pending_disconnection_delivery_notifier = None

        self._old_builder_list = None
        self._configured_builderid_list = None
Example #7
0
        class JhBuildMaster(BuildMaster):
            jhbuild_config = config
            def loadConfig(self, f):
                # modified from parent method to get slaves, projects, change
                # sources, schedulers, builders and web status ouf of
                # master.cfg [it would have been cleaner if jhbuild didn't
                # have to copy all that code.]
                localDict = {'basedir': os.path.expanduser(self.basedir)}
                try:
                    exec f in localDict
                except:
                    log.msg("error while parsing config file")
                    raise

                jhbuild_config.load()

                try:
                    config = localDict['BuildmasterConfig']
                except KeyError:
                    log.err("missing config dictionary")
                    log.err("config file must define BuildmasterConfig")
                    raise

                known_keys = ("bots", "slaves",
                              "sources", "change_source",
                              "schedulers", "builders", "mergeRequests",
                              "slavePortnum", "debugPassword", "logCompressionLimit",
                              "manhole", "status", "projectName", "projectURL",
                              "buildbotURL", "properties", "prioritizeBuilders",
                              "eventHorizon", "buildCacheSize", "logHorizon", "buildHorizon",
                              "changeHorizon", "logMaxSize", "logMaxTailSize",
                              "logCompressionMethod",
                              )
                for k in config.keys():
                    if k not in known_keys:
                        log.msg("unknown key '%s' defined in config dictionary" % k)

                # the 'slaves' list is read from the 'slaves.csv' file in the
                # current directory (unless instructed different from command line) 
                # it is a CSV file structured like this:
                #   slavename,password
                config['slaves'] = []
                slaves_csv_file = os.path.join(slaves_dir, 'slaves.csv')
                if os.path.exists(slaves_csv_file):
                    for x in csv.reader(file(slaves_csv_file)):
                        if not x or x[0].startswith('#'):
                            continue
                        kw = {}
                        build_slave = JhBuildSlave(x[0], x[1])
                        build_slave.load_extra_configuration(slaves_dir)
                        config['slaves'].append(build_slave)

                if len(config['slaves']) == 0:
                    log.msg('you must fill slaves.csv with slaves')

                module_set = jhbuild.moduleset.load(self.jhbuild_config)
                module_list = module_set.get_module_list(
                        self.jhbuild_config.modules,
                        self.jhbuild_config.skip,
                        include_optional_modules=True)
                config['projects'] = [x.name for x in module_list \
                                      if not x.name.startswith('meta-')]

                if self.jhbuild_config.jhbuildbot_svn_commits_box:
                    # trigger builds from mails to svn-commit-list
                    # (note Maildir must be correct, or everything will fail)
                    from jhbuild.buildbot.changes import GnomeMaildirSource
                    config['change_source'] = GnomeMaildirSource(
                            self.jhbuild_config.jhbuildbot_svn_commits_box,
                            modules=module_list,
                            prefix=None)
                else:
                    # support injection (use 'buildbot sendchange')
                    from buildbot.changes.pb import PBChangeSource
                    config['change_source'] = PBChangeSource()

                # Schedulers
                from jhbuild.buildbot.scheduler import SerialScheduler, NightlySerialScheduler, OnCommitScheduler
                config['schedulers'] = []
                for slave in config['slaves']:
                    s = None
                    for project in config['projects']:
                        buildername = str('%s-%s' % (project, slave.slavename))
                        scheduler_kwargs = {}
                        if slave.scheduler == 'nightly':
                            scheduler_class = NightlySerialScheduler
                            scheduler_kwargs = slave.nightly_kwargs
                        else:
                            scheduler_class = SerialScheduler
                        s = scheduler_class(buildername, project, upstream=s,
                                            builderNames=[buildername],
                                            **scheduler_kwargs)
                        config['schedulers'].append(s)
                        if self.jhbuild_config.jhbuildbot_svn_commits_box:
                            # schedulers that will launch job when receiving
                            # change notifications
                            s2 = OnCommitScheduler('oc-' + buildername,
                                    project, builderNames=[buildername])
                            config['schedulers'].append(s2)

                # Builders
                from jhbuild.buildbot.factory import JHBuildFactory
                config['builders'] = []
                for project in config['projects']:
                    for slave in config['slaves']:
                        f = JHBuildFactory(project, slave)
                        config['builders'].append({
                            'name' : "%s-%s" % (project, slave.slavename),
                            'slavename' : slave.slavename,
                            'builddir' : 'builddir/%s.%s' % (project, slave.slavename),
                            'factory' : f,
                            'category' : project
                        })

                # Status targets
                if not config.has_key('status'):
                    # let it be possible to define additional status in
                    # master.cfg
                    config['status'] = []

                from jhbuild.buildbot.status.web import JHBuildWebStatus
                config['status'].append(
                    JHBuildWebStatus(
                        self.jhbuild_config.moduleset,
                        config['projects'],
                        [x.slavename for x in config['slaves']],
                        http_port=8080, allowForce=True)
                )

                # remaining of the method is a straight copy from buildbot
                # ...
                try:
                    # required
                    schedulers = config['schedulers']
                    builders = config['builders']
                    slavePortnum = config['slavePortnum']
                    #slaves = config['slaves']
                    #change_source = config['change_source']

                    # optional
                    debugPassword = config.get('debugPassword')
                    manhole = config.get('manhole')
                    status = config.get('status', [])
                    projectName = config.get('projectName')
                    projectURL = config.get('projectURL')
                    buildbotURL = config.get('buildbotURL')
                    properties = config.get('properties', {})
                    buildCacheSize = config.get('buildCacheSize', None)
                    eventHorizon = config.get('eventHorizon', None)
                    logHorizon = config.get('logHorizon', None)
                    buildHorizon = config.get('buildHorizon', None)
                    logCompressionLimit = config.get('logCompressionLimit', 4*1024)
                    if logCompressionLimit is not None and not \
                            isinstance(logCompressionLimit, int):
                        raise ValueError("logCompressionLimit needs to be bool or int")
                    logCompressionMethod = config.get('logCompressionMethod', "bz2")
                    if logCompressionMethod not in ('bz2', 'gz'):
                        raise ValueError("logCompressionMethod needs to be 'bz2', or 'gz'")
                    logMaxSize = config.get('logMaxSize')
                    if logMaxSize is not None and not \
                            isinstance(logMaxSize, int):
                        raise ValueError("logMaxSize needs to be None or int")
                    logMaxTailSize = config.get('logMaxTailSize')
                    if logMaxTailSize is not None and not \
                            isinstance(logMaxTailSize, int):
                        raise ValueError("logMaxTailSize needs to be None or int")
                    mergeRequests = config.get('mergeRequests')
                    if mergeRequests is not None and not callable(mergeRequests):
                        raise ValueError("mergeRequests must be a callable")
                    prioritizeBuilders = config.get('prioritizeBuilders')
                    if prioritizeBuilders is not None and not callable(prioritizeBuilders):
                        raise ValueError("prioritizeBuilders must be callable")
                    changeHorizon = config.get("changeHorizon")
                    if changeHorizon is not None and not isinstance(changeHorizon, int):
                        raise ValueError("changeHorizon needs to be an int")

                except KeyError, e:
                    log.msg("config dictionary is missing a required parameter")
                    log.msg("leaving old configuration in place")
                    raise

                #if "bots" in config:
                #    raise KeyError("c['bots'] is no longer accepted")

                slaves = config.get('slaves', [])
                if "bots" in config:
                    m = ("c['bots'] is deprecated as of 0.7.6 and will be "
                         "removed by 0.8.0 . Please use c['slaves'] instead.")
                    log.msg(m)
                    warnings.warn(m, DeprecationWarning)
                    for name, passwd in config['bots']:
                        slaves.append(JhBuildSlave(name, passwd))

                if "bots" not in config and "slaves" not in config:
                    log.msg("config dictionary must have either 'bots' or 'slaves'")
                    log.msg("leaving old configuration in place")
                    raise KeyError("must have either 'bots' or 'slaves'")

                #if "sources" in config:
                #    raise KeyError("c['sources'] is no longer accepted")

                if changeHorizon is not None:
                    self.change_svc.changeHorizon = changeHorizon

                change_source = config.get('change_source', [])
                if isinstance(change_source, (list, tuple)):
                    change_sources = change_source
                else:
                    change_sources = [change_source]
                if "sources" in config:
                    m = ("c['sources'] is deprecated as of 0.7.6 and will be "
                         "removed by 0.8.0 . Please use c['change_source'] instead.")
                    log.msg(m)
                    warnings.warn(m, DeprecationWarning)
                    for s in config['sources']:
                        change_sources.append(s)

                # do some validation first
                for s in slaves:
                    assert interfaces.IBuildSlave.providedBy(s)
                    if s.slavename in ("debug", "change", "status"):
                        raise KeyError(
                            "reserved name '%s' used for a bot" % s.slavename)
                if config.has_key('interlocks'):
                    raise KeyError("c['interlocks'] is no longer accepted")

                assert isinstance(change_sources, (list, tuple))
                for s in change_sources:
                    assert interfaces.IChangeSource(s, None)
                # this assertion catches c['schedulers'] = Scheduler(), since
                # Schedulers are service.MultiServices and thus iterable.
                errmsg = "c['schedulers'] must be a list of Scheduler instances"
                assert isinstance(schedulers, (list, tuple)), errmsg
                for s in schedulers:
                    assert interfaces.IScheduler(s, None), errmsg
                assert isinstance(status, (list, tuple))
                for s in status:
                    assert interfaces.IStatusReceiver(s, None)

                slavenames = [s.slavename for s in slaves]
                buildernames = []
                dirnames = []

                # convert builders from objects to config dictionaries
                builders_dicts = []
                for b in builders:
                    if isinstance(b, buildbot.config.BuilderConfig):
                        builders_dicts.append(b.getConfigDict())
                    elif type(b) is dict:
                        builders_dicts.append(b)
                    else:
                        raise ValueError("builder %s is not a BuilderConfig object (or a dict)" % b)
                builders = builders_dicts

                for b in builders:
                    if b.has_key('slavename') and b['slavename'] not in slavenames:
                        raise ValueError("builder %s uses undefined slave %s" \
                                         % (b['name'], b['slavename']))
                    for n in b.get('slavenames', []):
                        if n not in slavenames:
                            raise ValueError("builder %s uses undefined slave %s" \
                                             % (b['name'], n))
                    if b['name'] in buildernames:
                        raise ValueError("duplicate builder name %s"
                                         % b['name'])
                    buildernames.append(b['name'])

                    # sanity check name (BuilderConfig does this too)
                    if b['name'].startswith("_"):
                        errmsg = ("builder names must not start with an "
                                  "underscore: " + b['name'])
                        log.err(errmsg)
                        raise ValueError(errmsg)

                    # Fix the dictionnary with default values, in case this wasn't
                    # specified with a BuilderConfig object (which sets the same defaults)
                    b.setdefault('builddir', buildbot.util.safeTranslate(b['name']))
                    b.setdefault('slavebuilddir', b['builddir'])

                    if b['builddir'] in dirnames:
                        raise ValueError("builder %s reuses builddir %s"
                                         % (b['name'], b['builddir']))
                    dirnames.append(b['builddir'])

                unscheduled_buildernames = buildernames[:]
                schedulernames = []
                for s in schedulers:
                    for b in s.listBuilderNames():
                        assert b in buildernames, \
                               "%s uses unknown builder %s" % (s, b)
                        if b in unscheduled_buildernames:
                            unscheduled_buildernames.remove(b)

                    if s.name in schedulernames:
                        # TODO: schedulers share a namespace with other Service
                        # children of the BuildMaster node, like status plugins, the
                        # Manhole, the ChangeMaster, and the BotMaster (although most
                        # of these don't have names)
                        msg = ("Schedulers must have unique names, but "
                               "'%s' was a duplicate" % (s.name,))
                        raise ValueError(msg)
                    schedulernames.append(s.name)

                if unscheduled_buildernames:
                    log.msg("Warning: some Builders have no Schedulers to drive them:"
                            " %s" % (unscheduled_buildernames,))

                # assert that all locks used by the Builds and their Steps are
                # uniquely named.
                lock_dict = {}
                for b in builders:
                    for l in b.get('locks', []):
                        if isinstance(l, locks.LockAccess): # User specified access to the lock
                            l = l.lockid
                        if lock_dict.has_key(l.name):
                            if lock_dict[l.name] is not l:
                                raise ValueError("Two different locks (%s and %s) "
                                                 "share the name %s"
                                                 % (l, lock_dict[l.name], l.name))
                        else:
                            lock_dict[l.name] = l
                    # TODO: this will break with any BuildFactory that doesn't use a
                    # .steps list, but I think the verification step is more
                    # important.
                    for s in b['factory'].steps:
                        for l in s[1].get('locks', []):
                            if isinstance(l, locks.LockAccess): # User specified access to the lock
                                l = l.lockid
                            if lock_dict.has_key(l.name):
                                if lock_dict[l.name] is not l:
                                    raise ValueError("Two different locks (%s and %s)"
                                                     " share the name %s"
                                                     % (l, lock_dict[l.name], l.name))
                            else:
                                lock_dict[l.name] = l

                if not isinstance(properties, dict):
                    raise ValueError("c['properties'] must be a dictionary")

                # slavePortnum supposed to be a strports specification
                if type(slavePortnum) is int:
                    slavePortnum = "tcp:%d" % slavePortnum

                # now we're committed to implementing the new configuration, so do
                # it atomically
                # TODO: actually, this is spread across a couple of Deferreds, so it
                # really isn't atomic.

                d = defer.succeed(None)

                self.projectName = projectName
                self.projectURL = projectURL
                self.buildbotURL = buildbotURL

                self.properties = Properties()
                self.properties.update(properties, self.configFileName)

                self.status.logCompressionLimit = logCompressionLimit
                self.status.logCompressionMethod = logCompressionMethod
                self.status.logMaxSize = logMaxSize
                self.status.logMaxTailSize = logMaxTailSize
                # Update any of our existing builders with the current log parameters.
                # This is required so that the new value is picked up after a
                # reconfig.
                for builder in self.botmaster.builders.values():
                    builder.builder_status.setLogCompressionLimit(logCompressionLimit)
                    builder.builder_status.setLogCompressionMethod(logCompressionMethod)
                    builder.builder_status.setLogMaxSize(logMaxSize)
                    builder.builder_status.setLogMaxTailSize(logMaxTailSize)

                if mergeRequests is not None:
                    self.botmaster.mergeRequests = mergeRequests
                if prioritizeBuilders is not None:
                    self.botmaster.prioritizeBuilders = prioritizeBuilders

                self.buildCacheSize = buildCacheSize
                self.eventHorizon = eventHorizon
                self.logHorizon = logHorizon
                self.buildHorizon = buildHorizon

                # self.slaves: Disconnect any that were attached and removed from the
                # list. Update self.checker with the new list of passwords, including
                # debug/change/status.
                d.addCallback(lambda res: self.loadConfig_Slaves(slaves))

                # self.debugPassword
                if debugPassword:
                    self.checker.addUser("debug", debugPassword)
                    self.debugPassword = debugPassword

                # self.manhole
                if manhole != self.manhole:
                    # changing
                    if self.manhole:
                        # disownServiceParent may return a Deferred
                        d.addCallback(lambda res: self.manhole.disownServiceParent())
                        def _remove(res):
                            self.manhole = None
                            return res
                        d.addCallback(_remove)
                    if manhole:
                        def _add(res):
                            self.manhole = manhole
                            manhole.setServiceParent(self)
                        d.addCallback(_add)

                # add/remove self.botmaster.builders to match builders. The
                # botmaster will handle startup/shutdown issues.
                d.addCallback(lambda res: self.loadConfig_Builders(builders))

                d.addCallback(lambda res: self.loadConfig_status(status))

                # Schedulers are added after Builders in case they start right away
                d.addCallback(lambda res: self.loadConfig_Schedulers(schedulers))
                # and Sources go after Schedulers for the same reason
                d.addCallback(lambda res: self.loadConfig_Sources(change_sources))

                # self.slavePort
                if self.slavePortnum != slavePortnum:
                    if self.slavePort:
                        def closeSlavePort(res):
                            d1 = self.slavePort.disownServiceParent()
                            self.slavePort = None
                            return d1
                        d.addCallback(closeSlavePort)
                    if slavePortnum is not None:
                        def openSlavePort(res):
                            self.slavePort = strports.service(slavePortnum,
                                                              self.slaveFactory)
                            self.slavePort.setServiceParent(self)
                        d.addCallback(openSlavePort)
                        log.msg("BuildMaster listening on port %s" % slavePortnum)
                    self.slavePortnum = slavePortnum

                log.msg("configuration update started")
                def _done(res):
                    self.readConfig = True
                    log.msg("configuration update complete")
                d.addCallback(_done)
                d.addCallback(lambda res: self.botmaster.maybeStartAllBuilds())
                return d
Example #8
0
    def __init__(self,
                 name,
                 password,
                 max_builds=None,
                 notify_on_missing=[],
                 missing_timeout=3600,
                 properties={},
                 locks=None,
                 keepalive_interval=3600):
        """
        @param name: botname this machine will supply when it connects
        @param password: password this machine will supply when
                         it connects
        @param max_builds: maximum number of simultaneous builds that will
                           be run concurrently on this buildslave (the
                           default is None for no limit)
        @param properties: properties that will be applied to builds run on
                           this slave
        @type properties: dictionary
        @param locks: A list of locks that must be acquired before this slave
                      can be used
        @type locks: dictionary
        """
        service.MultiService.__init__(self)
        self.slavename = name
        self.password = password

        # PB registration
        self.registration = None
        self.registered_port = None

        # these are set when the service is started, and unset when it is
        # stopped
        self.botmaster = None
        self.master = None

        self.slave_status = SlaveStatus(name)
        self.slave = None  # a RemoteReference to the Bot, when connected
        self.slave_commands = None
        self.slavebuilders = {}
        self.max_builds = max_builds
        self.access = []
        if locks:
            self.access = locks
        self.lock_subscriptions = []

        self.properties = Properties()
        self.properties.update(properties, "BuildSlave")
        self.properties.setProperty("slavename", name, "BuildSlave")

        self.lastMessageReceived = 0
        if isinstance(notify_on_missing, str):
            notify_on_missing = [notify_on_missing]
        self.notify_on_missing = notify_on_missing
        for i in notify_on_missing:
            if not isinstance(i, str):
                config.error('notify_on_missing arg %r is not a string' %
                             (i, ))
        self.missing_timeout = missing_timeout
        self.missing_timer = None
        self.keepalive_interval = keepalive_interval

        self.detached_subs = None

        self._old_builder_list = None
Example #9
0
 def __init__(self, reason, sources, buildername):
     self.reason = reason
     self.sources = sources
     self.buildername = buildername
     self.changes = []
     self.properties = Properties()
Example #10
0
 def __init__(self):
     self.sources = []
     self.reason = "Because"
     self.properties = Properties()
Example #11
0
class FakeChange:
    properties = Properties()

    def __init__(self, number=None):
        self.number = number
        self.who = "me"
Example #12
0
 def __init__(self, number=None):
     self.properties = Properties()
     self.number = number
     self.who = "me"
    def forceWithWebRequest(self, owner, builder_name, req):
        """Called by the web UI.
        Authentication is already done, thus owner is passed as argument
        We check the parameters, and launch the build, if everything is correct
        """
        if not builder_name in self.builderNames:
            # in the case of buildAll, this method will be called several times
            # for all the builders
            # we just do nothing on a builder that is not in our builderNames
            return defer.succeed(None)
        master = self.master
        properties = {}
        changeids = []
        # probably need to clean that out later as the IProperty is already a
        # validation mechanism

        validation = master.config.validation
        if self.branch.regex == None:
            self.branch.regex = validation['branch']
        if self.revision.regex == None:
            self.revision.regex = validation['revision']

        for param in self.all_fields:
            if owner and param == self.username:
                continue  # dont enforce username if auth
            param.update_from_post(master, properties, changeids, req)

        changeids = map(lambda a: type(a) == int and a or a.number, changeids)
        # everything is validated, we can create our source stamp, and buildrequest
        reason = properties[self.reason.name]
        branch = properties[self.branch.name]
        revision = properties[self.revision.name]
        repository = properties[self.repository.name]
        project = properties[self.project.name]
        if owner is None:
            owner = properties[self.username.name]

        std_prop_names = [
            self.branch.name, self.revision.name, self.repository.name,
            self.project.name, self.username.name
        ]
        real_properties = Properties()
        for pname, pvalue in properties.items():
            if not pname in std_prop_names:
                real_properties.setProperty(pname, pvalue, "Force Build Form")

        real_properties.setProperty("owner", owner, "Force Build Form")

        r = ("The web-page 'force build' button was pressed by '%s': %s\n" %
             (owner, reason))

        d = master.db.sourcestampsets.addSourceStampSet()

        def add_master_with_setid(sourcestampsetid):
            master.db.sourcestamps.addSourceStamp(
                sourcestampsetid=sourcestampsetid,
                branch=branch,
                revision=revision,
                project=project,
                repository=repository,
                changeids=changeids)
            return sourcestampsetid

        d.addCallback(add_master_with_setid)

        def got_setid(sourcestampsetid):
            return self.addBuildsetForSourceStamp(builderNames=[builder_name],
                                                  setid=sourcestampsetid,
                                                  reason=r,
                                                  properties=real_properties)

        d.addCallback(got_setid)
        return d
Example #14
0
    def command_FORCE(self, args, **kwargs):
        """force a build"""

        # FIXME: NEED TO THINK ABOUT!
        errReply = f"Try '{self.bot.commandPrefix}{self.command_FORCE.usage}'"
        args = self.splitArgs(args)
        if not args:
            raise UsageError(errReply)
        what = args.pop(0)
        if what != "build":
            raise UsageError(errReply)
        opts = ForceOptions()
        opts.parseOptions(args)

        builderName = opts['builder']
        builder = yield self.bot.getBuilder(buildername=builderName)
        branch = opts['branch']
        revision = opts['revision']
        codebase = opts['codebase']
        project = opts['project']
        reason = opts['reason']
        props = opts['props']

        if builderName is None:
            raise UsageError("you must provide a Builder, " + errReply)

        # keep weird stuff out of the branch, revision, and properties args.
        branch_validate = self.master.config.validation['branch']
        revision_validate = self.master.config.validation['revision']
        pname_validate = self.master.config.validation['property_name']
        pval_validate = self.master.config.validation['property_value']
        if branch and not branch_validate.match(branch):
            self.bot.log(f"Force: bad branch '{branch}'")
            self.send(f"Sorry, bad branch '{branch}'")
            return
        if revision and not revision_validate.match(revision):
            self.bot.log(f"Force: bad revision '{revision}'")
            self.send(f"Sorry, bad revision '{revision}'")
            return

        properties = Properties()
        properties.master = self.master

        if props:
            # split props into name:value dict
            pdict = {}
            propertylist = props.split(",")
            for prop in propertylist:
                splitproperty = prop.split("=", 1)
                pdict[splitproperty[0]] = splitproperty[1]

            # set properties
            for pname, pvalue in pdict.items():
                if not pname_validate.match(pname) \
                        or not pval_validate.match(pvalue):
                    self.bot.log(
                        f"Force: bad property name='{pname}', value='{pvalue}'"
                    )
                    self.send(
                        f"Sorry, bad property name='{pname}', value='{pvalue}'"
                    )
                    return
                properties.setProperty(pname, pvalue, "Force Build Chat")

        properties.setProperty("reason", reason, "Force Build Chat")
        properties.setProperty("owner", self.describeUser(),
                               "Force Build Chat")

        reason = f"forced: by {self.describeUser()}: {reason}"
        try:
            yield self.master.data.updates.addBuildset(
                builderids=[builder['builderid']],
                # For now, we just use
                # this as the id.
                scheduler="status.words",
                sourcestamps=[{
                    'codebase': codebase,
                    'branch': branch,
                    'revision': revision,
                    'project': project,
                    'repository': ""
                }],
                reason=reason,
                properties=properties.asDict(),
                waited_for=False)
        except AssertionError as e:
            self.send("I can't: " + str(e))
        else:
            self.send("Force build successfully requested.")
Example #15
0
    def test_rejects_build_on_instance_with_different_type_timeout_nonzero(
            self):
        """
        If latent worker supports getting its instance type from properties that
        are rendered from build then the buildrequestdistributor must not
        schedule any builds on workers that are running different instance type
        than what these builds will require.
        """
        controller = LatentController(self,
                                      'local',
                                      kind=Interpolate('%(prop:worker_kind)s'),
                                      build_wait_timeout=5)

        # a step that we can finish when we want
        stepcontroller = BuildStepController()
        config_dict = {
            'builders': [
                BuilderConfig(
                    name="testy",
                    workernames=["local"],
                    factory=BuildFactory([stepcontroller.step]),
                ),
            ],
            'workers': [controller.worker],
            'protocols': {
                'null': {}
            },
            'multiMaster':
            True,
        }

        master = self.getMaster(config_dict)
        builder_id = self.successResultOf(
            master.data.updates.findBuilderId('testy'))

        # create build request
        self.createBuildrequest(master, [builder_id],
                                properties=Properties(worker_kind='a'))

        # start the build and verify the kind of the worker. Note that the
        # buildmaster needs to restart the worker in order to change the worker
        # kind, so we allow it both to auto start and stop
        self.assertEqual(True, controller.starting)

        controller.auto_connect_worker = True
        controller.auto_disconnect_worker = True
        controller.auto_start(True)
        controller.auto_stop(True)
        controller.connect_worker()
        self.assertEqual('a', (yield controller.get_started_kind()))

        # before the other build finished, create another build request
        self.createBuildrequest(master, [builder_id],
                                properties=Properties(worker_kind='b'))
        stepcontroller.finish_step(SUCCESS)

        # give the botmaster chance to insubstantiate the worker and
        # maybe substantiate it for the pending build the builds on worker
        self.reactor.advance(0.1)

        # verify build has not started, even though the worker is waiting
        # for one
        self.assertIsNone((yield master.db.builds.getBuild(2)))
        self.assertTrue(controller.started)

        # wait until the latent worker times out, is insubstantiated,
        # is substantiated because of pending buildrequest and starts the build
        self.reactor.advance(6)
        self.assertIsNotNone((yield master.db.builds.getBuild(2)))

        # verify that the second build restarted with the expected instance
        # kind
        self.assertEqual('b', (yield controller.get_started_kind()))
        stepcontroller.finish_step(SUCCESS)

        dbdict = yield master.db.builds.getBuild(1)
        self.assertEqual(SUCCESS, dbdict['results'])
        dbdict = yield master.db.builds.getBuild(2)
        self.assertEqual(SUCCESS, dbdict['results'])
 def setUp(self):
     self.props = Properties()
     self.build = FakeBuild(self.props)
Example #17
0
 def getProperties(self):
     return Properties()
Example #18
0
 def upgradeToVersion3(self):
     # in version 3, self.properties became a Properties object
     propdict = self.properties
     self.properties = Properties()
     self.properties.update(propdict, "Upgrade from previous version")
     self.wasUpgraded = True
 def setUp(self):
     self.build = Properties(image="busybox:latest")
     self.patch(dockerbuildslave, 'client', docker)
Example #20
0
    def command_FORCE(self, args, who):
        errReply = "try 'force build [--branch=BRANCH] [--revision=REVISION] [--props=PROP1=VAL1,PROP2=VAL2...]  <WHICH> <REASON>'"
        args = self.splitArgs(args)
        if not args:
            raise UsageError(errReply)
        what = args.pop(0)
        if what != "build":
            raise UsageError(errReply)
        opts = ForceOptions()
        opts.parseOptions(args)

        which = opts['builder']
        branch = opts['branch']
        revision = opts['revision']
        reason = opts['reason']
        props = opts['props']

        if which is None:
            raise UsageError("you must provide a Builder, " + errReply)

        # keep weird stuff out of the branch, revision, and properties args.
        branch_validate = self.master.config.validation['branch']
        revision_validate = self.master.config.validation['revision']
        pname_validate = self.master.config.validation['property_name']
        pval_validate = self.master.config.validation['property_value']
        if branch and not branch_validate.match(branch):
            log.msg("bad branch '%s'" % branch)
            self.send("sorry, bad branch '%s'" % branch)
            return
        if revision and not revision_validate.match(revision):
            log.msg("bad revision '%s'" % revision)
            self.send("sorry, bad revision '%s'" % revision)
            return

        properties = Properties()
        if props:
            # split props into name:value dict
            pdict = {}
            propertylist = props.split(",")
            for i in range(0, len(propertylist)):
                splitproperty = propertylist[i].split("=", 1)
                pdict[splitproperty[0]] = splitproperty[1]

            # set properties
            for prop in pdict:
                pname = prop
                pvalue = pdict[prop]
                if not pname_validate.match(pname) \
                        or not pval_validate.match(pvalue):
                    log.msg("bad property name='%s', value='%s'" %
                            (pname, pvalue))
                    self.send("sorry, bad property name='%s', value='%s'" %
                              (pname, pvalue))
                    return
                properties.setProperty(pname, pvalue, "Force Build IRC")

        bc = self.getControl(which)

        reason = "forced: by %s: %s" % (self.describeUser(who), reason)
        ss = SourceStamp(branch=branch, revision=revision)
        d = bc.submitBuildRequest(ss, reason, props=properties.asDict())

        def subscribe(buildreq):
            ireq = IrcBuildRequest(self, self.useRevisions)
            buildreq.subscribe(ireq.started)

        d.addCallback(subscribe)
        d.addErrback(log.err, "while forcing a build")
Example #21
0
 def createTriggerProperties(self, properties):
     # make a new properties object from a dict rendered by the old
     # properties object
     trigger_properties = Properties()
     trigger_properties.update(properties, "Trigger")
     return trigger_properties
 def setUp(self):
     self.patch(openstack, "client", novaclient)
     self.patch(openstack, "loading", novaclient)
     self.patch(openstack, "session", novaclient)
     self.build = Properties(image=novaclient.TEST_UUIDS['image'])
Example #23
0
    def reconfigService(self,
                        name,
                        password,
                        max_builds=None,
                        notify_on_missing=None,
                        missing_timeout=DEFAULT_MISSING_TIMEOUT,
                        properties=None,
                        defaultProperties=None,
                        locks=None,
                        keepalive_interval=DEFAULT_KEEPALIVE_INTERVAL,
                        machine_name=None):
        # Given a Worker config arguments, configure this one identically.
        # Because Worker objects are remotely referenced, we can't replace them
        # without disconnecting the worker, yet there's no reason to do that.

        assert self.name == name
        self.password = password

        # adopt new instance's configuration parameters
        self.max_builds = max_builds
        self.access = []
        if locks:
            self.access = locks
        if notify_on_missing is None:
            notify_on_missing = []
        if isinstance(notify_on_missing, str):
            notify_on_missing = [notify_on_missing]
        self.notify_on_missing = notify_on_missing

        if self.missing_timeout != missing_timeout:
            running_missing_timer = self.missing_timer
            self.stopMissingTimer()
            self.missing_timeout = missing_timeout
            if running_missing_timer:
                self.startMissingTimer()

        self.properties = Properties()
        self.properties.update(properties or {}, "Worker")
        self.properties.setProperty("workername", name, "Worker")
        self.defaultProperties = Properties()
        self.defaultProperties.update(defaultProperties or {}, "Worker")

        # Note that before first reconfig self.machine will always be None and
        # out of sync with self.machine_name, thus more complex logic is needed.
        if self.machine is not None and self.machine_name != machine_name:
            self.machine.unregisterWorker(self)
            self.machine = None

        self.machine_name = machine_name
        if self.machine is None and self.machine_name is not None:
            self.machine = self.master.machine_manager.getMachineByName(
                self.machine_name)
            if self.machine is not None:
                self.machine.registerWorker(self)
                self.properties.setProperty("machine_name", self.machine_name,
                                            "Worker")
            else:
                log.err("Unknown machine '{}' for worker '{}'".format(
                    self.machine_name, self.name))

        # update our records with the worker manager
        if not self.registration:
            self.registration = yield self.master.workers.register(self)
        yield self.registration.update(self, self.master.config)

        # tracks config version for locks
        self.config_version = self.master.config_version
        self.updateLocks()
Example #24
0
    def command_FORCE(self, args):
        # FIXME: NEED TO THINK ABOUT!
        errReply = "try 'force build [--branch=BRANCH] [--revision=REVISION] [--props=PROP1=VAL1,PROP2=VAL2...]  <WHICH> <REASON>'"
        args = self.splitArgs(args)
        if not args:
            raise UsageError(errReply)
        what = args.pop(0)
        if what != "build":
            raise UsageError(errReply)
        opts = ForceOptions()
        opts.parseOptions(args)

        builderName = opts['builder']
        branch = opts['branch']
        revision = opts['revision']
        reason = opts['reason']
        props = opts['props']

        if builderName is None:
            raise UsageError("you must provide a Builder, " + errReply)

        # keep weird stuff out of the branch, revision, and properties args.
        branch_validate = self.master.config.validation['branch']
        revision_validate = self.master.config.validation['revision']
        pname_validate = self.master.config.validation['property_name']
        pval_validate = self.master.config.validation['property_value']
        if branch and not branch_validate.match(branch):
            log.msg("bad branch '%s'" % branch)
            self.send("sorry, bad branch '%s'" % branch)
            return
        if revision and not revision_validate.match(revision):
            log.msg("bad revision '%s'" % revision)
            self.send("sorry, bad revision '%s'" % revision)
            return

        properties = Properties()
        if props:
            # split props into name:value dict
            pdict = {}
            propertylist = props.split(",")
            for i in range(0, len(propertylist)):
                splitproperty = propertylist[i].split("=", 1)
                pdict[splitproperty[0]] = splitproperty[1]

            # set properties
            for prop in pdict:
                pname = prop
                pvalue = pdict[prop]
                if not pname_validate.match(pname) \
                        or not pval_validate.match(pvalue):
                    log.msg("bad property name='%s', value='%s'" %
                            (pname, pvalue))
                    self.send("sorry, bad property name='%s', value='%s'" %
                              (pname, pvalue))
                    return
                properties.setProperty(pname, pvalue, "Force Build chat")

        reason = u"forced: by %s: %s" % (self.describeUser(), reason)
        d = self.master.data.updates.addBuildset(
            builderNames=[builderName],
            scheduler=u"status.words",  # For now, we just use this as the id.
            sourcestamps=[{
                'branch': branch,
                'revision': revision
            }],
            reason=reason,
            properties=properties.asDict(),
            waited_for=False)

        @d.addCallback
        def subscribe(xxx_todo_changeme):
            (bsid, brids) = xxx_todo_changeme
            assert 0, "rewrite to not use the status hierarchy"  # TODO
            # ireq = BuildRequest(self, self.useRevisions)
            # buildreq.subscribe(ireq.started)

        d.addErrback(log.err, "while forcing a build")
 class FakeBuild:
     properties = Properties(foo='foo', bar='bar', baz='baz')
Example #26
0
def buildUIDSchedFunc(sched, t, ssid):
    """Return a Properties instance with 'builduid' set to a randomly generated
    id."""
    props = Properties()
    props.setProperty('builduid', genBuildUID(), 'buildUIDSchedFunc')
    return props
Example #27
0
    def checkConfig(self, name, password, max_builds=None,
                    notify_on_missing=None,
                    missing_timeout=10 * 60,   # Ten minutes
                    properties=None, locks=None, keepalive_interval=3600):
        """
        @param name: botname this machine will supply when it connects
        @param password: password this machine will supply when
                         it connects
        @param max_builds: maximum number of simultaneous builds that will
                           be run concurrently on this worker (the
                           default is None for no limit)
        @param properties: properties that will be applied to builds run on
                           this worker
        @type properties: dictionary
        @param locks: A list of locks that must be acquired before this worker
                      can be used
        @type locks: dictionary
        """
        self.name = name = ascii2unicode(name)

        if properties is None:
            properties = {}

        self.password = password

        # protocol registration
        self.registration = None

        # these are set when the service is started
        self.manager = None
        self.workerid = None

        self.worker_status = WorkerStatus(name)
        self.worker_commands = None
        self.workerforbuilders = {}
        self.max_builds = max_builds
        self.access = []
        if locks:
            self.access = locks
        self.lock_subscriptions = []

        self.properties = Properties()
        self.properties.update(properties, "Worker")
        self.properties.setProperty("slavename", name, "Worker (deprecated)")
        self.properties.setProperty("workername", name, "Worker")

        self.lastMessageReceived = 0

        if notify_on_missing is None:
            notify_on_missing = []
        if isinstance(notify_on_missing, str):
            notify_on_missing = [notify_on_missing]
        self.notify_on_missing = notify_on_missing
        for i in notify_on_missing:
            if not isinstance(i, str):
                config.error(
                    'notify_on_missing arg %r is not a string' % (i,))

        self.missing_timeout = missing_timeout
        self.missing_timer = None

        # a protocol connection, if we're currently connected
        self.conn = None

        self._old_builder_list = None
 def setUp(self):
     self.setUpTestReactor()
     self.build = Properties(image="busybox:latest",
                             builder="docker_worker")
     self.worker = None
Example #29
0
 def oneTest(self, props, expected):
     p = Properties()
     p.update(props, "test")
     r = repo.RepoDownloadsFromProperties(list(props))
     self.assertEqual(r.getRenderingFor(p), expected)
 def setUp(self):
     self.props = Properties()