def testBasic(self):
s = transfer.StringDownload("Hello World", "hello.txt")
s.build = Mock()
s.build.getProperties.return_value = Properties()
s.build.getSlaveCommandVersion.return_value = 1
s._step_status = Mock()
s.buildslave = Mock()
s.remote = Mock()
s.start()
for c in s.remote.method_calls:
name, command, args = c
commandName = command[3]
kwargs = command[-1]
if commandName == 'downloadFile':
self.assertEquals(kwargs['slavedest'], 'hello.txt')
reader = kwargs['reader']
data = reader.remote_read(100)
self.assertEquals(data, "Hello World")
break
else:
self.assert_(False, "No downloadFile command found")
def trigger(self, ssid, set_props=None):
"""Trigger this scheduler with the given sourcestamp ID. Returns a
deferred that will fire when the buildset is finished."""
# properties for this buildset are composed of our own properties,
# potentially overridden by anything from the triggering build
props = Properties()
props.updateFromProperties(self.properties)
if set_props:
props.updateFromProperties(set_props)
# note that this does not use the buildset subscriptions mechanism, as
# the duration of interest to the caller is bounded by the lifetime of
# this process.
d = self.addBuildsetForSourceStamp(reason=self.reason,
ssid=ssid,
properties=props)
def setup_waiter(bsid):
self._waiters[bsid] = d = defer.Deferred()
self._updateWaiters()
return d
d.addCallback(setup_waiter)
return d
def __init__(self, name, password, max_builds=None,
notify_on_missing=[], missing_timeout=3600,
properties={}):
"""
@param name: botname this machine will supply when it connects
@param password: password this machine will supply when
it connects
@param max_builds: maximum number of simultaneous builds that will
be run concurrently on this buildslave (the
default is None for no limit)
@param properties: properties that will be applied to builds run on
this slave
@type properties: dictionary
"""
service.MultiService.__init__(self)
self.slavename = name
self.password = password
self.botmaster = None # no buildmaster yet
self.slave_status = SlaveStatus(name)
self.slave = None # a RemoteReference to the Bot, when connected
self.slave_commands = None
self.slavebuilders = {}
self.max_builds = max_builds
self.properties = Properties()
self.properties.update(properties, "BuildSlave")
self.properties.setProperty("slavename", name, "BuildSlave")
self.lastMessageReceived = 0
if isinstance(notify_on_missing, str):
notify_on_missing = [notify_on_missing]
self.notify_on_missing = notify_on_missing
for i in notify_on_missing:
assert isinstance(i, str)
self.missing_timeout = missing_timeout
self.missing_timer = None
def __init__(self, name, builderNames, properties,
codebases = DefaultCodebases):
"""
Initialize a Scheduler.
@param name: name of this scheduler (used as a key for state)
@type name: unicode
@param builderNames: list of builders this scheduler may start
@type builderNames: list of unicode
@param properties: properties to add to builds triggered by this
scheduler
@type properties: dictionary
@param codebases: codebases that are necessary to process the changes
@type codebases: dict with following struct:
key: '<codebase>'
value: {'repository':'<repo>', 'branch':'<br>', 'revision:'<rev>'}
@param consumeChanges: true if this scheduler wishes to be informed
about the addition of new changes. Defaults to False. This should
be passed explicitly from subclasses to indicate their interest in
consuming changes.
@type consumeChanges: boolean
"""
service.MultiService.__init__(self)
self.name = name
"name of this scheduler; used to identify replacements on reconfig"
ok = True
if not isinstance(builderNames, (list, tuple)):
ok = False
else:
for b in builderNames:
if not isinstance(b, basestring):
ok = False
if not ok:
config.error(
"The builderNames argument to a scheduler must be a list "
"of Builder names.")
self.builderNames = builderNames
"list of builder names to start in each buildset"
self.properties = Properties()
"properties that are contributed to each buildset"
self.properties.update(properties, "Scheduler")
self.properties.setProperty("scheduler", name, "Scheduler")
self.objectid = None
self.master = None
# Set the codebases that are necessary to process the changes
# These codebases will always result in a sourcestamp with or without changes
if codebases is not None:
if not isinstance(codebases, dict):
config.error("Codebases must be a dict of dicts")
for codebase, codebase_attrs in codebases.iteritems():
if not isinstance(codebase_attrs, dict):
config.error("Codebases must be a dict of dicts")
if (codebases != BaseScheduler.DefaultCodebases and
'repository' not in codebase_attrs):
config.error("The key 'repository' is mandatory in codebases")
else:
config.error("Codebases cannot be None")
self.codebases = codebases
# internal variables
self._change_subscription = None
self._change_consumption_lock = defer.DeferredLock()
def command_FORCE(self, args):
# FIXME: NEED TO THINK ABOUT!
errReply = "try '%s'" % (self.command_FORCE.usage)
args = self.splitArgs(args)
if not args:
raise UsageError(errReply)
what = args.pop(0)
if what != "build":
raise UsageError(errReply)
opts = ForceOptions()
opts.parseOptions(args)
builderName = opts['builder']
builder = yield self.getBuilder(buildername=builderName)
branch = opts['branch']
revision = opts['revision']
codebase = opts['codebase']
project = opts['project']
reason = opts['reason']
props = opts['props']
if builderName is None:
raise UsageError("you must provide a Builder, " + errReply)
# keep weird stuff out of the branch, revision, and properties args.
branch_validate = self.master.config.validation['branch']
revision_validate = self.master.config.validation['revision']
pname_validate = self.master.config.validation['property_name']
pval_validate = self.master.config.validation['property_value']
if branch and not branch_validate.match(branch):
log.msg("bad branch '%s'" % branch)
self.send("sorry, bad branch '%s'" % branch)
return
if revision and not revision_validate.match(revision):
log.msg("bad revision '%s'" % revision)
self.send("sorry, bad revision '%s'" % revision)
return
properties = Properties()
properties.master = self.master
if props:
# split props into name:value dict
pdict = {}
propertylist = props.split(",")
for prop in propertylist:
splitproperty = prop.split("=", 1)
pdict[splitproperty[0]] = splitproperty[1]
# set properties
for prop in pdict:
pname = prop
pvalue = pdict[prop]
if not pname_validate.match(pname) \
or not pval_validate.match(pvalue):
log.msg("bad property name='%s', value='%s'" %
(pname, pvalue))
self.send("sorry, bad property name='%s', value='%s'" %
(pname, pvalue))
return
properties.setProperty(pname, pvalue, "Force Build chat")
reason = "forced: by %s: %s" % (self.describeUser(), reason)
try:
yield self.master.data.updates.addBuildset(
builderids=[builder['builderid']],
# For now, we just use
# this as the id.
scheduler="status.words",
sourcestamps=[{
'codebase': codebase,
'branch': branch,
'revision': revision,
'project': project,
'repository': "null"
}],
reason=reason,
properties=properties.asDict(),
waited_for=False)
except AssertionError as e:
self.send("I can't: " + str(e))
def checkConfig(self,
name,
password,
max_builds=None,
notify_on_missing=None,
missing_timeout=None,
properties=None,
defaultProperties=None,
locks=None,
keepalive_interval=DEFAULT_KEEPALIVE_INTERVAL,
machine_name=None):
"""
@param name: botname this machine will supply when it connects
@param password: password this machine will supply when
it connects
@param max_builds: maximum number of simultaneous builds that will
be run concurrently on this worker (the
default is None for no limit)
@param properties: properties that will be applied to builds run on
this worker
@type properties: dictionary
@param defaultProperties: properties that will be applied to builds
run on this worker only if the property
has not been set by another source
@type defaultProperties: dictionary
@param locks: A list of locks that must be acquired before this worker
can be used
@type locks: dictionary
@param machine_name: The name of the machine to associate with the
worker.
"""
self.name = name = bytes2unicode(name)
self.machine_name = machine_name
self.password = password
# protocol registration
self.registration = None
self._graceful = False
self._paused = False
# these are set when the service is started
self.manager = None
self.workerid = None
self.worker_status = WorkerStatus(name)
self.worker_commands = None
self.workerforbuilders = {}
self.max_builds = max_builds
self.access = []
if locks:
self.access = locks
self.lock_subscriptions = []
self.properties = Properties()
self.properties.update(properties or {}, "Worker")
self.properties.setProperty("workername", name, "Worker")
self.defaultProperties = Properties()
self.defaultProperties.update(defaultProperties or {}, "Worker")
if self.machine_name is not None:
self.properties.setProperty('machine_name', self.machine_name,
'Worker')
self.machine = None
self.lastMessageReceived = 0
if notify_on_missing is None:
notify_on_missing = []
if isinstance(notify_on_missing, str):
notify_on_missing = [notify_on_missing]
self.notify_on_missing = notify_on_missing
for i in notify_on_missing:
if not isinstance(i, str):
config.error('notify_on_missing arg %r is not a string' %
(i, ))
self.missing_timeout = missing_timeout
self.missing_timer = None
# a protocol connection, if we're currently connected
self.conn = None
# during disconnection self.conn will be set to None before all disconnection notifications
# are delivered. During that period _pending_disconnection_delivery_notifier will be set to
# a notifier and allows interested users to wait until all disconnection notifications are
# delivered.
self._pending_disconnection_delivery_notifier = None
self._old_builder_list = None
self._configured_builderid_list = None
class JhBuildMaster(BuildMaster):
jhbuild_config = config
def loadConfig(self, f):
# modified from parent method to get slaves, projects, change
# sources, schedulers, builders and web status ouf of
# master.cfg [it would have been cleaner if jhbuild didn't
# have to copy all that code.]
localDict = {'basedir': os.path.expanduser(self.basedir)}
try:
exec f in localDict
except:
log.msg("error while parsing config file")
raise
jhbuild_config.load()
try:
config = localDict['BuildmasterConfig']
except KeyError:
log.err("missing config dictionary")
log.err("config file must define BuildmasterConfig")
raise
known_keys = ("bots", "slaves",
"sources", "change_source",
"schedulers", "builders", "mergeRequests",
"slavePortnum", "debugPassword", "logCompressionLimit",
"manhole", "status", "projectName", "projectURL",
"buildbotURL", "properties", "prioritizeBuilders",
"eventHorizon", "buildCacheSize", "logHorizon", "buildHorizon",
"changeHorizon", "logMaxSize", "logMaxTailSize",
"logCompressionMethod",
)
for k in config.keys():
if k not in known_keys:
log.msg("unknown key '%s' defined in config dictionary" % k)
# the 'slaves' list is read from the 'slaves.csv' file in the
# current directory (unless instructed different from command line)
# it is a CSV file structured like this:
# slavename,password
config['slaves'] = []
slaves_csv_file = os.path.join(slaves_dir, 'slaves.csv')
if os.path.exists(slaves_csv_file):
for x in csv.reader(file(slaves_csv_file)):
if not x or x[0].startswith('#'):
continue
kw = {}
build_slave = JhBuildSlave(x[0], x[1])
build_slave.load_extra_configuration(slaves_dir)
config['slaves'].append(build_slave)
if len(config['slaves']) == 0:
log.msg('you must fill slaves.csv with slaves')
module_set = jhbuild.moduleset.load(self.jhbuild_config)
module_list = module_set.get_module_list(
self.jhbuild_config.modules,
self.jhbuild_config.skip,
include_optional_modules=True)
config['projects'] = [x.name for x in module_list \
if not x.name.startswith('meta-')]
if self.jhbuild_config.jhbuildbot_svn_commits_box:
# trigger builds from mails to svn-commit-list
# (note Maildir must be correct, or everything will fail)
from jhbuild.buildbot.changes import GnomeMaildirSource
config['change_source'] = GnomeMaildirSource(
self.jhbuild_config.jhbuildbot_svn_commits_box,
modules=module_list,
prefix=None)
else:
# support injection (use 'buildbot sendchange')
from buildbot.changes.pb import PBChangeSource
config['change_source'] = PBChangeSource()
# Schedulers
from jhbuild.buildbot.scheduler import SerialScheduler, NightlySerialScheduler, OnCommitScheduler
config['schedulers'] = []
for slave in config['slaves']:
s = None
for project in config['projects']:
buildername = str('%s-%s' % (project, slave.slavename))
scheduler_kwargs = {}
if slave.scheduler == 'nightly':
scheduler_class = NightlySerialScheduler
scheduler_kwargs = slave.nightly_kwargs
else:
scheduler_class = SerialScheduler
s = scheduler_class(buildername, project, upstream=s,
builderNames=[buildername],
**scheduler_kwargs)
config['schedulers'].append(s)
if self.jhbuild_config.jhbuildbot_svn_commits_box:
# schedulers that will launch job when receiving
# change notifications
s2 = OnCommitScheduler('oc-' + buildername,
project, builderNames=[buildername])
config['schedulers'].append(s2)
# Builders
from jhbuild.buildbot.factory import JHBuildFactory
config['builders'] = []
for project in config['projects']:
for slave in config['slaves']:
f = JHBuildFactory(project, slave)
config['builders'].append({
'name' : "%s-%s" % (project, slave.slavename),
'slavename' : slave.slavename,
'builddir' : 'builddir/%s.%s' % (project, slave.slavename),
'factory' : f,
'category' : project
})
# Status targets
if not config.has_key('status'):
# let it be possible to define additional status in
# master.cfg
config['status'] = []
from jhbuild.buildbot.status.web import JHBuildWebStatus
config['status'].append(
JHBuildWebStatus(
self.jhbuild_config.moduleset,
config['projects'],
[x.slavename for x in config['slaves']],
http_port=8080, allowForce=True)
)
# remaining of the method is a straight copy from buildbot
# ...
try:
# required
schedulers = config['schedulers']
builders = config['builders']
slavePortnum = config['slavePortnum']
#slaves = config['slaves']
#change_source = config['change_source']
# optional
debugPassword = config.get('debugPassword')
manhole = config.get('manhole')
status = config.get('status', [])
projectName = config.get('projectName')
projectURL = config.get('projectURL')
buildbotURL = config.get('buildbotURL')
properties = config.get('properties', {})
buildCacheSize = config.get('buildCacheSize', None)
eventHorizon = config.get('eventHorizon', None)
logHorizon = config.get('logHorizon', None)
buildHorizon = config.get('buildHorizon', None)
logCompressionLimit = config.get('logCompressionLimit', 4*1024)
if logCompressionLimit is not None and not \
isinstance(logCompressionLimit, int):
raise ValueError("logCompressionLimit needs to be bool or int")
logCompressionMethod = config.get('logCompressionMethod', "bz2")
if logCompressionMethod not in ('bz2', 'gz'):
raise ValueError("logCompressionMethod needs to be 'bz2', or 'gz'")
logMaxSize = config.get('logMaxSize')
if logMaxSize is not None and not \
isinstance(logMaxSize, int):
raise ValueError("logMaxSize needs to be None or int")
logMaxTailSize = config.get('logMaxTailSize')
if logMaxTailSize is not None and not \
isinstance(logMaxTailSize, int):
raise ValueError("logMaxTailSize needs to be None or int")
mergeRequests = config.get('mergeRequests')
if mergeRequests is not None and not callable(mergeRequests):
raise ValueError("mergeRequests must be a callable")
prioritizeBuilders = config.get('prioritizeBuilders')
if prioritizeBuilders is not None and not callable(prioritizeBuilders):
raise ValueError("prioritizeBuilders must be callable")
changeHorizon = config.get("changeHorizon")
if changeHorizon is not None and not isinstance(changeHorizon, int):
raise ValueError("changeHorizon needs to be an int")
except KeyError, e:
log.msg("config dictionary is missing a required parameter")
log.msg("leaving old configuration in place")
raise
#if "bots" in config:
# raise KeyError("c['bots'] is no longer accepted")
slaves = config.get('slaves', [])
if "bots" in config:
m = ("c['bots'] is deprecated as of 0.7.6 and will be "
"removed by 0.8.0 . Please use c['slaves'] instead.")
log.msg(m)
warnings.warn(m, DeprecationWarning)
for name, passwd in config['bots']:
slaves.append(JhBuildSlave(name, passwd))
if "bots" not in config and "slaves" not in config:
log.msg("config dictionary must have either 'bots' or 'slaves'")
log.msg("leaving old configuration in place")
raise KeyError("must have either 'bots' or 'slaves'")
#if "sources" in config:
# raise KeyError("c['sources'] is no longer accepted")
if changeHorizon is not None:
self.change_svc.changeHorizon = changeHorizon
change_source = config.get('change_source', [])
if isinstance(change_source, (list, tuple)):
change_sources = change_source
else:
change_sources = [change_source]
if "sources" in config:
m = ("c['sources'] is deprecated as of 0.7.6 and will be "
"removed by 0.8.0 . Please use c['change_source'] instead.")
log.msg(m)
warnings.warn(m, DeprecationWarning)
for s in config['sources']:
change_sources.append(s)
# do some validation first
for s in slaves:
assert interfaces.IBuildSlave.providedBy(s)
if s.slavename in ("debug", "change", "status"):
raise KeyError(
"reserved name '%s' used for a bot" % s.slavename)
if config.has_key('interlocks'):
raise KeyError("c['interlocks'] is no longer accepted")
assert isinstance(change_sources, (list, tuple))
for s in change_sources:
assert interfaces.IChangeSource(s, None)
# this assertion catches c['schedulers'] = Scheduler(), since
# Schedulers are service.MultiServices and thus iterable.
errmsg = "c['schedulers'] must be a list of Scheduler instances"
assert isinstance(schedulers, (list, tuple)), errmsg
for s in schedulers:
assert interfaces.IScheduler(s, None), errmsg
assert isinstance(status, (list, tuple))
for s in status:
assert interfaces.IStatusReceiver(s, None)
slavenames = [s.slavename for s in slaves]
buildernames = []
dirnames = []
# convert builders from objects to config dictionaries
builders_dicts = []
for b in builders:
if isinstance(b, buildbot.config.BuilderConfig):
builders_dicts.append(b.getConfigDict())
elif type(b) is dict:
builders_dicts.append(b)
else:
raise ValueError("builder %s is not a BuilderConfig object (or a dict)" % b)
builders = builders_dicts
for b in builders:
if b.has_key('slavename') and b['slavename'] not in slavenames:
raise ValueError("builder %s uses undefined slave %s" \
% (b['name'], b['slavename']))
for n in b.get('slavenames', []):
if n not in slavenames:
raise ValueError("builder %s uses undefined slave %s" \
% (b['name'], n))
if b['name'] in buildernames:
raise ValueError("duplicate builder name %s"
% b['name'])
buildernames.append(b['name'])
# sanity check name (BuilderConfig does this too)
if b['name'].startswith("_"):
errmsg = ("builder names must not start with an "
"underscore: " + b['name'])
log.err(errmsg)
raise ValueError(errmsg)
# Fix the dictionnary with default values, in case this wasn't
# specified with a BuilderConfig object (which sets the same defaults)
b.setdefault('builddir', buildbot.util.safeTranslate(b['name']))
b.setdefault('slavebuilddir', b['builddir'])
if b['builddir'] in dirnames:
raise ValueError("builder %s reuses builddir %s"
% (b['name'], b['builddir']))
dirnames.append(b['builddir'])
unscheduled_buildernames = buildernames[:]
schedulernames = []
for s in schedulers:
for b in s.listBuilderNames():
assert b in buildernames, \
"%s uses unknown builder %s" % (s, b)
if b in unscheduled_buildernames:
unscheduled_buildernames.remove(b)
if s.name in schedulernames:
# TODO: schedulers share a namespace with other Service
# children of the BuildMaster node, like status plugins, the
# Manhole, the ChangeMaster, and the BotMaster (although most
# of these don't have names)
msg = ("Schedulers must have unique names, but "
"'%s' was a duplicate" % (s.name,))
raise ValueError(msg)
schedulernames.append(s.name)
if unscheduled_buildernames:
log.msg("Warning: some Builders have no Schedulers to drive them:"
" %s" % (unscheduled_buildernames,))
# assert that all locks used by the Builds and their Steps are
# uniquely named.
lock_dict = {}
for b in builders:
for l in b.get('locks', []):
if isinstance(l, locks.LockAccess): # User specified access to the lock
l = l.lockid
if lock_dict.has_key(l.name):
if lock_dict[l.name] is not l:
raise ValueError("Two different locks (%s and %s) "
"share the name %s"
% (l, lock_dict[l.name], l.name))
else:
lock_dict[l.name] = l
# TODO: this will break with any BuildFactory that doesn't use a
# .steps list, but I think the verification step is more
# important.
for s in b['factory'].steps:
for l in s[1].get('locks', []):
if isinstance(l, locks.LockAccess): # User specified access to the lock
l = l.lockid
if lock_dict.has_key(l.name):
if lock_dict[l.name] is not l:
raise ValueError("Two different locks (%s and %s)"
" share the name %s"
% (l, lock_dict[l.name], l.name))
else:
lock_dict[l.name] = l
if not isinstance(properties, dict):
raise ValueError("c['properties'] must be a dictionary")
# slavePortnum supposed to be a strports specification
if type(slavePortnum) is int:
slavePortnum = "tcp:%d" % slavePortnum
# now we're committed to implementing the new configuration, so do
# it atomically
# TODO: actually, this is spread across a couple of Deferreds, so it
# really isn't atomic.
d = defer.succeed(None)
self.projectName = projectName
self.projectURL = projectURL
self.buildbotURL = buildbotURL
self.properties = Properties()
self.properties.update(properties, self.configFileName)
self.status.logCompressionLimit = logCompressionLimit
self.status.logCompressionMethod = logCompressionMethod
self.status.logMaxSize = logMaxSize
self.status.logMaxTailSize = logMaxTailSize
# Update any of our existing builders with the current log parameters.
# This is required so that the new value is picked up after a
# reconfig.
for builder in self.botmaster.builders.values():
builder.builder_status.setLogCompressionLimit(logCompressionLimit)
builder.builder_status.setLogCompressionMethod(logCompressionMethod)
builder.builder_status.setLogMaxSize(logMaxSize)
builder.builder_status.setLogMaxTailSize(logMaxTailSize)
if mergeRequests is not None:
self.botmaster.mergeRequests = mergeRequests
if prioritizeBuilders is not None:
self.botmaster.prioritizeBuilders = prioritizeBuilders
self.buildCacheSize = buildCacheSize
self.eventHorizon = eventHorizon
self.logHorizon = logHorizon
self.buildHorizon = buildHorizon
# self.slaves: Disconnect any that were attached and removed from the
# list. Update self.checker with the new list of passwords, including
# debug/change/status.
d.addCallback(lambda res: self.loadConfig_Slaves(slaves))
# self.debugPassword
if debugPassword:
self.checker.addUser("debug", debugPassword)
self.debugPassword = debugPassword
# self.manhole
if manhole != self.manhole:
# changing
if self.manhole:
# disownServiceParent may return a Deferred
d.addCallback(lambda res: self.manhole.disownServiceParent())
def _remove(res):
self.manhole = None
return res
d.addCallback(_remove)
if manhole:
def _add(res):
self.manhole = manhole
manhole.setServiceParent(self)
d.addCallback(_add)
# add/remove self.botmaster.builders to match builders. The
# botmaster will handle startup/shutdown issues.
d.addCallback(lambda res: self.loadConfig_Builders(builders))
d.addCallback(lambda res: self.loadConfig_status(status))
# Schedulers are added after Builders in case they start right away
d.addCallback(lambda res: self.loadConfig_Schedulers(schedulers))
# and Sources go after Schedulers for the same reason
d.addCallback(lambda res: self.loadConfig_Sources(change_sources))
# self.slavePort
if self.slavePortnum != slavePortnum:
if self.slavePort:
def closeSlavePort(res):
d1 = self.slavePort.disownServiceParent()
self.slavePort = None
return d1
d.addCallback(closeSlavePort)
if slavePortnum is not None:
def openSlavePort(res):
self.slavePort = strports.service(slavePortnum,
self.slaveFactory)
self.slavePort.setServiceParent(self)
d.addCallback(openSlavePort)
log.msg("BuildMaster listening on port %s" % slavePortnum)
self.slavePortnum = slavePortnum
log.msg("configuration update started")
def _done(res):
self.readConfig = True
log.msg("configuration update complete")
d.addCallback(_done)
d.addCallback(lambda res: self.botmaster.maybeStartAllBuilds())
return d
def __init__(self,
name,
password,
max_builds=None,
notify_on_missing=[],
missing_timeout=3600,
properties={},
locks=None,
keepalive_interval=3600):
"""
@param name: botname this machine will supply when it connects
@param password: password this machine will supply when
it connects
@param max_builds: maximum number of simultaneous builds that will
be run concurrently on this buildslave (the
default is None for no limit)
@param properties: properties that will be applied to builds run on
this slave
@type properties: dictionary
@param locks: A list of locks that must be acquired before this slave
can be used
@type locks: dictionary
"""
service.MultiService.__init__(self)
self.slavename = name
self.password = password
# PB registration
self.registration = None
self.registered_port = None
# these are set when the service is started, and unset when it is
# stopped
self.botmaster = None
self.master = None
self.slave_status = SlaveStatus(name)
self.slave = None # a RemoteReference to the Bot, when connected
self.slave_commands = None
self.slavebuilders = {}
self.max_builds = max_builds
self.access = []
if locks:
self.access = locks
self.lock_subscriptions = []
self.properties = Properties()
self.properties.update(properties, "BuildSlave")
self.properties.setProperty("slavename", name, "BuildSlave")
self.lastMessageReceived = 0
if isinstance(notify_on_missing, str):
notify_on_missing = [notify_on_missing]
self.notify_on_missing = notify_on_missing
for i in notify_on_missing:
if not isinstance(i, str):
config.error('notify_on_missing arg %r is not a string' %
(i, ))
self.missing_timeout = missing_timeout
self.missing_timer = None
self.keepalive_interval = keepalive_interval
self.detached_subs = None
self._old_builder_list = None
def __init__(self, reason, sources, buildername):
self.reason = reason
self.sources = sources
self.buildername = buildername
self.changes = []
self.properties = Properties()
def __init__(self):
self.sources = []
self.reason = "Because"
self.properties = Properties()
class FakeChange:
properties = Properties()
def __init__(self, number=None):
self.number = number
self.who = "me"
def __init__(self, number=None):
self.properties = Properties()
self.number = number
self.who = "me"
def forceWithWebRequest(self, owner, builder_name, req):
"""Called by the web UI.
Authentication is already done, thus owner is passed as argument
We check the parameters, and launch the build, if everything is correct
"""
if not builder_name in self.builderNames:
# in the case of buildAll, this method will be called several times
# for all the builders
# we just do nothing on a builder that is not in our builderNames
return defer.succeed(None)
master = self.master
properties = {}
changeids = []
# probably need to clean that out later as the IProperty is already a
# validation mechanism
validation = master.config.validation
if self.branch.regex == None:
self.branch.regex = validation['branch']
if self.revision.regex == None:
self.revision.regex = validation['revision']
for param in self.all_fields:
if owner and param == self.username:
continue # dont enforce username if auth
param.update_from_post(master, properties, changeids, req)
changeids = map(lambda a: type(a) == int and a or a.number, changeids)
# everything is validated, we can create our source stamp, and buildrequest
reason = properties[self.reason.name]
branch = properties[self.branch.name]
revision = properties[self.revision.name]
repository = properties[self.repository.name]
project = properties[self.project.name]
if owner is None:
owner = properties[self.username.name]
std_prop_names = [
self.branch.name, self.revision.name, self.repository.name,
self.project.name, self.username.name
]
real_properties = Properties()
for pname, pvalue in properties.items():
if not pname in std_prop_names:
real_properties.setProperty(pname, pvalue, "Force Build Form")
real_properties.setProperty("owner", owner, "Force Build Form")
r = ("The web-page 'force build' button was pressed by '%s': %s\n" %
(owner, reason))
d = master.db.sourcestampsets.addSourceStampSet()
def add_master_with_setid(sourcestampsetid):
master.db.sourcestamps.addSourceStamp(
sourcestampsetid=sourcestampsetid,
branch=branch,
revision=revision,
project=project,
repository=repository,
changeids=changeids)
return sourcestampsetid
d.addCallback(add_master_with_setid)
def got_setid(sourcestampsetid):
return self.addBuildsetForSourceStamp(builderNames=[builder_name],
setid=sourcestampsetid,
reason=r,
properties=real_properties)
d.addCallback(got_setid)
return d
def command_FORCE(self, args, **kwargs):
"""force a build"""
# FIXME: NEED TO THINK ABOUT!
errReply = f"Try '{self.bot.commandPrefix}{self.command_FORCE.usage}'"
args = self.splitArgs(args)
if not args:
raise UsageError(errReply)
what = args.pop(0)
if what != "build":
raise UsageError(errReply)
opts = ForceOptions()
opts.parseOptions(args)
builderName = opts['builder']
builder = yield self.bot.getBuilder(buildername=builderName)
branch = opts['branch']
revision = opts['revision']
codebase = opts['codebase']
project = opts['project']
reason = opts['reason']
props = opts['props']
if builderName is None:
raise UsageError("you must provide a Builder, " + errReply)
# keep weird stuff out of the branch, revision, and properties args.
branch_validate = self.master.config.validation['branch']
revision_validate = self.master.config.validation['revision']
pname_validate = self.master.config.validation['property_name']
pval_validate = self.master.config.validation['property_value']
if branch and not branch_validate.match(branch):
self.bot.log(f"Force: bad branch '{branch}'")
self.send(f"Sorry, bad branch '{branch}'")
return
if revision and not revision_validate.match(revision):
self.bot.log(f"Force: bad revision '{revision}'")
self.send(f"Sorry, bad revision '{revision}'")
return
properties = Properties()
properties.master = self.master
if props:
# split props into name:value dict
pdict = {}
propertylist = props.split(",")
for prop in propertylist:
splitproperty = prop.split("=", 1)
pdict[splitproperty[0]] = splitproperty[1]
# set properties
for pname, pvalue in pdict.items():
if not pname_validate.match(pname) \
or not pval_validate.match(pvalue):
self.bot.log(
f"Force: bad property name='{pname}', value='{pvalue}'"
)
self.send(
f"Sorry, bad property name='{pname}', value='{pvalue}'"
)
return
properties.setProperty(pname, pvalue, "Force Build Chat")
properties.setProperty("reason", reason, "Force Build Chat")
properties.setProperty("owner", self.describeUser(),
"Force Build Chat")
reason = f"forced: by {self.describeUser()}: {reason}"
try:
yield self.master.data.updates.addBuildset(
builderids=[builder['builderid']],
# For now, we just use
# this as the id.
scheduler="status.words",
sourcestamps=[{
'codebase': codebase,
'branch': branch,
'revision': revision,
'project': project,
'repository': ""
}],
reason=reason,
properties=properties.asDict(),
waited_for=False)
except AssertionError as e:
self.send("I can't: " + str(e))
else:
self.send("Force build successfully requested.")
def test_rejects_build_on_instance_with_different_type_timeout_nonzero(
self):
"""
If latent worker supports getting its instance type from properties that
are rendered from build then the buildrequestdistributor must not
schedule any builds on workers that are running different instance type
than what these builds will require.
"""
controller = LatentController(self,
'local',
kind=Interpolate('%(prop:worker_kind)s'),
build_wait_timeout=5)
# a step that we can finish when we want
stepcontroller = BuildStepController()
config_dict = {
'builders': [
BuilderConfig(
name="testy",
workernames=["local"],
factory=BuildFactory([stepcontroller.step]),
),
],
'workers': [controller.worker],
'protocols': {
'null': {}
},
'multiMaster':
True,
}
master = self.getMaster(config_dict)
builder_id = self.successResultOf(
master.data.updates.findBuilderId('testy'))
# create build request
self.createBuildrequest(master, [builder_id],
properties=Properties(worker_kind='a'))
# start the build and verify the kind of the worker. Note that the
# buildmaster needs to restart the worker in order to change the worker
# kind, so we allow it both to auto start and stop
self.assertEqual(True, controller.starting)
controller.auto_connect_worker = True
controller.auto_disconnect_worker = True
controller.auto_start(True)
controller.auto_stop(True)
controller.connect_worker()
self.assertEqual('a', (yield controller.get_started_kind()))
# before the other build finished, create another build request
self.createBuildrequest(master, [builder_id],
properties=Properties(worker_kind='b'))
stepcontroller.finish_step(SUCCESS)
# give the botmaster chance to insubstantiate the worker and
# maybe substantiate it for the pending build the builds on worker
self.reactor.advance(0.1)
# verify build has not started, even though the worker is waiting
# for one
self.assertIsNone((yield master.db.builds.getBuild(2)))
self.assertTrue(controller.started)
# wait until the latent worker times out, is insubstantiated,
# is substantiated because of pending buildrequest and starts the build
self.reactor.advance(6)
self.assertIsNotNone((yield master.db.builds.getBuild(2)))
# verify that the second build restarted with the expected instance
# kind
self.assertEqual('b', (yield controller.get_started_kind()))
stepcontroller.finish_step(SUCCESS)
dbdict = yield master.db.builds.getBuild(1)
self.assertEqual(SUCCESS, dbdict['results'])
dbdict = yield master.db.builds.getBuild(2)
self.assertEqual(SUCCESS, dbdict['results'])
def setUp(self):
self.props = Properties()
self.build = FakeBuild(self.props)
def getProperties(self):
return Properties()
def upgradeToVersion3(self):
# in version 3, self.properties became a Properties object
propdict = self.properties
self.properties = Properties()
self.properties.update(propdict, "Upgrade from previous version")
self.wasUpgraded = True
def setUp(self):
self.build = Properties(image="busybox:latest")
self.patch(dockerbuildslave, 'client', docker)
def command_FORCE(self, args, who):
errReply = "try 'force build [--branch=BRANCH] [--revision=REVISION] [--props=PROP1=VAL1,PROP2=VAL2...] <WHICH> <REASON>'"
args = self.splitArgs(args)
if not args:
raise UsageError(errReply)
what = args.pop(0)
if what != "build":
raise UsageError(errReply)
opts = ForceOptions()
opts.parseOptions(args)
which = opts['builder']
branch = opts['branch']
revision = opts['revision']
reason = opts['reason']
props = opts['props']
if which is None:
raise UsageError("you must provide a Builder, " + errReply)
# keep weird stuff out of the branch, revision, and properties args.
branch_validate = self.master.config.validation['branch']
revision_validate = self.master.config.validation['revision']
pname_validate = self.master.config.validation['property_name']
pval_validate = self.master.config.validation['property_value']
if branch and not branch_validate.match(branch):
log.msg("bad branch '%s'" % branch)
self.send("sorry, bad branch '%s'" % branch)
return
if revision and not revision_validate.match(revision):
log.msg("bad revision '%s'" % revision)
self.send("sorry, bad revision '%s'" % revision)
return
properties = Properties()
if props:
# split props into name:value dict
pdict = {}
propertylist = props.split(",")
for i in range(0, len(propertylist)):
splitproperty = propertylist[i].split("=", 1)
pdict[splitproperty[0]] = splitproperty[1]
# set properties
for prop in pdict:
pname = prop
pvalue = pdict[prop]
if not pname_validate.match(pname) \
or not pval_validate.match(pvalue):
log.msg("bad property name='%s', value='%s'" %
(pname, pvalue))
self.send("sorry, bad property name='%s', value='%s'" %
(pname, pvalue))
return
properties.setProperty(pname, pvalue, "Force Build IRC")
bc = self.getControl(which)
reason = "forced: by %s: %s" % (self.describeUser(who), reason)
ss = SourceStamp(branch=branch, revision=revision)
d = bc.submitBuildRequest(ss, reason, props=properties.asDict())
def subscribe(buildreq):
ireq = IrcBuildRequest(self, self.useRevisions)
buildreq.subscribe(ireq.started)
d.addCallback(subscribe)
d.addErrback(log.err, "while forcing a build")
def createTriggerProperties(self, properties):
# make a new properties object from a dict rendered by the old
# properties object
trigger_properties = Properties()
trigger_properties.update(properties, "Trigger")
return trigger_properties
def setUp(self):
self.patch(openstack, "client", novaclient)
self.patch(openstack, "loading", novaclient)
self.patch(openstack, "session", novaclient)
self.build = Properties(image=novaclient.TEST_UUIDS['image'])
def reconfigService(self,
name,
password,
max_builds=None,
notify_on_missing=None,
missing_timeout=DEFAULT_MISSING_TIMEOUT,
properties=None,
defaultProperties=None,
locks=None,
keepalive_interval=DEFAULT_KEEPALIVE_INTERVAL,
machine_name=None):
# Given a Worker config arguments, configure this one identically.
# Because Worker objects are remotely referenced, we can't replace them
# without disconnecting the worker, yet there's no reason to do that.
assert self.name == name
self.password = password
# adopt new instance's configuration parameters
self.max_builds = max_builds
self.access = []
if locks:
self.access = locks
if notify_on_missing is None:
notify_on_missing = []
if isinstance(notify_on_missing, str):
notify_on_missing = [notify_on_missing]
self.notify_on_missing = notify_on_missing
if self.missing_timeout != missing_timeout:
running_missing_timer = self.missing_timer
self.stopMissingTimer()
self.missing_timeout = missing_timeout
if running_missing_timer:
self.startMissingTimer()
self.properties = Properties()
self.properties.update(properties or {}, "Worker")
self.properties.setProperty("workername", name, "Worker")
self.defaultProperties = Properties()
self.defaultProperties.update(defaultProperties or {}, "Worker")
# Note that before first reconfig self.machine will always be None and
# out of sync with self.machine_name, thus more complex logic is needed.
if self.machine is not None and self.machine_name != machine_name:
self.machine.unregisterWorker(self)
self.machine = None
self.machine_name = machine_name
if self.machine is None and self.machine_name is not None:
self.machine = self.master.machine_manager.getMachineByName(
self.machine_name)
if self.machine is not None:
self.machine.registerWorker(self)
self.properties.setProperty("machine_name", self.machine_name,
"Worker")
else:
log.err("Unknown machine '{}' for worker '{}'".format(
self.machine_name, self.name))
# update our records with the worker manager
if not self.registration:
self.registration = yield self.master.workers.register(self)
yield self.registration.update(self, self.master.config)
# tracks config version for locks
self.config_version = self.master.config_version
self.updateLocks()
def command_FORCE(self, args):
# FIXME: NEED TO THINK ABOUT!
errReply = "try 'force build [--branch=BRANCH] [--revision=REVISION] [--props=PROP1=VAL1,PROP2=VAL2...] <WHICH> <REASON>'"
args = self.splitArgs(args)
if not args:
raise UsageError(errReply)
what = args.pop(0)
if what != "build":
raise UsageError(errReply)
opts = ForceOptions()
opts.parseOptions(args)
builderName = opts['builder']
branch = opts['branch']
revision = opts['revision']
reason = opts['reason']
props = opts['props']
if builderName is None:
raise UsageError("you must provide a Builder, " + errReply)
# keep weird stuff out of the branch, revision, and properties args.
branch_validate = self.master.config.validation['branch']
revision_validate = self.master.config.validation['revision']
pname_validate = self.master.config.validation['property_name']
pval_validate = self.master.config.validation['property_value']
if branch and not branch_validate.match(branch):
log.msg("bad branch '%s'" % branch)
self.send("sorry, bad branch '%s'" % branch)
return
if revision and not revision_validate.match(revision):
log.msg("bad revision '%s'" % revision)
self.send("sorry, bad revision '%s'" % revision)
return
properties = Properties()
if props:
# split props into name:value dict
pdict = {}
propertylist = props.split(",")
for i in range(0, len(propertylist)):
splitproperty = propertylist[i].split("=", 1)
pdict[splitproperty[0]] = splitproperty[1]
# set properties
for prop in pdict:
pname = prop
pvalue = pdict[prop]
if not pname_validate.match(pname) \
or not pval_validate.match(pvalue):
log.msg("bad property name='%s', value='%s'" %
(pname, pvalue))
self.send("sorry, bad property name='%s', value='%s'" %
(pname, pvalue))
return
properties.setProperty(pname, pvalue, "Force Build chat")
reason = u"forced: by %s: %s" % (self.describeUser(), reason)
d = self.master.data.updates.addBuildset(
builderNames=[builderName],
scheduler=u"status.words", # For now, we just use this as the id.
sourcestamps=[{
'branch': branch,
'revision': revision
}],
reason=reason,
properties=properties.asDict(),
waited_for=False)
@d.addCallback
def subscribe(xxx_todo_changeme):
(bsid, brids) = xxx_todo_changeme
assert 0, "rewrite to not use the status hierarchy" # TODO
# ireq = BuildRequest(self, self.useRevisions)
# buildreq.subscribe(ireq.started)
d.addErrback(log.err, "while forcing a build")
class FakeBuild:
properties = Properties(foo='foo', bar='bar', baz='baz')
def buildUIDSchedFunc(sched, t, ssid):
"""Return a Properties instance with 'builduid' set to a randomly generated
id."""
props = Properties()
props.setProperty('builduid', genBuildUID(), 'buildUIDSchedFunc')
return props
def checkConfig(self, name, password, max_builds=None,
notify_on_missing=None,
missing_timeout=10 * 60, # Ten minutes
properties=None, locks=None, keepalive_interval=3600):
"""
@param name: botname this machine will supply when it connects
@param password: password this machine will supply when
it connects
@param max_builds: maximum number of simultaneous builds that will
be run concurrently on this worker (the
default is None for no limit)
@param properties: properties that will be applied to builds run on
this worker
@type properties: dictionary
@param locks: A list of locks that must be acquired before this worker
can be used
@type locks: dictionary
"""
self.name = name = ascii2unicode(name)
if properties is None:
properties = {}
self.password = password
# protocol registration
self.registration = None
# these are set when the service is started
self.manager = None
self.workerid = None
self.worker_status = WorkerStatus(name)
self.worker_commands = None
self.workerforbuilders = {}
self.max_builds = max_builds
self.access = []
if locks:
self.access = locks
self.lock_subscriptions = []
self.properties = Properties()
self.properties.update(properties, "Worker")
self.properties.setProperty("slavename", name, "Worker (deprecated)")
self.properties.setProperty("workername", name, "Worker")
self.lastMessageReceived = 0
if notify_on_missing is None:
notify_on_missing = []
if isinstance(notify_on_missing, str):
notify_on_missing = [notify_on_missing]
self.notify_on_missing = notify_on_missing
for i in notify_on_missing:
if not isinstance(i, str):
config.error(
'notify_on_missing arg %r is not a string' % (i,))
self.missing_timeout = missing_timeout
self.missing_timer = None
# a protocol connection, if we're currently connected
self.conn = None
self._old_builder_list = None
def setUp(self):
self.setUpTestReactor()
self.build = Properties(image="busybox:latest",
builder="docker_worker")
self.worker = None
def oneTest(self, props, expected):
p = Properties()
p.update(props, "test")
r = repo.RepoDownloadsFromProperties(list(props))
self.assertEqual(r.getRenderingFor(p), expected)
def setUp(self):
self.props = Properties()
