def __init__(self, name, builderNames, minute=0, hour='*',
dayOfMonth='*', month='*', dayOfWeek='*',
branch=NoBranch, fileIsImportant=None, onlyIfChanged=False,
properties={}, change_filter=None, onlyImportant=False):
Timed.__init__(self, name=name, builderNames=builderNames, properties=properties)
# If True, only important changes will be added to the buildset.
self.onlyImportant = onlyImportant
if fileIsImportant and not callable(fileIsImportant):
raise config.ConfigErrors([
"fileIsImportant must be a callable" ])
if branch is Nightly.NoBranch:
raise config.ConfigErrors([
"Nightly parameter 'branch' is required" ])
self.minute = minute
self.hour = hour
self.dayOfMonth = dayOfMonth
self.month = month
self.dayOfWeek = dayOfWeek
self.branch = branch
self.onlyIfChanged = onlyIfChanged
self.fileIsImportant = fileIsImportant
self.change_filter = filter.ChangeFilter.fromSchedulerConstructorArgs(
change_filter=change_filter)
self.reason = "The Nightly scheduler named '%s' triggered this build" % self.name
def __init__(self, **kwargs):
BuildStep.__init__(self, **kwargs)
if self.upstreamSteps is None:
raise config.ConfigErrors(["you must supply upstreamSteps"])
if len(self.upstreamSteps) < 1:
raise config.ConfigErrors(
["upstreamSteps must be a non-empty list"])
if self.idlePolicy not in self.VALID_IDLE_POLICIES:
raise config.ConfigErrors([
"invalid value for idlePolicy: %r (must be one of %s)" %
(self.idlePolicy, ", ".join(map(repr,
self.VALID_IDLE_POLICIES)))
])
# list of build steps (as BuildStepStatus objects) that we're
# currently waiting on
self._blocking_steps = []
# set of builders (as BuilderStatus objects) that have to start
# a Build before we can block on one of their BuildSteps
self._blocking_builders = set()
self._overall_code = builder.SUCCESS # assume the best
self._overall_text = []
self._timer = None # object returned by reactor.callLater()
self._timed_out = False
def __init__(self,
name,
shouldntBeSet=NotSet,
treeStableTimer=None,
builderNames=None,
branch=NotABranch,
branches=NotABranch,
fileIsImportant=None,
properties={},
categories=None,
change_filter=None,
onlyImportant=False):
if shouldntBeSet is not self.NotSet:
raise config.ConfigErrors(
["pass arguments to schedulers using keyword arguments"])
if fileIsImportant and not callable(fileIsImportant):
raise config.ConfigErrors(["fileIsImportant must be a callable"])
# initialize parent classes
base.BaseScheduler.__init__(self, name, builderNames, properties)
self.treeStableTimer = treeStableTimer
self.fileIsImportant = fileIsImportant
self.onlyImportant = onlyImportant
self.change_filter = self.getChangeFilter(branch=branch,
branches=branches,
change_filter=change_filter,
categories=categories)
# the IDelayedCall used to wake up when this scheduler's
# treeStableTimer expires.
self._stable_timers = bbcollections.defaultdict(lambda: None)
self._stable_timers_lock = defer.DeferredLock()
def test_str(self):
ex = config.ConfigErrors()
self.assertEqual(str(ex), "")
ex = config.ConfigErrors(["a"])
self.assertEqual(str(ex), "a")
ex = config.ConfigErrors(["a", "b"])
self.assertEqual(str(ex), "a\nb")
ex = config.ConfigErrors(["a"])
ex.addError('c')
self.assertEqual(str(ex), "a\nc")
def getChangeFilter(self, branch, branches, change_filter, categories):
if branch is NotABranch and not change_filter:
raise config.ConfigErrors([
"the 'branch' argument to SingleBranchScheduler is " +
"mandatory unless change_filter is provided"
])
elif branches is not NotABranch:
raise config.ConfigErrors([
"the 'branches' argument is not allowed for " +
"SingleBranchScheduler"
])
return filter.ChangeFilter.fromSchedulerConstructorArgs(
change_filter=change_filter, branch=branch, categories=categories)
def __init__(self,
s,
slavedest,
workdir=None,
maxsize=None,
blocksize=16 * 1024,
mode=None,
**buildstep_kwargs):
BuildStep.__init__(self, **buildstep_kwargs)
self.addFactoryArguments(
s=s,
slavedest=slavedest,
workdir=workdir,
maxsize=maxsize,
blocksize=blocksize,
mode=mode,
)
self.s = s
self.slavedest = slavedest
self.workdir = workdir
self.maxsize = maxsize
self.blocksize = blocksize
if not isinstance(mode, (int, type(None))):
raise config.ConfigErrors(['mode must be an integer or None'])
self.mode = mode
def __init__(self,
slavesrc,
masterdest,
workdir=None,
maxsize=None,
blocksize=16 * 1024,
compress=None,
url=None,
**buildstep_kwargs):
BuildStep.__init__(self, **buildstep_kwargs)
self.addFactoryArguments(
slavesrc=slavesrc,
masterdest=masterdest,
workdir=workdir,
maxsize=maxsize,
blocksize=blocksize,
compress=compress,
url=url,
)
self.slavesrc = slavesrc
self.masterdest = masterdest
self.workdir = workdir
self.maxsize = maxsize
self.blocksize = blocksize
if compress not in (None, 'gz', 'bz2'):
raise config.ConfigErrors(
["'compress' must be one of None, 'gz', or 'bz2'"])
self.compress = compress
self.url = url
def __init__(self,
slavesrc,
masterdest,
workdir=None,
maxsize=None,
blocksize=16 * 1024,
mode=None,
keepstamp=False,
url=None,
**buildstep_kwargs):
BuildStep.__init__(self, **buildstep_kwargs)
self.addFactoryArguments(
slavesrc=slavesrc,
masterdest=masterdest,
workdir=workdir,
maxsize=maxsize,
blocksize=blocksize,
mode=mode,
keepstamp=keepstamp,
url=url,
)
self.slavesrc = slavesrc
self.masterdest = masterdest
self.workdir = workdir
self.maxsize = maxsize
self.blocksize = blocksize
if not isinstance(mode, (int, type(None))):
raise config.ConfigErrors(['mode must be an integer or None'])
self.mode = mode
self.keepstamp = keepstamp
self.url = url
def __init__(self, name, builderNames, properties):
"""
Initialize a Scheduler.
@param name: name of this scheduler (used as a key for state)
@type name: unicode
@param builderNames: list of builders this scheduler may start
@type builderNames: list of unicode
@param properties: properties to add to builds triggered by this
scheduler
@type properties: dictionary
@param consumeChanges: true if this scheduler wishes to be informed
about the addition of new changes. Defaults to False. This should
be passed explicitly from subclasses to indicate their interest in
consuming changes.
@type consumeChanges: boolean
"""
service.MultiService.__init__(self)
self.name = name
"name of this scheduler; used to identify replacements on reconfig"
ok = True
if not isinstance(builderNames, (list, tuple)):
ok = False
else:
for b in builderNames:
if not isinstance(b, basestring):
ok = False
if not ok:
raise config.ConfigErrors([
"The builderNames argument to a scheduler must be a list "
"of Builder names."
])
self.builderNames = builderNames
"list of builder names to start in each buildset"
self.properties = Properties()
"properties that are contributed to each buildset"
self.properties.update(properties, "Scheduler")
self.properties.setProperty("scheduler", name, "Scheduler")
self.schedulerid = None
"""ID of this scheduler; set just before the scheduler starts, and set
to None after stopService is complete."""
self.master = None
"""BuildMaster instance; set just before the scheduler starts, and set
to None after stopService is complete."""
# internal variables
self._change_subscription = None
self._change_consumption_lock = defer.DeferredLock()
self._objectid = None
def __init__(self, name, builderNames, periodicBuildTimer,
branch=None, properties={}, onlyImportant=False):
Timed.__init__(self, name=name, builderNames=builderNames,
properties=properties)
if periodicBuildTimer <= 0:
raise config.ConfigErrors([
"periodicBuildTimer must be positive" ])
self.periodicBuildTimer = periodicBuildTimer
self.branch = branch
self.reason = "The Periodic scheduler named '%s' triggered this build" % self.name
def __init__(self,
schedulerNames=[],
sourceStamp=None,
updateSourceStamp=None,
alwaysUseLatest=False,
waitForFinish=False,
set_properties={},
copy_properties=[],
**kwargs):
if not schedulerNames:
raise config.ConfigErrors(
["You must specify a scheduler to trigger"])
if sourceStamp and (updateSourceStamp is not None):
raise config.ConfigErrors(
["You can't specify both sourceStamp and updateSourceStamp"])
if sourceStamp and alwaysUseLatest:
raise config.ConfigErrors(
["You can't specify both sourceStamp and alwaysUseLatest"])
if alwaysUseLatest and (updateSourceStamp is not None):
raise config.ConfigErrors([
"You can't specify both alwaysUseLatest and updateSourceStamp"
])
self.schedulerNames = schedulerNames
self.sourceStamp = sourceStamp
if updateSourceStamp is not None:
self.updateSourceStamp = updateSourceStamp
else:
self.updateSourceStamp = not (alwaysUseLatest or sourceStamp)
self.alwaysUseLatest = alwaysUseLatest
self.waitForFinish = waitForFinish
self.set_properties = set_properties
self.copy_properties = copy_properties
self.running = False
self.ended = False
LoggingBuildStep.__init__(self, **kwargs)
self.addFactoryArguments(schedulerNames=schedulerNames,
sourceStamp=sourceStamp,
updateSourceStamp=updateSourceStamp,
alwaysUseLatest=alwaysUseLatest,
waitForFinish=waitForFinish,
set_properties=set_properties,
copy_properties=copy_properties)
def __init__(self, name, upstream, builderNames, properties={}):
base.BaseScheduler.__init__(self, name, builderNames, properties)
if not interfaces.IScheduler.providedBy(upstream):
raise config.ConfigErrors(
["upstream must be another Scheduler instance"])
self.upstream_name = upstream.name
self._buildset_addition_subscr = None
self._buildset_completion_subscr = None
# the subscription lock makes sure that we're done inserting a
# subcription into the DB before registering that the buildset is
# complete.
self._subscription_lock = defer.DeferredLock()
def __init__(self, sphinx_sourcedir='.', sphinx_builddir=None,
sphinx_builder=None, sphinx = 'sphinx-build', tags = [],
defines = {}, mode='incremental', **kwargs):
errors = []
if sphinx_builddir is None:
# Who the heck is not interested in the built doc ?
errors.append("Sphinx argument sphinx_builddir is required")
if mode not in ('incremental', 'full'):
errors.append("Sphinx argument mode has to be 'incremental' or" +
"'full' is required")
if errors:
raise config.ConfigErrors(errors)
self.warnings = 0
self.success = False
ShellCommand.__init__(self, **kwargs)
# build the command
command = [sphinx]
if sphinx_builder is not None:
command.extend(['-b', sphinx_builder])
for tag in tags:
command.extend(['-t', tag])
for key in sorted(defines):
if defines[key] is None:
command.extend(['-D', key])
elif isinstance(defines[key], bool):
command.extend(['-D',
'%s=%d' % (key, defines[key] and 1 or 0)])
else:
command.extend(['-D', '%s=%s' % (key, defines[key])])
if mode == 'full':
command.extend(['-E']) # Don't use a saved environment
command.extend([sphinx_sourcedir, sphinx_builddir])
self.setCommand(command)
self.addFactoryArguments(
sphinx = sphinx,
sphinx_sourcedir = sphinx_sourcedir,
sphinx_builddir = sphinx_builddir,
sphinx_builder = sphinx_builder,
tags = tags,
defines = defines,
mode = mode,
)
def reconfigServiceWithBuildbotConfig(self, new_config):
if self.configured_db_url is None:
self.configured_db_url = new_config.db['db_url']
elif self.configured_db_url != new_config.db['db_url']:
config.error(
"Cannot change c['db']['db_url'] after the master has started",
)
if self.config.mq['type'] != new_config.mq['type']:
raise config.ConfigErrors([
"Cannot change c['mq']['type'] after the master has started",
])
return super().reconfigServiceWithBuildbotConfig(new_config)
def __init__(self,
serverIP,
serverPort,
debug=None,
maxMemoryItems=None,
maxDiskItems=None,
chunkSize=200,
**kwargs):
"""
@serverIP: IP of the autobahn server
@serverPort: Port of the autobahn server
@debug: Save the json with nice formatting.
@maxMemoryItems: Maximum number of items to keep queued in memory.
@maxDiskItems: Maximum number of items to buffer to disk, if 0, doesn't
use disk at all.
@chunkSize: maximum number of items to send in each at each PUSH.
"""
if not serverIP and not serverPort:
raise config.ConfigErrors(
['AutobahnStatusPush requires a serverIP and serverPort'])
# Parameters.
self.serverIP = serverIP
self.serverPort = serverPort
self.serverUrl = "ws://{0}:{1}/ws".format(serverIP, serverPort)
self.debug = debug
self.chunkSize = chunkSize
self.lastPushWasSuccessful = True
self.protocol = None
self.factory = None
if maxDiskItems != 0:
# The queue directory is determined by the server url.
path = ('events_' +
urlparse.urlparse(self.serverUrl)[1].split(':')[0])
queue = PersistentQueue(
primaryQueue=MemoryQueue(maxItems=maxMemoryItems),
secondaryQueue=DiskQueue(path, maxItems=maxDiskItems))
else:
path = None
queue = MemoryQueue(maxItems=maxMemoryItems)
self.connectToAutobahn()
# Use the unbounded method.
StatusPush.__init__(self,
serverPushCb=AutobahnStatusPush.pushHttp,
queue=queue,
path=path,
**kwargs)
def reconfigService(self, new_config):
if self.configured_db_url is None:
self.configured_db_url = new_config.db['db_url']
elif (self.configured_db_url != new_config.db['db_url']):
config.error(
"Cannot change c['db']['db_url'] after the master has started",
)
if self.config.mq['type'] != new_config.mq['type']:
raise config.ConfigErrors([
"Cannot change c['mq']['type'] after the master has started",
])
return config.ReconfigurableServiceMixin.reconfigService(
self, new_config)
def __init__(self, serverUrl, subject="katana", **kwargs):
"""
@serverUrl: The Nats server to be used to push events notifications to.
@subject: The subject to use when publishing data
"""
if not serverUrl:
raise config.ConfigErrors(['NatsStatusPush requires a serverUrl'])
# Parameters.
self.serverUrl = serverUrl
self.subject = subject
self.client = None
# Use the unbounded method.
QueuedStatusPush.__init__(self, **kwargs)
def __init__(self, property=None, extract_fn=None, strip=True, **kwargs):
self.property = property
self.extract_fn = extract_fn
self.strip = strip
if not ((property is not None) ^ (extract_fn is not None)):
raise config.ConfigErrors(
["Exactly one of property and extract_fn must be set"])
ShellCommand.__init__(self, **kwargs)
self.addFactoryArguments(property=self.property)
self.addFactoryArguments(extract_fn=self.extract_fn)
self.addFactoryArguments(strip=self.strip)
self.property_changes = {}
def __init__(self,
serverUrl,
debug=None,
maxMemoryItems=None,
maxDiskItems=None,
chunkSize=200,
maxHttpRequestSize=2**20,
extra_post_params=None,
**kwargs):
"""
@serverUrl: Base URL to be used to push events notifications.
@maxMemoryItems: Maximum number of items to keep queued in memory.
@maxDiskItems: Maximum number of items to buffer to disk, if 0, doesn't
use disk at all.
@debug: Save the json with nice formatting.
@chunkSize: maximum number of items to send in each at each HTTP POST.
@maxHttpRequestSize: limits the size of encoded data for AE, the default
is 1MB.
"""
if not serverUrl:
raise config.ConfigErrors(['HttpStatusPush requires a serverUrl'])
# Parameters.
self.serverUrl = serverUrl
self.extra_post_params = extra_post_params or {}
self.debug = debug
self.chunkSize = chunkSize
self.lastPushWasSuccessful = True
self.maxHttpRequestSize = maxHttpRequestSize
if maxDiskItems != 0:
# The queue directory is determined by the server url.
path = ('events_' +
urlparse.urlparse(self.serverUrl)[1].split(':')[0])
queue = PersistentQueue(
primaryQueue=MemoryQueue(maxItems=maxMemoryItems),
secondaryQueue=DiskQueue(path, maxItems=maxDiskItems))
else:
path = None
queue = MemoryQueue(maxItems=maxMemoryItems)
# Use the unbounded method.
StatusPush.__init__(self,
serverPushCb=HttpStatusPush.pushHttp,
queue=queue,
path=path,
**kwargs)
def reconfigService(self, new_config):
if self.config.db['db_url'] != new_config.db['db_url']:
raise config.ConfigErrors([
"Cannot change c['db']['db_url'] after the master has started",
])
# adjust the db poller
if (self.config.db['db_poll_interval'] !=
new_config.db['db_poll_interval']):
if self.db_loop:
self.db_loop.stop()
self.db_loop = None
poll_interval = new_config.db['db_poll_interval']
if poll_interval:
self.db_loop = task.LoopingCall(self.pollDatabase)
self.db_loop.start(poll_interval, now=False)
return config.ReconfigurableServiceMixin.reconfigService(
self, new_config)
def test_addError(self):
ex = config.ConfigErrors(['a'])
ex.addError('c')
self.assertEqual(ex.errors, ['a', 'c'])
def test_constr(self):
ex = config.ConfigErrors(['a', 'b'])
self.assertEqual(ex.errors, ['a', 'b'])
def setUp(self):
self.cfg = config.MasterConfig()
self.errors = config.ConfigErrors()
self.patch(config, '_errors', self.errors)
def loadConfig(cls, basedir, filename):
raise config_module.ConfigErrors(['oh noes'])
def __init__(self,
basedir,
configFileName=None,
umask=None,
reactor=None,
config_loader=None):
service.AsyncMultiService.__init__(self)
if reactor is None:
from twisted.internet import reactor
self.reactor = reactor
self.setName("buildmaster")
self.umask = umask
self.basedir = basedir
if basedir is not None: # None is used in tests
assert os.path.isdir(self.basedir)
if config_loader is not None and configFileName is not None:
raise config.ConfigErrors([
"Can't specify both `config_loader` and `configFilename`.",
])
elif config_loader is None:
if configFileName is None:
configFileName = 'master.cfg'
config_loader = config.FileLoader(self.basedir, configFileName)
self.config_loader = config_loader
self.configFileName = configFileName
# flag so we don't try to do fancy things before the master is ready
self._master_initialized = False
# set up child services
self.create_child_services()
# db configured values
self.configured_db_url = None
# configuration / reconfiguration handling
self.config = config.MasterConfig()
self.reconfig_active = False
self.reconfig_requested = False
self.reconfig_notifier = None
# this stores parameters used in the tac file, and is accessed by the
# WebStatus to duplicate those values.
self.log_rotation = LogRotation()
# local cache for this master's object ID
self._object_id = None
# Check environment is sensible
check_functional_environment(self.config)
# figure out local hostname
try:
self.hostname = os.uname()[1] # only on unix
except AttributeError:
self.hostname = socket.getfqdn()
# public attributes
self.name = ("%s:%s" %
(self.hostname, os.path.abspath(self.basedir or '.')))
self.name = self.name.decode('ascii', 'replace')
self.masterid = None
def loadConfig(cls):
raise config_module.ConfigErrors(['oh noes'])
def __init__(self,
name,
password,
max_builds=None,
notify_on_missing=[],
missing_timeout=3600,
properties={},
locks=None,
keepalive_interval=3600):
"""
@param name: botname this machine will supply when it connects
@param password: password this machine will supply when
it connects
@param max_builds: maximum number of simultaneous builds that will
be run concurrently on this buildslave (the
default is None for no limit)
@param properties: properties that will be applied to builds run on
this slave
@type properties: dictionary
@param locks: A list of locks that must be acquired before this slave
can be used
@type locks: dictionary
"""
service.MultiService.__init__(self)
self.slavename = name
self.password = password
# PB registration
self.registration = None
self.registered_port = None
# these are set when the service is started, and unset when it is
# stopped
self.botmaster = None
self.master = None
self.slave_status = SlaveStatus(name)
self.slave = None # a RemoteReference to the Bot, when connected
self.slave_commands = None
self.slavebuilders = {}
self.max_builds = max_builds
self.access = []
if locks:
self.access = locks
self.lock_subscriptions = []
self.properties = Properties()
self.properties.update(properties, "BuildSlave")
self.properties.setProperty("slavename", name, "BuildSlave")
self.lastMessageReceived = 0
if isinstance(notify_on_missing, str):
notify_on_missing = [notify_on_missing]
self.notify_on_missing = notify_on_missing
for i in notify_on_missing:
if not isinstance(i, str):
raise config.ConfigErrors(
['notify_on_missing arg %r is not a string' % (i, )])
self.missing_timeout = missing_timeout
self.missing_timer = None
self.keepalive_interval = keepalive_interval
self.detached_subs = None
self._old_builder_list = None
def test_nonempty(self):
empty = config.ConfigErrors()
full = config.ConfigErrors(['a'])
self.failUnless(not empty)
self.failIf(not full)
def test_error_no_raise(self):
e = config.ConfigErrors()
self.patch(config, "_errors", e)
config.error("message")
self.assertEqual(e.errors, ["message"])
def loadConfig(cls, b, f):
raise config.ConfigErrors(['oh noes'])
