def __init__(self,
path=None,
generator=None,
definitions=None,
options=None,
cmake=DEFAULT_CMAKE,
**kwargs):
self.path = path
self.generator = generator
if not (definitions is None or isinstance(definitions, dict)
or IRenderable.providedBy(definitions)):
config.error(
'definitions must be a dictionary or implement IRenderable')
self.definitions = definitions
if not (options is None or isinstance(options, (list, tuple))
or IRenderable.providedBy(options)):
config.error(
'options must be a list, a tuple or implement IRenderable')
self.options = options
self.cmake = cmake
kwargs = self.setupShellMixin(kwargs, prohibitArgs=['command'])
super(CMake, self).__init__(**kwargs)
def canStartBuild(self, workerforbuilder, buildrequest):
can_start = True
# check whether the locks that the build will acquire can actually be
# acquired
locks = self.config.locks
if IRenderable.providedBy(locks):
# collect properties that would be set for a build if we
# started it now and render locks using it
props = Properties()
Build.setupPropertiesKnownBeforeBuildStarts(props, [buildrequest],
self, workerforbuilder)
locks = yield props.render(locks)
locks = [(self.botmaster.getLockFromLockAccess(access), access)
for access in locks]
if locks:
can_start = Build._canAcquireLocks(locks, workerforbuilder)
if can_start is False:
defer.returnValue(can_start)
if callable(self.config.canStartBuild):
can_start = yield self.config.canStartBuild(self, workerforbuilder,
buildrequest)
defer.returnValue(can_start)
def checkConfig(self,
name,
password,
hyper_host,
hyper_accesskey,
hyper_secretkey,
image,
hyper_size="s3",
masterFQDN=None,
**kwargs):
DockerBaseWorker.checkConfig(self,
name,
password,
image=image,
masterFQDN=masterFQDN,
**kwargs)
if not Hyper:
config.error(
"The python modules 'docker>=2.0' and 'hyper_sh' are needed to use a"
" HyperLatentWorker")
if not IRenderable.providedBy(
hyper_size) and hyper_size not in self.ALLOWED_SIZES:
config.error("Size is not valid {!r} vs {!r}".format(
hyper_size, self.ALLOWED_SIZES))
def add_chromite_step(self, script, params, b_params, legacy=False):
"""Adds a step that runs a chromite command.
Args:
script: Name of the script to run from chromite/bin.
params: space-delimited string of parameters to pass to the cbuildbot
command, or IRenderable.
b_params: An array of StepParameters.
legacy: Use a different directory for some legacy invocations.
"""
script_subdir = 'buildbot' if legacy else 'bin'
cmd = ['%s/%s/%s' % (self.chromite_dir, script_subdir, script)]
if b_params:
cmd.extend(b_params)
if not params:
pass
elif isinstance(params, basestring):
cmd.extend(params.split())
elif IRenderable.providedBy(params):
cmd += [params]
else:
raise TypeError("Unsupported 'params' type: %s" % (type(params),))
self.f_cbuild.addStep(chromium_step.AnnotatedCommand,
command=cmd,
timeout=self.timeout,
name=script,
description=script,
usePTY=False,
**self.step_args)
def canStartWithWorkerForBuilder(self,
workerforbuilder,
buildrequests=None):
locks = self.config.locks
if IRenderable.providedBy(locks):
if buildrequests is None:
raise RuntimeError("buildrequests parameter must be specified "
" when using renderable builder locks. Not "
"specifying buildrequests is deprecated")
# collect properties that would be set for a build if we
# started it now and render locks using it
props = Properties()
Build.setupPropertiesKnownBeforeBuildStarts(
props, buildrequests, self, workerforbuilder)
locks = yield props.render(locks)
# Make sure we don't warn and throw an exception at the same time
if buildrequests is None:
warnings.warn("Not passing corresponding buildrequests to "
"Builder.canStartWithWorkerForBuilder is deprecated")
locks = [(self.botmaster.getLockFromLockAccess(access), access)
for access in locks]
can_start = Build.canStartWithWorkerForBuilder(locks, workerforbuilder)
defer.returnValue(can_start)
def __init__(self, function, *args, **kwargs):
if not callable(function) and not IRenderable.providedBy(function):
config.error("function given to Transform neither callable nor renderable")
self._function = function
self._args = args
self._kwargs = kwargs
def canStartBuild(self, workerforbuilder, buildrequest):
can_start = True
# check whether the locks that the build will acquire can actually be
# acquired
locks = self.config.locks
if IRenderable.providedBy(locks):
# collect properties that would be set for a build if we
# started it now and render locks using it
props = Properties()
Build.setupPropertiesKnownBeforeBuildStarts(
props, [buildrequest], self, workerforbuilder)
locks = yield props.render(locks)
locks = [(self.botmaster.getLockFromLockAccess(access), access)
for access in locks]
if locks:
can_start = Build._canAcquireLocks(locks, workerforbuilder)
if can_start is False:
defer.returnValue(can_start)
if callable(self.config.canStartBuild):
can_start = yield self.config.canStartBuild(
self, workerforbuilder, buildrequest)
defer.returnValue(can_start)
def canStartWithWorkerForBuilder(self, workerforbuilder, buildrequests=None):
locks = self.config.locks
if IRenderable.providedBy(locks):
if buildrequests is None:
raise RuntimeError("buildrequests parameter must be specified "
" when using renderable builder locks. Not "
"specifying buildrequests is deprecated")
# collect properties that would be set for a build if we
# started it now and render locks using it
props = Properties()
Build.setupPropertiesKnownBeforeBuildStarts(props, buildrequests,
self, workerforbuilder)
locks = yield props.render(locks)
# Make sure we don't warn and throw an exception at the same time
if buildrequests is None:
warnings.warn(
"Not passing corresponding buildrequests to "
"Builder.canStartWithWorkerForBuilder is deprecated")
locks = [(self.botmaster.getLockFromLockAccess(access), access)
for access in locks]
can_start = Build.canStartWithWorkerForBuilder(locks, workerforbuilder)
defer.returnValue(can_start)
def add_chromite_step(self, script, params, b_params, legacy=False):
"""Adds a step that runs a chromite command.
Args:
script: Name of the script to run from chromite/bin.
params: space-delimited string of parameters to pass to the cbuildbot
command, or IRenderable.
b_params: An array of StepParameters.
legacy: Use a different directory for some legacy invocations.
"""
script_subdir = 'buildbot' if legacy else 'bin'
cmd = ['%s/%s/%s' % (self.chromite_dir, script_subdir, script)]
if b_params:
cmd.extend(b_params)
if not params:
pass
elif isinstance(params, basestring):
cmd.extend(params.split())
elif IRenderable.providedBy(params):
cmd += [params]
else:
raise TypeError("Unsupported 'params' type: %s" % (type(params), ))
self.f_cbuild.addStep(chromium_step.AnnotatedCommand,
command=cmd,
timeout=self.timeout,
name=script,
description=script,
usePTY=False,
**self.step_args)
def __init__(self, function, *args, **kwargs):
if not callable(function) and not IRenderable.providedBy(function):
config.error("function given to Transform neither callable nor renderable")
self._function = function
self._args = args
self._kwargs = kwargs
def getRenderingFor(self, build):
# Test whether we're going to render
is_present = self.condition(build)
# Choose our inner IRenderer and render it
renderer = (self.present) if is_present else (self.absent)
# Disable 'too many positional arguments' error | pylint: disable=E1121
return IRenderable(renderer).getRenderingFor(build)
def setProperty(self, name, value, source, runtime=False):
name = util.bytes2unicode(name)
if not IRenderable.providedBy(value):
json.dumps(value) # Let the exception propagate ...
source = util.bytes2unicode(source)
self.properties[name] = (value, source)
if runtime:
self.runtime.add(name)
def __init__(self, args=tuple(), command=tuple(), **kwargs):
command, args = command or self.command, args or self.args
if not IRenderable.providedBy(command) and not command:
raise ValueError('No command was provided')
kwargs['command'] = util.FlattenList([command, args])
kwargs = self.setupShellMixin(kwargs)
super().__init__(**kwargs)
def checkConfig(self, stream, topic, formatter, **kwargs):
super().checkConfig(**kwargs)
if not isinstance(stream, str):
config.error('`stream` must be an instance of str')
if not (isinstance(topic, str) or IRenderable.providedBy(topic)):
config.error('`topic` must be a renderable or an instance of str')
if not isinstance(formatter, (type(None), Formatter)):
config.error('`formatter` must be an instance of '
'ursabot.formatters.Formatter')
super().checkConfig(**kwargs)
def __init__(self, path=None, generator=None, definitions=None,
options=None, cmake=DEFAULT_CMAKE, **kwargs):
self.path = path
self.generator = generator
if not (definitions is None or isinstance(definitions, dict)
or IRenderable.providedBy(definitions)):
config.error('definitions must be a dictionary or implement IRenderable')
self.definitions = definitions
if not (options is None or isinstance(options, (list, tuple))
or IRenderable.providedBy(options)):
config.error('options must be a list, a tuple or implement IRenderable')
self.options = options
self.cmake = cmake
kwargs = self.setupShellMixin(kwargs, prohibitArgs=['command'])
super(CMake, self).__init__(**kwargs)
def __init__(self, **kwargs):
self.worker = None
for p in self.__class__.parms:
if p in kwargs:
setattr(self, p, kwargs.pop(p))
if kwargs:
config.error(
f"{self.__class__}.__init__ got unexpected keyword argument(s) "
f"{list(kwargs)}")
self._pendingLogObservers = []
if not isinstance(self.name, str) and not IRenderable.providedBy(
self.name):
config.error(
f"BuildStep name must be a string or a renderable object: "
f"{repr(self.name)}")
check_param_length(self.name, f'Step {self.__class__.__name__} name',
Model.steps.c.name.type.length)
if isinstance(self.description, str):
self.description = [self.description]
if isinstance(self.descriptionDone, str):
self.descriptionDone = [self.descriptionDone]
if isinstance(self.descriptionSuffix, str):
self.descriptionSuffix = [self.descriptionSuffix]
if self.updateBuildSummaryPolicy is None:
# compute default value for updateBuildSummaryPolicy
self.updateBuildSummaryPolicy = [EXCEPTION, RETRY, CANCELLED]
if self.flunkOnFailure or self.haltOnFailure or self.warnOnFailure:
self.updateBuildSummaryPolicy.append(FAILURE)
if self.warnOnWarnings or self.flunkOnWarnings:
self.updateBuildSummaryPolicy.append(WARNINGS)
if self.updateBuildSummaryPolicy is False:
self.updateBuildSummaryPolicy = []
if self.updateBuildSummaryPolicy is True:
self.updateBuildSummaryPolicy = ALL_RESULTS
if not isinstance(self.updateBuildSummaryPolicy, list):
config.error("BuildStep updateBuildSummaryPolicy must be "
"a list of result ids or boolean but it is "
f"{repr(self.updateBuildSummaryPolicy)}")
self._acquiringLocks = []
self.stopped = False
self.master = None
self.statistics = {}
self.logs = {}
self._running = False
self.stepid = None
self.results = None
self._start_unhandled_deferreds = None
self._test_result_submitters = {}
def checkConfig(self, name, password, hyper_host,
hyper_accesskey, hyper_secretkey, image, hyper_size="s3", masterFQDN=None, **kwargs):
DockerBaseWorker.checkConfig(self, name, password, image=image, masterFQDN=masterFQDN, **kwargs)
if not Hyper:
config.error("The python modules 'docker-py>=1.4' and 'hyper_sh' are needed to use a"
" HyperLatentWorker")
if not IRenderable.providedBy(hyper_size) and hyper_size not in self.ALLOWED_SIZES:
config.error("Size is not valid {!r} vs {!r}".format(
hyper_size, self.ALLOWED_SIZES))
def __init__(self, **kwargs):
self.worker = None
self._registerOldWorkerAttr("worker", name="buildslave")
for p in self.__class__.parms:
if p in kwargs:
setattr(self, p, kwargs.pop(p))
if kwargs:
config.error("%s.__init__ got unexpected keyword argument(s) %s" %
(self.__class__, list(kwargs)))
self._pendingLogObservers = []
if not isinstance(self.name, str) and not IRenderable.providedBy(
self.name):
config.error(
"BuildStep name must be a string or a renderable object: "
"%r" % (self.name, ))
if isinstance(self.description, str):
self.description = [self.description]
if isinstance(self.descriptionDone, str):
self.descriptionDone = [self.descriptionDone]
if isinstance(self.descriptionSuffix, str):
self.descriptionSuffix = [self.descriptionSuffix]
if self.updateBuildSummaryPolicy is None: # compute default value for updateBuildSummaryPolicy
self.updateBuildSummaryPolicy = [EXCEPTION, RETRY, CANCELLED]
if self.flunkOnFailure or self.haltOnFailure or self.warnOnFailure:
self.updateBuildSummaryPolicy.append(FAILURE)
if self.warnOnWarnings or self.flunkOnWarnings:
self.updateBuildSummaryPolicy.append(WARNINGS)
if self.updateBuildSummaryPolicy is False:
self.updateBuildSummaryPolicy = []
if self.updateBuildSummaryPolicy is True:
self.updateBuildSummaryPolicy = ALL_RESULTS
if not isinstance(self.updateBuildSummaryPolicy, list):
config.error("BuildStep updateBuildSummaryPolicy must be "
"a list of result ids or boolean but it is %r" %
(self.updateBuildSummaryPolicy, ))
self._acquiringLock = None
self.stopped = False
self.master = None
self.statistics = {}
self.logs = {}
self._running = False
self.stepid = None
self.results = None
self._start_unhandled_deferreds = None
def __init__(self, **kwargs):
self.worker = None
self._registerOldWorkerAttr("worker", name="buildslave")
for p in self.__class__.parms:
if p in kwargs:
setattr(self, p, kwargs.pop(p))
if kwargs:
config.error("%s.__init__ got unexpected keyword argument(s) %s"
% (self.__class__, list(kwargs)))
self._pendingLogObservers = []
if not isinstance(self.name, str) and not IRenderable.providedBy(self.name):
config.error("BuildStep name must be a string or a renderable object: "
"%r" % (self.name,))
if isinstance(self.description, str):
self.description = [self.description]
if isinstance(self.descriptionDone, str):
self.descriptionDone = [self.descriptionDone]
if isinstance(self.descriptionSuffix, str):
self.descriptionSuffix = [self.descriptionSuffix]
if self.updateBuildSummaryPolicy is None: # compute default value for updateBuildSummaryPolicy
self.updateBuildSummaryPolicy = [EXCEPTION, RETRY, CANCELLED]
if self.flunkOnFailure or self.haltOnFailure or self.warnOnFailure:
self.updateBuildSummaryPolicy.append(FAILURE)
if self.warnOnWarnings or self.flunkOnWarnings:
self.updateBuildSummaryPolicy.append(WARNINGS)
if self.updateBuildSummaryPolicy is False:
self.updateBuildSummaryPolicy = []
if self.updateBuildSummaryPolicy is True:
self.updateBuildSummaryPolicy = ALL_RESULTS
if not isinstance(self.updateBuildSummaryPolicy, list):
config.error("BuildStep updateBuildSummaryPolicy must be "
"a list of result ids or boolean but it is %r" %
(self.updateBuildSummaryPolicy,))
self._acquiringLock = None
self.stopped = False
self.master = None
self.statistics = {}
self.logs = {}
self._running = False
self.stepid = None
self.results = None
self._start_unhandled_deferreds = None
def canStartBuild(self, workerforbuilder, buildrequest):
can_start = True
# check whether the locks that the build will acquire can actually be
# acquired
locks = self.config.locks
worker = workerforbuilder.worker
props = None
# don't unnecessarily setup properties for build
def setupPropsIfNeeded(props):
if props is not None:
return props
props = Properties()
Build.setupPropertiesKnownBeforeBuildStarts(
props, [buildrequest], self, workerforbuilder)
return props
if worker.builds_may_be_incompatible:
# Check if the latent worker is actually compatible with the build.
# The instance type of the worker may depend on the properties of
# the build that substantiated it.
props = setupPropsIfNeeded(props)
can_start = yield worker.isCompatibleWithBuild(props)
if not can_start:
return False
if IRenderable.providedBy(locks):
# collect properties that would be set for a build if we
# started it now and render locks using it
props = setupPropsIfNeeded(props)
locks = yield props.render(locks)
locks = yield self.botmaster.getLockFromLockAccesses(
locks, self.config_version)
if locks:
can_start = Build._canAcquireLocks(locks, workerforbuilder)
if can_start is False:
return can_start
if callable(self.config.canStartBuild):
can_start = yield self.config.canStartBuild(
self, workerforbuilder, buildrequest)
return can_start
def canStartBuild(self, workerforbuilder, buildrequest):
can_start = True
# check whether the locks that the build will acquire can actually be
# acquired
locks = self.config.locks
worker = workerforbuilder.worker
props = None
# don't unnecessarily setup properties for build
def setupPropsIfNeeded(props):
if props is not None:
return
props = Properties()
Build.setupPropertiesKnownBeforeBuildStarts(props, [buildrequest],
self, workerforbuilder)
return props
if worker.builds_may_be_incompatible:
# Check if the latent worker is actually compatible with the build.
# The instance type of the worker may depend on the properties of
# the build that substantiated it.
props = setupPropsIfNeeded(props)
can_start = yield worker.isCompatibleWithBuild(props)
if not can_start:
return False
if IRenderable.providedBy(locks):
# collect properties that would be set for a build if we
# started it now and render locks using it
props = setupPropsIfNeeded(props)
locks = yield props.render(locks)
locks = [(self.botmaster.getLockFromLockAccess(access), access)
for access in locks]
if locks:
can_start = Build._canAcquireLocks(locks, workerforbuilder)
if can_start is False:
return can_start
if callable(self.config.canStartBuild):
can_start = yield self.config.canStartBuild(self, workerforbuilder,
buildrequest)
return can_start
def check_locks(self):
# assert that all locks used by the Builds and their Steps are
# uniquely named.
lock_dict = {}
def check_lock(lock):
if isinstance(lock, locks.LockAccess):
lock = lock.lockid
if lock.name in lock_dict:
if lock_dict[lock.name] is not lock:
msg = "Two locks share the same name, '%s'" % lock.name
error(msg)
else:
lock_dict[lock.name] = lock
for b in self.builders:
if b.locks and not IRenderable.providedBy(b.locks):
for lock in b.locks:
check_lock(lock)
def check_locks(self):
# assert that all locks used by the Builds and their Steps are
# uniquely named.
lock_dict = {}
def check_lock(lock):
if isinstance(lock, locks.LockAccess):
lock = lock.lockid
if lock.name in lock_dict:
if lock_dict[lock.name] is not lock:
msg = "Two locks share the same name, '%s'" % lock.name
error(msg)
else:
lock_dict[lock.name] = lock
for b in self.builders:
if b.locks and not IRenderable.providedBy(b.locks):
for lock in b.locks:
check_lock(lock)
def __init__(self,
args=tuple(),
command=tuple(),
as_shell=False,
quote=True,
**kwargs):
command, args = command or self.command, args or self.args
if not IRenderable.providedBy(command) and not command:
raise ValueError('No command was provided')
cmd = util.FlattenList([command, args])
if as_shell:
# runs the command as is without quoting any arguments of it
cmd = util.Transform(' '.join, cmd)
kwargs['command'] = cmd
kwargs = self.setupShellMixin(kwargs)
super().__init__(**kwargs)
def render(self, value):
renderable = IRenderable(value)
return defer.maybeDeferred(renderable.getRenderingFor, self)
def render(self, value):
renderable = IRenderable(value)
return renderable.getRenderingFor(self)
def render(self, value):
"""
Return a variant of value that has any WithProperties objects
substituted. This recurses into Python's compound data types.
"""
return IRenderable(value).render(self)
def render(self, value):
self.props.build = self
return defer.maybeDeferred(
IRenderable(value).getRenderingFor, self.props)
def render(self, value):
return IRenderable(value).getRenderingFor(self)
def render(self, value):
self.props.build = self
return IRenderable(value).getRenderingFor(self.props)
def hasRenderable(l):
for s in l:
if IRenderable.providedBy(s):
return True
return False
def render(self, value):
renderable = IRenderable(value)
return renderable.getRenderingFor(self)