def masterConfig(self, step):
c = {}
from buildbot.config import BuilderConfig
from buildbot.process.factory import BuildFactory
from buildbot.plugins import schedulers
c['schedulers'] = [
schedulers.AnyBranchScheduler(name="sched", builderNames=["testy"])
]
f = BuildFactory()
f.addStep(step)
c['builders'] = [
BuilderConfig(name="testy", workernames=["local1"], factory=f)
]
return c
def test_latent_workers_start_in_parallel(self):
"""
If there are two latent workers configured, and two build
requests for them, both workers will start substantiating
conccurently.
"""
controllers = [
LatentController('local1'),
LatentController('local2'),
]
config_dict = {
'builders': [
BuilderConfig(name="testy",
workernames=["local1", "local2"],
factory=BuildFactory()),
],
'workers': [controller.worker for controller in controllers],
'protocols': {
'null': {}
},
'multiMaster':
True,
}
master = self.getMaster(config_dict)
builder_id = self.successResultOf(
master.data.updates.findBuilderId('testy'))
# Request two builds.
for i in range(2):
bsid, brids = self.successResultOf(
master.data.updates.addBuildset(
waited_for=False,
builderids=[builder_id],
sourcestamps=[
{
'codebase': '',
'repository': '',
'branch': None,
'revision': None,
'project': ''
},
],
))
# Check that both workers were requested to start.
self.assertEqual(controllers[0].started, True)
self.assertEqual(controllers[1].started, True)
def build_triggerer(c, distro, arch, rosdistro, machines, ordered_repos):
f = BuildFactory()
for repos in ordered_repos:
f.addStep(
Trigger(schedulerNames=[
t.replace('_', '-') + '-' + rosdistro + '-' + distro + '-' +
arch + '-debtrigger' for t in repos
],
waitForFinish=True,
alwaysRun=True))
# Add to builders
c['builders'].append(
BuilderConfig(name='build_triggerer' + '_' + rosdistro + '_' + distro +
'_' + arch,
slavenames=machines,
factory=f))
return 'build_triggerer' + '_' + rosdistro + '_' + distro + '_' + arch
def masterConfig():
c = {}
from buildbot.config import BuilderConfig
from buildbot.process.factory import BuildFactory
from buildbot.plugins import steps, schedulers, util
c['schedulers'] = [
schedulers.ForceScheduler(name="force", builderNames=["testy"])
]
f = BuildFactory()
# do a bunch of transfer to exercise the protocol
f.addStep(steps.ShellCommand(command=["echo", util.URLForBuild]))
c['builders'] = [
BuilderConfig(name="testy", workernames=["local1"], factory=f)
]
return c
def masterConfig():
c = {}
from buildbot.config import BuilderConfig
from buildbot.process.factory import BuildFactory
from buildbot.plugins import steps
c['schedulers'] = [
schedulers.AnyBranchScheduler(name="sched1", builderNames=["testy1"]),
schedulers.ForceScheduler(name="sched2", builderNames=["testy2"])
]
f = BuildFactory()
f.addStep(steps.ShellCommand(command='echo hello'))
c['builders'] = [
BuilderConfig(name=name, workernames=["local1"], factory=f)
for name in ['testy1', 'testy2']
]
return c
def windows_builder(buildtype, arch):
if buildtype == "rtdist":
factory = rtdist_factory
elif buildtype == "runtime":
factory = runtime_factory
else:
factory = sdk_factory
if arch == "amd64":
platform = "win_amd64"
else:
platform = "win32"
return BuilderConfig(name='-'.join((buildtype, "windows", arch)),
slavenames=config.windows_slaves,
factory=factory,
properties={"buildtype": buildtype, "arch": arch, "platform": platform})
def test_failed_substantiations_get_requeued(self):
"""
If a latent worker fails to substantiate, the build request becomes unclaimed.
"""
controller = LatentController(self, 'local')
config_dict = {
'builders': [
BuilderConfig(
name="testy",
workernames=["local"],
factory=BuildFactory(),
),
],
'workers': [controller.worker],
'protocols': {
'null': {}
},
# Disable checks about missing scheduler.
'multiMaster':
True,
}
master = self.getMaster(config_dict)
builder_id = self.successResultOf(
master.data.updates.findBuilderId('testy'))
# Trigger a buildrequest
bsid, brids = self.createBuildrequest(master, [builder_id])
unclaimed_build_requests = []
self.successResultOf(
master.mq.startConsuming(
lambda key, request: unclaimed_build_requests.append(request),
('buildrequests', None, 'unclaimed')))
# The worker fails to substantiate.
controller.start_instance(
Failure(TestException("substantiation failed")))
# Flush the errors logged by the failure.
self.flushLoggedErrors(TestException)
# When the substantiation fails, the buildrequest becomes unclaimed.
self.assertEqual(
set(brids),
{req['buildrequestid']
for req in unclaimed_build_requests})
controller.auto_stop(True)
def test_stalled_substantiation_then_timeout_get_requeued(self):
"""
If a latent worker substantiate, but not connect, and then be unsubstantiated,
the build request becomes unclaimed.
"""
controller = LatentController('local')
config_dict = {
'builders': [
BuilderConfig(
name="testy",
workernames=["local"],
factory=BuildFactory(),
),
],
'workers': [controller.worker],
'protocols': {
'null': {}
},
# Disable checks about missing scheduler.
'multiMaster':
True,
}
master = self.getMaster(config_dict)
builder_id = self.successResultOf(
master.data.updates.findBuilderId('testy'))
# Trigger a buildrequest
bsid, brids = self.createBuildrequest(master, [builder_id])
unclaimed_build_requests = []
self.successResultOf(
master.mq.startConsuming(
lambda key, request: unclaimed_build_requests.append(request),
('buildrequests', None, 'unclaimed')))
# We never start the worker, rather timeout it.
master.reactor.advance(controller.worker.missing_timeout)
# Flush the errors logged by the failure.
self.flushLoggedErrors(defer.TimeoutError)
# When the substantiation fails, the buildrequest becomes unclaimed.
self.assertEqual(
set(brids),
set([req['buildrequestid'] for req in unclaimed_build_requests]),
)
controller.auto_stop(True)
def getBaseConfig(self):
return {
'builders': [
BuilderConfig(name="testy",
workernames=["local1", "local2"],
factory=BuildFactory(
[steps.ShellCommand(command='echo hello')])),
],
'workers': [Worker('local' + str(i), 'pass') for i in range(3)],
'schedulers':
[ForceScheduler(name="force", builderNames=["testy"])],
'protocols': {
'null': {}
},
'multiMaster':
True,
}
def create_config_for_step(self, step):
config_dict = {
'builders': [
BuilderConfig(name="builder",
workernames=["worker1"],
factory=BuildFactory([step])
),
],
'workers': [self.createLocalWorker('worker1')],
'protocols': {'null': {}},
# Disable checks about missing scheduler.
'multiMaster': True,
}
yield self.setup_master(config_dict)
builder_id = yield self.master.data.updates.findBuilderId('builder')
return builder_id
def masterConfig():
c = {}
from buildbot.config import BuilderConfig
from buildbot.process.factory import BuildFactory
from buildbot.plugins import steps, schedulers
c['schedulers'] = [
schedulers.AnyBranchScheduler(name="sched", builderNames=["testy"]),
schedulers.ForceScheduler(name="force", builderNames=["testy"])
]
f = BuildFactory()
f.addStep(steps.ShellCommand(command='sleep 100', name='sleep'))
c['builders'] = [
BuilderConfig(name="testy", workernames=["local1"], factory=f)
]
return c
def masterConfig(build_set_summary):
c = {}
from buildbot.config import BuilderConfig
from buildbot.process.factory import BuildFactory
from buildbot.plugins import steps, schedulers, reporters
c['schedulers'] = [
schedulers.AnyBranchScheduler(
name="sched",
builderNames=["testy"])
]
f = BuildFactory()
f.addStep(steps.ShellCommand(command='echo hello'))
c['builders'] = [
BuilderConfig(name="testy",
workernames=["local1"],
factory=f)
]
formatter = MessageFormatter(template='This is a message.')
formatter_worker = MessageFormatterMissingWorker(template='No worker.')
if build_set_summary:
generators_mail = [
BuildSetStatusGenerator(mode='all'),
WorkerMissingGenerator(workers='all'),
]
generators_pushover = [
BuildSetStatusGenerator(mode='all', message_formatter=formatter),
WorkerMissingGenerator(workers=['local1'], message_formatter=formatter_worker),
]
else:
generators_mail = [
BuildStatusGenerator(mode='all'),
WorkerMissingGenerator(workers='all'),
]
generators_pushover = [
BuildStatusGenerator(mode='all', message_formatter=formatter),
WorkerMissingGenerator(workers=['local1'], message_formatter=formatter_worker),
]
c['services'] = [
reporters.MailNotifier("*****@*****.**", generators=generators_mail),
reporters.PushoverNotifier('1234', 'abcd', generators=generators_pushover)
]
return c
def masterConfig():
c = {}
from buildbot.config import BuilderConfig
from buildbot.process.factory import BuildFactory
from buildbot.plugins import steps, schedulers, reporters
c['schedulers'] = [
schedulers.AnyBranchScheduler(
name="sched",
builderNames=["testy"])]
f = BuildFactory()
f.addStep(steps.ShellCommand(command='echo hello'))
c['builders'] = [
BuilderConfig(name="testy",
workernames=["local1"],
factory=f)]
c['services'] = [reporters.MailNotifier("*****@*****.**", mode="all")]
return c
def masterConfig(is_list_mkdir=True, is_list_rmdir=True):
c = {}
from buildbot.config import BuilderConfig
from buildbot.process.factory import BuildFactory
from buildbot.plugins import schedulers
c['schedulers'] = [
schedulers.AnyBranchScheduler(name="sched", builderNames=["testy"])
]
f = BuildFactory()
f.addStep(
TestCompositeMixinStep(is_list_mkdir=is_list_mkdir,
is_list_rmdir=is_list_rmdir))
c['builders'] = [
BuilderConfig(name="testy", workernames=["local1"], factory=f)
]
return c
def as_config(self):
factory = BuildFactory(self.steps)
properties = self._render_properties()
workernames = [w.name for w in self.workers]
return BuilderConfig(name=self.name,
workernames=workernames,
factory=factory,
properties=properties,
description=self.__doc__,
tags=self.tags,
env=self.env,
locks=self.locks,
builddir=str(self.builddir),
workerbuilddir=str(self.workerbuilddir),
nextWorker=self.next_worker,
nextBuild=self.next_build,
collapseRequests=self.collapse_requests,
canStartBuild=self.can_start_build)
def masterConfig():
c = {}
from buildbot.config import BuilderConfig
from buildbot.process.factory import BuildFactory
from buildbot.plugins import steps, schedulers
c['schedulers'] = [
schedulers.Try_Userpass(name="try",
builderNames=["testy"],
port=8031,
userpass=[("alice", "pw1")])
]
f = BuildFactory()
f.addStep(steps.ShellCommand(command='echo hello'))
c['builders'] = [
BuilderConfig(name="testy", workernames=["local1"], factory=f)
]
return c
def addProject(self):
"""
Add project to buildbot_configuration:
"""
self._all_builder_names = []
# Create builders after we resolve all group_builder_names.
for group_name, members in self._raw['groups'].items():
for member_name in members:
builder_name = self._getEnvironmentBuilderName(member_name)
# Don't add the same builder twice.
if builder_name in self._all_builder_names:
continue
# Create new builder for environment.
self._all_builder_names.append(builder_name)
steps = self._getSteps(member_name)
slaves = self._parent.getSlaves(member_name)
builder = BuilderConfig(
name=builder_name,
slavenames=slaves,
category=self._name,
nextSlave=self._nextSlave,
factory=RunStepsFactory(
project=self,
steps=steps,
environment=self._parent.getStepEnvironment(
member_name),
),
)
self._parent.addBuilder(builder)
# self._parent.addNotifications(
# builder=builder_name,
# configuration={
# 'email_success'
# },
# ))
self._addTryBuilders()
self._addGateKeepers()
def test_build_stop_with_retry_during_substantiation(self):
"""
If master is shutting down during latent worker substantiating, the build becomes retry.
"""
controller = LatentController('local')
config_dict = {
'builders': [
BuilderConfig(name="testy",
workernames=["local"],
factory=BuildFactory(),
),
],
'workers': [controller.worker],
'protocols': {'null': {}},
# Disable checks about missing scheduler.
'multiMaster': True,
}
master = self.getMaster(config_dict)
builder = master.botmaster.builders['testy']
builder_id = self.successResultOf(builder.getBuilderId())
# Trigger a buildrequest
_, brids = self.createBuildrequest(master, [builder_id])
unclaimed_build_requests = []
self.successResultOf(master.mq.startConsuming(
lambda key, request: unclaimed_build_requests.append(request),
('buildrequests', None, 'unclaimed')))
# Stop the build
build = builder.getBuild(0)
build.stopBuild('no reason', results=RETRY)
# Indicate that the worker can't start an instance.
controller.start_instance(False)
dbdict = yield master.db.builds.getBuildByNumber(builder_id, 1)
self.assertEqual(RETRY, dbdict['results'])
self.assertEqual(
set(brids),
{req['buildrequestid'] for req in unclaimed_build_requests}
)
controller.auto_stop(True)
self.flushLoggedErrors(LatentWorkerFailedToSubstantiate)
def make_builders(self,
name,
factory,
build_category=None,
build_for=None,
build_requires=(),
next_slave=None):
"""Produce the builders for the given factory.
:param name: base name for the builders.
:param factory: :class:`BuildFactory` instance
:param build_requires: list of capability requirements that the
buildslave must match to run a builder
from the factory.
:param build_for: a dict whose keys are capability names and values are
corresponding :class:`VersionFilter` instances.
:param build_category: forwarded to :class:`BuilderConfig`
instantiation
:param next_slave: forwarded to :class:`BuilderConfig` instantiation as
``nextSlave``.
"""
slavenames = self.filter_slaves_by_requires(build_requires)
if not slavenames:
# buildbot does not allow builders with empty list of slaves
return ()
base_conf = dict(name=name,
category=build_category,
factory=factory,
nextSlave=next_slave,
slavenames=list(slavenames))
# forward requirement in the build properties
if build_requires:
base_conf['properties'] = dict(
build_requires=[str(req) for req in build_requires])
preconfs = [base_conf]
for cap_name, cap_vf in factory.build_for.items():
preconfs = self.dispatch_builders_by_capability(
preconfs, cap_name, cap_vf)
return [BuilderConfig(**conf) for conf in preconfs]
def get_config_single_step(self, step):
c = {}
from buildbot.config import BuilderConfig
from buildbot.process.factory import BuildFactory
from buildbot.plugins import steps, schedulers
c['schedulers'] = [
schedulers.ForceScheduler(name="force", builderNames=["testy"])
]
f = BuildFactory()
f.addStep(
steps.FileUpload(workersrc="dir/noexist_path",
masterdest="master_dest"))
c['builders'] = [
BuilderConfig(name="testy", workernames=["local1"], factory=f)
]
return c
def macosx_builder(osxver):
if osxver in ('10.6', '10.7', '10.8'):
workernames = config.macosx_10_6_workers
buildsteps = build_steps_10_6
elif osxver.startswith('11.'):
workernames = config.macosx_11_0_workers
buildsteps = build_steps_11_0
else:
workernames = config.macosx_10_9_workers
buildsteps = build_steps_10_9
factory = BuildFactory()
for step in buildsteps:
factory.addStep(step)
return BuilderConfig(name='macosx' + ('-' + osxver if osxver else ''),
workernames=workernames,
factory=factory,
properties={"osxtarget": osxver, "buildtype": "sdk"})
def create_master_config(self):
self.num_reconfig += 1
from buildbot.config import BuilderConfig
from buildbot.process.factory import BuildFactory
from buildbot.steps.shell import ShellCommand
from buildbot.util.service import BuildbotService
class MyShellCommand(ShellCommand):
def getResultSummary(self):
service = self.master.service_manager.namedServices[
'myService']
return dict(step="num reconfig: %d" % (service.num_reconfig, ))
class MyService(BuildbotService):
name = "myService"
def reconfigService(self, num_reconfig):
self.num_reconfig = num_reconfig
return defer.succeed(None)
config_dict = {
'builders': [
BuilderConfig(name="builder",
workernames=["worker1"],
factory=BuildFactory(
[MyShellCommand(command='echo hei')])),
],
'workers': [self.createLocalWorker('worker1')],
'protocols': {
'null': {}
},
# Disable checks about missing scheduler.
'multiMaster':
True,
'db_url':
'sqlite://', # we need to make sure reconfiguration uses the same URL
'services': [MyService(num_reconfig=self.num_reconfig)]
}
if self.num_reconfig == 3:
config_dict['services'].append(
MyService(name="myService2", num_reconfig=self.num_reconfig))
return config_dict
def generate_builder(target: TestTarget) -> BuilderConfig:
factory = util.BuildFactory()
factory.addStep(download_step)
factory.addStep(
GenerateStartMovieCommands(
directory=target.directory,
game_id=target.game_id,
debugflags=target.debugflags,
name=f"Generate commands: {target.builder_name}",
command=[
"cat",
os.path.join(target.directory, "test_scripts.txt")
],
haltOnFailure=True,
**default_step_kwargs,
))
return BuilderConfig(name=target.builder_name,
workernames=["director-worker"],
factory=factory)
def masterConfig(usePTY):
c = {}
from buildbot.config import BuilderConfig
from buildbot.process.factory import BuildFactory
from buildbot.plugins import steps, schedulers
c['schedulers'] = [
schedulers.ForceScheduler(name="force", builderNames=["testy"])
]
f = BuildFactory()
f.addStep(
steps.ShellCommand(
command=
'if [ -t 1 ] ; then echo in a terminal; else echo "not a terminal"; fi',
usePTY=usePTY))
c['builders'] = [
BuilderConfig(name="testy", workernames=["local1"], factory=f)
]
return c
def macosx_builder(buildtype, osxver):
if buildtype == "sdk":
name = '-'.join((buildtype, "macosx" + osxver))
else:
name = '-'.join((buildtype, "macosx"))
if buildtype == "rtdist":
factory = rtdist_factory
elif buildtype == "runtime":
factory = runtime_factory
else:
factory = sdk_factory
return BuilderConfig(name=name,
slavenames=config.macosx_slaves,
factory=factory,
properties={
"osxtarget": osxver,
"buildtype": buildtype
})
def masterConfig(use_interpolation):
c = {}
from buildbot.config import BuilderConfig
from buildbot.process.factory import BuildFactory
from buildbot.plugins import schedulers, steps, util
c['services'] = [
FakeSecretReporter('http://example.com/hook',
auth=('user', Interpolate('%(secret:httppasswd)s')))
]
c['schedulers'] = [
schedulers.ForceScheduler(name="force", builderNames=["testy"])
]
c['secretsProviders'] = [
FakeSecretStorage(
secretdict={
"foo": "secretvalue",
"something": "more",
'httppasswd': 'myhttppasswd'
})
]
f = BuildFactory()
if use_interpolation:
if os.name == "posix":
# on posix we can also check whether the password was passed to the command
command = Interpolate(
'echo %(secret:foo)s | ' +
'sed "s/secretvalue/The password was there/"')
else:
command = Interpolate('echo %(secret:foo)s')
else:
command = ['echo', util.Secret('foo')]
f.addStep(steps.ShellCommand(command=command))
c['builders'] = [
BuilderConfig(name="testy", workernames=["local1"], factory=f)
]
return c
def addSimpleBuilder(self, name, buildername, category, repourl,
builderconfig, sos, sbranch, bparams):
"""Private.
Add a single builder on the given OS type for the given repo and branch.
"""
factory = BuildFactory()
factory.addStep(
Git(repourl=repourl,
mode='full',
submodules=True,
method='copy',
branch=sbranch,
getDescription={'tags': True}))
if "tag" in builderconfig and not (builderconfig["tag"] is None):
stag = builderconfig["tag"].encode('ascii', 'ignore')
factory.addStep(
ShellCommand(command=['git', 'checkout', stag],
workdir="build",
description="checkout tag"))
# Delegate all knowlege of how to build to a script called buildshim in the project
# directory. Call this script in nine standardize steps. It should ignore steps that it doesn't need.
# Pass the step name as the first arg, and if params was given in the json for this builder, pass that as the
# second arg.
for step in [
"patch", "install_deps", "configure", "compile", "check",
"package", "upload", "compile_extra", "uninstall_deps"
]:
factory.addStep(
ShellCommand(command=["./buildshim", step, bparams],
description=step,
haltOnFailure=True))
self['builders'].append(
BuilderConfig(name=buildername,
slavenames=self._os2slaves[sos],
factory=factory,
category=category))
return factory
def test_worker_accepts_builds_after_failure(self):
"""
If a latent worker fails to substantiate, the worker is still able to accept jobs.
"""
controller = LatentController('local')
config_dict = {
'builders': [
BuilderConfig(name="testy",
workernames=["local"],
factory=BuildFactory(),
),
],
'workers': [controller.worker],
'protocols': {'null': {}},
# Disable checks about missing scheduler.
'multiMaster': True,
}
master = self.getMaster(config_dict)
builder_id = self.successResultOf(
master.data.updates.findBuilderId('testy'))
# Trigger a buildrequest
bsid, brids = self.createBuildrequest(master, [builder_id])
unclaimed_build_requests = []
self.successResultOf(master.mq.startConsuming(
lambda key, request: unclaimed_build_requests.append(request),
('buildrequests', None, 'unclaimed')))
# The worker fails to substantiate.
controller.start_instance(
Failure(TestException("substantiation failed")))
# Flush the errors logged by the failure.
self.flushLoggedErrors(TestException)
# If the worker started again after the failure, then the retry logic will have
# already kicked in to start a new build on this (the only) worker. We check that
# a new instance was requested, which indicates that the worker
# accepted the build.
self.assertEqual(controller.started, True)
controller.auto_stop(True)
def masterConfig():
c = {}
from buildbot.config import BuilderConfig
from buildbot.process.factory import BuildFactory
from buildbot.plugins import schedulers, steps
c['schedulers'] = [
schedulers.ForceScheduler(
name="force",
builderNames=["testy"])]
c['secretsProviders'] = [FakeSecretStorage(
secretdict={"foo": "bar", "something": "more"})]
f = BuildFactory()
f.addStep(steps.ShellCommand(command=Interpolate('echo %(secrets:foo)s')))
c['builders'] = [
BuilderConfig(name="testy",
workernames=["local1"],
factory=f)]
return c
def __init__(self, name, workers):
self.name = utils.name(name)
self.workers = workers
self.build = BuildFactory()
BuildmasterConfig['builders'].append(BuilderConfig(
name=self.name,
factory=self.build,
slavenames=self.workers))
BuildmasterConfig['schedulers'].append(ForceScheduler(
name=scheduler_name(self, 'force'),
builderNames=[self.name],
branch=FixedParameter(name="reason", default=""),
reason=FixedParameter(name="reason", default="Forced build"),
revision=FixedParameter(name="revision", default=""),
repository=FixedParameter(name="repository", default=""),
project=FixedParameter(name="project", default=""),
properties=[],
username=FixedParameter(name="username", default="WebUI")))
def __init__(self, name, branch, slavenames=None, factory=None, category=None):
BuilderConfig.__init__(self, name=name, slavenames=slavenames,
factory=factory, category=category)
self.branch = branch
def __init__(self, name, **kwargs):
BuilderConfig.__init__(self, name, builddir = "work/builders/" + name, slavenames = get_slavenames(), **kwargs)
