def make_lint():
f = BuildFactory()
f.addStep(
GitNoBranch(repourl="https://github.com/dolphin-emu/dolphin.git",
progress=True,
mode="incremental"))
# Make sure the baserev exists locally. GitHub's "baserev" property is
# the current base branch head, which isn't guaranteed to exist in the PR
# branch (e.g. if it hasn't been rebased).
f.addStep(ShellCommand(command=['git', 'fetch', 'origin']))
f.addStep(
ShellCommand(
command=['Tools/lint.sh',
WithProperties("%s...", "baserev")],
logEnviron=False,
description="lint",
descriptionDone="lint",
haltOnFailure=False))
return f
def create_master_config(self):
self.num_reconfig += 1
from buildbot.config import BuilderConfig
from buildbot.process.factory import BuildFactory
from buildbot.steps.shell import ShellCommand
from buildbot.util.service import BuildbotService
class MyShellCommand(ShellCommand):
def getResultSummary(self):
service = self.master.service_manager.namedServices[
'myService']
return dict(step="num reconfig: %d" % (service.num_reconfig, ))
class MyService(BuildbotService):
name = "myService"
def reconfigService(self, num_reconfig):
self.num_reconfig = num_reconfig
return defer.succeed(None)
config_dict = {
'builders': [
BuilderConfig(name="builder",
workernames=["worker1"],
factory=BuildFactory(
[MyShellCommand(command='echo hei')])),
],
'workers': [self.createLocalWorker('worker1')],
'protocols': {
'null': {}
},
# Disable checks about missing scheduler.
'multiMaster':
True,
'db_url':
'sqlite://', # we need to make sure reconfiguration uses the same URL
'services': [MyService(num_reconfig=self.num_reconfig)]
}
if self.num_reconfig == 3:
config_dict['services'].append(
MyService(name="myService2", num_reconfig=self.num_reconfig))
return config_dict
def get_config_single_step(self, step):
c = {}
from buildbot.config import BuilderConfig
from buildbot.process.factory import BuildFactory
from buildbot.plugins import steps, schedulers
c['schedulers'] = [
schedulers.ForceScheduler(name="force", builderNames=["testy"])
]
f = BuildFactory()
f.addStep(
steps.FileUpload(workersrc="dir/noexist_path",
masterdest="master_dest"))
c['builders'] = [
BuilderConfig(name="testy", workernames=["local1"], factory=f)
]
return c
def create_config_for_step(self, step):
config_dict = {
'builders': [
BuilderConfig(name="builder",
workernames=["worker1"],
factory=BuildFactory([step])),
],
'workers': [self.createLocalWorker('worker1')],
'protocols': {
'null': {}
},
# Disable checks about missing scheduler.
'multiMaster':
True,
}
yield self.getMaster(config_dict)
builder_id = yield self.master.data.updates.findBuilderId('builder')
return builder_id
def testCommand(self):
f = BuildFactory()
f.addStep(SetProperty(command=["echo", "value"], property="propname"))
f.addStep(
ShellCommand(
command=["echo", WithProperties("%(propname)s")]))
ss = SourceStamp()
req = FakeBuildRequest("Testing", {ss.repository: ss}, None)
b = f.newBuild([req])
b.build_status = FakeBuildStatus()
b.slavebuilder = FakeSlaveBuilder()
b.builder = FakeBuilder()
b.setProperty = setProperty
# This shouldn't raise an exception
b.setupBuild(None)
def testCommand(self):
f = BuildFactory()
f.addStep(SetPropertyFromCommand(command=["echo", "value"], property="propname"))
f.addStep(ShellCommand(command=["echo", WithProperties("%(propname)s")]))
ss = mock.Mock(name="sourcestamp")
ss.repository = 'repo'
ss.changes = []
ss.patch = ss.patch_info = None
req = FakeBuildRequest("Testing", {ss.repository: ss}, None)
b = f.newBuild([req])
b.master = mock.Mock(name='master')
b.build_status = FakeBuildStatus()
b.workerforbuilder = FakeWorkerForBuilder()
# This shouldn't raise an exception
b.setupBuild(None)
def test_builder_as_config():
class Test(Builder):
env = {'A': 'a'}
tags = ['test']
steps = []
properties = {'A': 'a'}
with pytest.raises(TypeError):
Test(name='test')
workers = [LocalWorker('worker_a'), LocalWorker('worker_b')]
conf = Test(name='test', workers=workers).as_config()
assert isinstance(conf, BuilderConfig)
assert conf.name == 'test'
assert conf.workernames == ['worker_a', 'worker_b']
assert conf.factory == BuildFactory([])
assert conf.tags == ['test']
assert conf.env == {'A': 'a'}
assert conf.properties == {'A': 'a'}
def masterConfig():
c = {}
from buildbot.config import BuilderConfig
from buildbot.process.factory import BuildFactory
from buildbot.plugins import steps, schedulers, util
c['schedulers'] = [
schedulers.ForceScheduler(
name="force",
builderNames=["testy"])]
f = BuildFactory()
# do a bunch of transfer to exercise the protocol
f.addStep(steps.ShellCommand(command=["echo", util.URLForBuild]))
c['builders'] = [
BuilderConfig(name="testy",
workernames=["local1"],
factory=f)
]
return c
def masterConfig(usePTY):
c = {}
from buildbot.config import BuilderConfig
from buildbot.process.factory import BuildFactory
from buildbot.plugins import steps, schedulers
c['schedulers'] = [
schedulers.ForceScheduler(name="force", builderNames=["testy"])
]
f = BuildFactory()
f.addStep(
steps.ShellCommand(
command=
'if [ -t 1 ] ; then echo in a terminal; else echo "not a terminal"; fi',
usePTY=usePTY))
c['builders'] = [
BuilderConfig(name="testy", workernames=["local1"], factory=f)
]
return c
def masterConfig(use_interpolation):
c = {}
from buildbot.config import BuilderConfig
from buildbot.process.factory import BuildFactory
from buildbot.plugins import schedulers, steps, util
c['services'] = [
FakeSecretReporter('http://example.com/hook',
auth=('user', Interpolate('%(secret:httppasswd)s')))
]
c['schedulers'] = [
schedulers.ForceScheduler(name="force", builderNames=["testy"])
]
c['secretsProviders'] = [
FakeSecretStorage(
secretdict={
"foo": "secretvalue",
"something": "more",
'httppasswd': 'myhttppasswd'
})
]
f = BuildFactory()
if use_interpolation:
if os.name == "posix":
# on posix we can also check whether the password was passed to the command
command = Interpolate(
'echo %(secret:foo)s | ' +
'sed "s/secretvalue/The password was there/"')
else:
command = Interpolate('echo %(secret:foo)s')
else:
command = ['echo', util.Secret('foo')]
f.addStep(steps.ShellCommand(command=command))
c['builders'] = [
BuilderConfig(name="testy", workernames=["local1"], factory=f)
]
return c
def test_stalled_substantiation_then_timeout_get_requeued(self):
"""
If a latent worker substantiate, but not connect, and then be unsubstantiated,
the build request becomes unclaimed.
"""
controller = LatentController('local')
config_dict = {
'builders': [
BuilderConfig(name="testy",
workernames=["local"],
factory=BuildFactory(),
),
],
'workers': [controller.worker],
'protocols': {'null': {}},
# Disable checks about missing scheduler.
'multiMaster': True,
}
master = self.getMaster(config_dict)
builder_id = self.successResultOf(
master.data.updates.findBuilderId('testy'))
# Trigger a buildrequest
bsid, brids = self.createBuildrequest(master, [builder_id])
unclaimed_build_requests = []
self.successResultOf(master.mq.startConsuming(
lambda key, request: unclaimed_build_requests.append(request),
('buildrequests', None, 'unclaimed')))
# We never start the worker, rather timeout it.
master.reactor.advance(controller.worker.missing_timeout)
# Flush the errors logged by the failure.
self.flushLoggedErrors(defer.TimeoutError)
# When the substantiation fails, the buildrequest becomes unclaimed.
self.assertEqual(
set(brids),
{req['buildrequestid'] for req in unclaimed_build_requests}
)
controller.auto_stop(True)
def test_worker_accepts_builds_after_failure(self):
"""
If a latent worker fails to substantiate, the worker is still able to accept jobs.
"""
controller = LatentController('local')
config_dict = {
'builders': [
BuilderConfig(name="testy",
workernames=["local"],
factory=BuildFactory(),
),
],
'workers': [controller.worker],
'protocols': {'null': {}},
# Disable checks about missing scheduler.
'multiMaster': True,
}
master = self.getMaster(config_dict)
builder_id = self.successResultOf(
master.data.updates.findBuilderId('testy'))
# Trigger a buildrequest
bsid, brids = self.createBuildrequest(master, [builder_id])
unclaimed_build_requests = []
self.successResultOf(master.mq.startConsuming(
lambda key, request: unclaimed_build_requests.append(request),
('buildrequests', None, 'unclaimed')))
# The worker fails to substantiate.
controller.start_instance(
Failure(TestException("substantiation failed")))
# Flush the errors logged by the failure.
self.flushLoggedErrors(TestException)
# If the worker started again after the failure, then the retry logic will have
# already kicked in to start a new build on this (the only) worker. We check that
# a new instance was requested, which indicates that the worker
# accepted the build.
self.assertEqual(controller.started, True)
controller.auto_stop(True)
def make_dolphin_freebsd_build(mode="normal"):
f = BuildFactory()
mode = mode.split(",")
f.addStep(
GitNoBranch(repourl="https://github.com/dolphin-emu/dolphin.git",
progress=True,
mode="incremental"))
f.addStep(
ShellCommand(command=["mkdir", "-p", "build"],
logEnviron=False,
description="mkbuilddir",
descriptionDone="mkbuilddir"))
cmake_cmd = ["cmake", "..", "-GNinja"]
cmake_cmd.append("-DDISTRIBUTOR=dolphin-emu.org")
f.addStep(
ShellCommand(command=cmake_cmd,
workdir="build/build",
description="configuring",
descriptionDone="configure",
haltOnFailure=True,
env={"WX_CONFIG": "wxgtk2u-3.0-config"}))
f.addStep(
Compile(command=["ninja"],
workdir="build/build",
description="building",
descriptionDone="build",
haltOnFailure=True))
f.addStep(
Test(command=["ninja", "unittests"],
workdir="build/build",
description="testing",
descriptionDone="test",
haltOnFailure=True))
return f
def test_failed_substantiations_get_requeued(self):
"""
If a latent worker fails to substantiate, the build request becomes unclaimed.
"""
controller = LatentController('local')
config_dict = {
'builders': [
BuilderConfig(name="testy",
workernames=["local"],
factory=BuildFactory(),
),
],
'workers': [controller.worker],
'protocols': {'null': {}},
# Disable checks about missing scheduler.
'multiMaster': True,
}
master = self.getMaster(config_dict)
builder_id = self.successResultOf(
master.data.updates.findBuilderId('testy'))
# Trigger a buildrequest
bsid, brids = self.createBuildrequest(master, [builder_id])
unclaimed_build_requests = []
self.successResultOf(master.mq.startConsuming(
lambda key, request: unclaimed_build_requests.append(request),
('buildrequests', None, 'unclaimed')))
# The worker fails to substantiate.
controller.start_instance(
Failure(TestException("substantiation failed")))
# Flush the errors logged by the failure.
self.flushLoggedErrors(TestException)
# When the substantiation fails, the buildrequest becomes unclaimed.
self.assertEqual(
set(brids),
set([req['buildrequestid'] for req in unclaimed_build_requests]),
)
controller.auto_stop(True)
def addSimpleBuilder(self, name, buildername, category, repourl,
builderconfig, sos, sbranch, bparams):
"""Private.
Add a single builder on the given OS type for the given repo and branch.
"""
factory = BuildFactory()
factory.addStep(
Git(repourl=repourl,
mode='full',
submodules=True,
method='copy',
branch=sbranch,
getDescription={'tags': True}))
if "tag" in builderconfig and not (builderconfig["tag"] is None):
stag = builderconfig["tag"].encode('ascii', 'ignore')
factory.addStep(
ShellCommand(command=['git', 'checkout', stag],
workdir="build",
description="checkout tag"))
# Delegate all knowlege of how to build to a script called buildshim in the project
# directory. Call this script in nine standardize steps. It should ignore steps that it doesn't need.
# Pass the step name as the first arg, and if params was given in the json for this builder, pass that as the
# second arg.
for step in [
"patch", "install_deps", "configure", "compile", "check",
"package", "upload", "compile_extra", "uninstall_deps"
]:
factory.addStep(
ShellCommand(command=["./buildshim", step, bparams],
description=step,
haltOnFailure=True))
self['builders'].append(
BuilderConfig(name=buildername,
slavenames=self._os2slaves[sos],
factory=factory,
category=category))
return factory
def create_factory():
factory = BuildFactory()
factory.addSteps(common.initTargetProperty())
factory.addSteps(common.initNameProperty())
factory.addStep(common.determineBuildId())
buildProperties = COMMON_PROPERTIES.copy()
buildProperties.update({
'virtual_builder_name':
util.Interpolate('Build for %(prop:box)s'),
})
factory.addStep(
steps.Trigger(
name="Call the 'build' scheduler",
schedulerNames=['build'],
waitForFinish=True,
set_properties=buildProperties,
))
runTestProperties = COMMON_PROPERTIES.copy()
runTestProperties.update({
"appendTestRunId":
util.Property("appendTestRunId"),
"backend_ssl":
util.Property("backend_ssl"),
"buildHosts": ["bb-host"],
"buildId":
util.Property("buildId"),
"test_branch":
util.Property('branch'),
"use_callgrind":
util.Property("use_callgrind"),
"use_valgrind":
util.Property("use_valgrind"),
})
factory.addStep(
ParallelRunTestTrigger(
name="Call the 'run_test' for each test set",
schedulerNames=['run_test'],
waitForFinish=True,
set_properties=runTestProperties,
))
return factory
def sync_s3_debians(c, machines):
with open(os.path.dirname(os.path.realpath(__file__)) +
"/spec.yaml") as file:
spec_list = yaml.full_load(file)
f = BuildFactory()
if spec_list["sync_s3"]:
f.addStep(
ShellCommand(
name='s3-syncing',
command=[
's3cmd', '--acl-public', '--delete-removed', '--verbose',
'sync', spec_list["local_repo_path"],
's3://{s3_bucket}'.format(s3_bucket=spec_list["s3_bucket"])
]))
# Add to builders
c['builders'].append(
BuilderConfig(name='sync_s3_debians', slavenames=machines, factory=f))
# return name of builder created
return 'sync_s3_debians'
def masterConfig():
c = {}
from buildbot.config import BuilderConfig
from buildbot.process.factory import BuildFactory
from buildbot.plugins import schedulers
c['schedulers'] = [
schedulers.AnyBranchScheduler(
name="sched",
builderNames=["testy"]),
schedulers.ForceScheduler(
name="force",
builderNames=["testy"])]
f = BuildFactory()
f.addStep(DisconnectingStep())
c['builders'] = [
BuilderConfig(name="testy",
workernames=["local1"],
factory=f)]
return c
def masterConfig():
c = {}
from buildbot.config import BuilderConfig
from buildbot.process.factory import BuildFactory
from buildbot.plugins import steps, schedulers, util
c['schedulers'] = [
schedulers.ForceScheduler(
name="force",
codebases=[util.CodebaseParameter("foo", patch=util.PatchParameter())],
builderNames=["testy"])]
f = BuildFactory()
f.addStep(MySource(codebase='foo'))
# if the patch was applied correctly, then make will work!
f.addStep(steps.ShellCommand(command=["make"]))
c['builders'] = [
BuilderConfig(name="testy",
workernames=["local1"],
factory=f)]
return c
def masterConfig():
c = {}
from buildbot.config import BuilderConfig
from buildbot.process.factory import BuildFactory
from buildbot.plugins import schedulers, steps
c['schedulers'] = [
schedulers.ForceScheduler(
name="force",
builderNames=["testy"])]
c['secretsProviders'] = [FakeSecretStorage(
secretdict={"foo": "bar", "something": "more"})]
f = BuildFactory()
f.addStep(steps.ShellCommand(command=Interpolate('echo %(secrets:foo)s')))
c['builders'] = [
BuilderConfig(name="testy",
workernames=["local1"],
factory=f)]
return c
def create_single_worker_config(self, controller_kwargs=None):
if not controller_kwargs:
controller_kwargs = {}
controller = LatentController(self, 'local', **controller_kwargs)
config_dict = {
'builders': [
BuilderConfig(name="testy",
workernames=["local"],
factory=BuildFactory(),
),
],
'workers': [controller.worker],
'protocols': {'null': {}},
# Disable checks about missing scheduler.
'multiMaster': True,
}
master = yield self.getMaster(config_dict)
builder_id = yield master.data.updates.findBuilderId('testy')
return controller, master, builder_id
def __init__(self, name, workers):
self.name = utils.name(name)
self.workers = workers
self.build = BuildFactory()
BuildmasterConfig['builders'].append(BuilderConfig(
name=self.name,
factory=self.build,
slavenames=self.workers))
BuildmasterConfig['schedulers'].append(ForceScheduler(
name=scheduler_name(self, 'force'),
builderNames=[self.name],
branch=FixedParameter(name="reason", default=""),
reason=FixedParameter(name="reason", default="Forced build"),
revision=FixedParameter(name="revision", default=""),
repository=FixedParameter(name="repository", default=""),
project=FixedParameter(name="project", default=""),
properties=[],
username=FixedParameter(name="username", default="WebUI")))
def test_build_stop_with_cancelled_during_substantiation(self):
"""
If a build is stopping during latent worker substantiating, the build becomes cancelled
"""
controller = LatentController(self, 'local')
config_dict = {
'builders': [
BuilderConfig(
name="testy",
workernames=["local"],
factory=BuildFactory(),
),
],
'workers': [controller.worker],
'protocols': {
'null': {}
},
# Disable checks about missing scheduler.
'multiMaster':
True,
}
master = self.getMaster(config_dict)
builder = master.botmaster.builders['testy']
builder_id = self.successResultOf(builder.getBuilderId())
# Trigger a buildrequest
self.createBuildrequest(master, [builder_id])
# Stop the build
build = builder.getBuild(0)
build.stopBuild('no reason', results=CANCELLED)
# Indicate that the worker can't start an instance.
controller.start_instance(False)
dbdict = self.successResultOf(
master.db.builds.getBuildByNumber(builder_id, 1))
self.assertEqual(CANCELLED, dbdict['results'])
controller.auto_stop(True)
self.flushLoggedErrors(LatentWorkerFailedToSubstantiate)
def masterConfig():
global num_reconfig
num_reconfig += 1
c = {}
from buildbot.config import BuilderConfig
from buildbot.process.factory import BuildFactory
from buildbot.schedulers.forcesched import ForceScheduler
from buildbot.steps.shell import ShellCommand
from buildbot.util.service import BuildbotService
class MyShellCommand(ShellCommand):
def getResultSummary(self):
service = self.master.service_manager.namedServices['myService']
return dict(step=u"num reconfig: %d" % (service.num_reconfig,))
class MyService(BuildbotService):
name = "myService"
def reconfigService(self, num_reconfig):
self.num_reconfig = num_reconfig
return defer.succeed(None)
c['schedulers'] = [
ForceScheduler(
name="force",
builderNames=["testy"])]
f = BuildFactory()
f.addStep(MyShellCommand(command='echo hei'))
c['builders'] = [
BuilderConfig(name="testy",
workernames=["local1"],
factory=f)]
c['services'] = [MyService(num_reconfig=num_reconfig)]
if num_reconfig == 3:
c['services'].append(
MyService(name="myService2", num_reconfig=num_reconfig))
return c
def test_failed_substantiations_get_exception(self):
"""
If a latent worker fails to substantiate, the result is an exception.
"""
controller = LatentController(self, 'local')
config_dict = {
'builders': [
BuilderConfig(
name="testy",
workernames=["local"],
factory=BuildFactory(),
),
],
'workers': [controller.worker],
'protocols': {
'null': {}
},
# Disable checks about missing scheduler.
'multiMaster':
True,
}
master = self.getMaster(config_dict)
builder_id = self.successResultOf(
master.data.updates.findBuilderId('testy'))
# Trigger a buildrequest
self.createBuildrequest(master, [builder_id])
# The worker fails to substantiate.
controller.start_instance(
Failure(LatentWorkerCannotSubstantiate("substantiation failed")))
# Flush the errors logged by the failure.
self.flushLoggedErrors(LatentWorkerCannotSubstantiate)
dbdict = self.successResultOf(
master.db.builds.getBuildByNumber(builder_id, 1))
# When the substantiation fails, the result is an exception.
self.assertEqual(EXCEPTION, dbdict['results'])
controller.auto_stop(True)
def masterConfig():
c = {}
from buildbot.config import BuilderConfig
from buildbot.process.factory import BuildFactory
from buildbot.plugins import schedulers
c['schedulers'] = [
schedulers.ForceScheduler(name="force", builderNames=["testy"])
]
c['secretsProviders'] = [
HashiCorpVaultSecretProvider(vaultToken='my_vaulttoken',
vaultServer="http://localhost:8200")
]
f = BuildFactory()
f.addStep(ShellCommand(command=[Interpolate('echo %(secret:key)s')]))
c['builders'] = [
BuilderConfig(name="testy", workernames=["local1"], factory=f)
]
return c
def setup_inplace(self):
self.builders.clear()
self.schedulers.clear()
builder_name = self.DUMMY_NAME
trigger_name = self.DUMMY_TRIGGER
worker_names = self.inplace_workers.names
self.builders.named_set(
BuilderConfig(name=builder_name,
workernames=worker_names,
factory=BuildFactory()))
self.schedulers.named_set(
ForceScheduler(name=trigger_name, builderNames=[builder_name]))
for project in self.projects:
builder_name = "%s_Builder" % project.name
trigger_name = "Force_%s_Build" % project.name
builder_factory = InplaceBuildFactory(self, project)
self.builders.named_set(
BuilderConfig(name=builder_name,
workernames=worker_names,
factory=builder_factory))
self.schedulers.named_set(
ForceScheduler(name=trigger_name, builderNames=[builder_name]))
def configure(self, config_dict):
steps = []
if self.logHorizon is not None:
steps.append(LogChunksJanitor(logHorizon=self.logHorizon))
if self.build_data_horizon is not None:
steps.append(
BuildDataJanitor(build_data_horizon=self.build_data_horizon))
if not steps:
return
hour = self.hour
kwargs = self.kwargs
super().configure(config_dict)
nightly_kwargs = {}
# we take the defaults of Nightly, except for hour
for arg in ('minute', 'dayOfMonth', 'month', 'dayOfWeek'):
if arg in kwargs:
nightly_kwargs[arg] = kwargs[arg]
self.schedulers.append(
Nightly(name=JANITOR_NAME,
builderNames=[JANITOR_NAME],
hour=hour,
**nightly_kwargs))
self.schedulers.append(
ForceScheduler(name=JANITOR_NAME + "_force",
builderNames=[JANITOR_NAME]))
self.builders.append(
BuilderConfig(name=JANITOR_NAME,
workername=JANITOR_NAME,
factory=BuildFactory(steps=steps)))
self.protocols.setdefault('null', {})
self.workers.append(LocalWorker(JANITOR_NAME))
def masterConfig():
c = {}
from buildbot.config import BuilderConfig
from buildbot.process.factory import BuildFactory
from buildbot.plugins import steps, schedulers
c['schedulers'] = [
schedulers.ForceScheduler(name="force", builderNames=["testy"])
]
f = BuildFactory()
# do a bunch of transfer to exercise the protocol
f.addStep(steps.StringDownload("filecontent", slavedest="dir/file1.txt"))
f.addStep(steps.StringDownload("filecontent2", slavedest="dir/file2.txt"))
f.addStep(
steps.FileUpload(slavesrc="dir/file2.txt", masterdest="master.txt"))
f.addStep(
steps.FileDownload(mastersrc="master.txt", slavedest="dir/file3.txt"))
f.addStep(steps.DirectoryUpload(slavesrc="dir", masterdest="dir"))
c['builders'] = [
BuilderConfig(name="testy", slavenames=["local1"], factory=f)
]
return c
def masterConfig():
c = {}
from buildbot.config import BuilderConfig
from buildbot.process.factory import BuildFactory
from buildbot.plugins import steps, schedulers
c['schedulers'] = [
schedulers.ForceScheduler(
name="force",
builderNames=["testy"])]
f = BuildFactory()
f.addStep(steps.ShellCommand(command='echo hello'))
c['builders'] = [
BuilderConfig(name="testy",
workernames=["local1"],
properties={
'virtual_builder_name': 'virtual_testy',
'virtual_builder_description': 'I am a virtual builder',
'virtual_builder_tags': ['virtual'],
},
factory=f)]
return c
