def create_factory():
factory = BuildFactory()
factory.addSteps(common.initTargetProperty())
factory.addSteps(common.initNameProperty())
factory.addStep(
steps.Trigger(name="Call the 'build' scheduler",
schedulerNames=['build'],
waitForFinish=True,
haltOnFailure=True,
copy_properties=COMMON_PROPERTIES,
set_properties={
'virtual_builder_name':
util.Interpolate('Build for %(prop:box)s'),
}))
factory.addStep(
steps.Trigger(name="Call the 'run_test' scheduler",
schedulerNames=['run_test'],
waitForFinish=True,
copy_properties=COMMON_PROPERTIES,
set_properties={
'test_branch': util.Property('branch'),
"test_set": common.renderTestSet,
"use_valgrind": util.Property("use_valgrind"),
"use_callgrind": util.Property("use_callgrind"),
"backend_ssl": util.Property("backend_ssl")
}))
return factory
def create_factory():
factory = BuildFactory()
factory.addSteps(common.initTargetProperty())
factory.addStep(
steps.Trigger(name="Call the 'build' scheduler",
schedulerNames=['build'],
waitForFinish=True,
haltOnFailure=True,
copy_properties=COMMON_PROPERTIES,
set_properties={
'virtual_builder_name':
util.Interpolate('Build for %(prop:box)s'),
'box':
'ubuntu_bionic_libvirt',
'try_already_running':
'no',
}))
factory.addStep(
steps.Trigger(name="Call the 'run_performance_test' scheduler",
schedulerNames=['run_performance_test_trigger'],
waitForFinish=True,
copy_properties=COMMON_PROPERTIES,
set_properties={
'test_branch': util.Property('branch'),
'host': 'max-tst-01',
}))
return factory
def createFactory():
factory = BuildFactory()
factory.addSteps(common.initTargetProperty())
factory.addStep(steps.Trigger(
name="Call the 'build' scheduler. Build CentOS",
schedulerNames=['build'],
waitForFinish=True,
haltOnFailure=True,
copy_properties=COMMON_BUILD_AND_TEST_SNAPSHOT_PROPERTIES,
set_properties={
"box": util.Property("box"),
'try_already_running': 'yes',
"target": util.Property("target"),
'virtual_builder_name': util.Interpolate('Build for %(prop:box)s'),
}
))
factory.addStep(steps.Trigger(
name="Call the 'run_test_snapshot' scheduler. Run functional tests",
schedulerNames=['run_test_snapshot'],
waitForFinish=True,
copy_properties=COMMON_BUILD_AND_TEST_SNAPSHOT_PROPERTIES,
set_properties={
"box": util.Property("box"),
"target": util.Property("target"),
"test_branch": util.Property("branch"),
"test_set": common.renderTestSet,
"backend_ssl": util.Property("backend_ssl"),
"use_valgrind": util.Property("use_valgrind"),
}
))
return factory
def masterConfig():
c = {}
from buildbot.config import BuilderConfig
from buildbot.process.factory import BuildFactory
from buildbot.plugins import steps, schedulers
c['schedulers'] = [
schedulers.Triggerable(name="trigsched", builderNames=["build"]),
schedulers.AnyBranchScheduler(name="sched", builderNames=["testy"])
]
f = BuildFactory()
f.addStep(steps.ShellCommand(command='echo hello'))
f.addStep(
steps.Trigger(schedulerNames=['trigsched'],
waitForFinish=True,
updateSourceStamp=True))
f.addStep(steps.ShellCommand(command='echo world'))
f2 = BuildFactory()
f2.addStep(steps.ShellCommand(command='echo ola'))
c['builders'] = [
BuilderConfig(name="testy", workernames=["local1"], factory=f),
BuilderConfig(name="build", workernames=["local1"], factory=f2)
]
return c
def setupTriggerConfiguration(triggeredFactory, nextBuild=None):
c = {}
c['schedulers'] = [
schedulers.Triggerable(name="trigsched", builderNames=["triggered"]),
schedulers.AnyBranchScheduler(name="sched", builderNames=["main"])
]
f = BuildFactory()
f.addStep(
steps.Trigger(schedulerNames=['trigsched'],
waitForFinish=True,
updateSourceStamp=True))
f.addStep(steps.ShellCommand(command='echo world'))
mainBuilder = BuilderConfig(name="main", workernames=["local1"], factory=f)
triggeredBuilderKwargs = {
'name': "triggered",
'workernames': ["local1"],
'factory': triggeredFactory
}
if nextBuild is not None:
triggeredBuilderKwargs['nextBuild'] = nextBuild
triggeredBuilder = BuilderConfig(**triggeredBuilderKwargs)
c['builders'] = [mainBuilder, triggeredBuilder]
return c
def masterConfig(num_concurrent, extra_steps=None):
if extra_steps is None:
extra_steps = []
c = {}
c['schedulers'] = [
schedulers.ForceScheduler(
name="force",
builderNames=["testy"])]
triggereables = []
for i in range(num_concurrent):
c['schedulers'].append(
schedulers.Triggerable(
name="trigsched" + str(i),
builderNames=["build"]))
triggereables.append("trigsched" + str(i))
f = BuildFactory()
f.addStep(steps.ShellCommand(command='echo hello'))
f.addStep(steps.Trigger(schedulerNames=triggereables,
waitForFinish=True,
updateSourceStamp=True))
f.addStep(steps.ShellCommand(command='echo world'))
f2 = BuildFactory()
f2.addStep(steps.ShellCommand(command='echo ola'))
for step in extra_steps:
f2.addStep(step)
c['builders'] = [
BuilderConfig(name="testy",
workernames=["marathon0"],
factory=f),
BuilderConfig(name="build",
workernames=["marathon" + str(i)
for i in range(num_concurrent)],
factory=f2)]
url = os.environ.get('BBTEST_MARATHON_URL')
creds = os.environ.get('BBTEST_MARATHON_CREDS')
if creds is not None:
user, password = creds.split(":")
else:
user = password = None
masterFQDN = os.environ.get('masterFQDN')
marathon_extra_config = {
}
c['workers'] = [
MarathonLatentWorker('marathon' + str(i), url, user, password, 'buildbot/buildbot-worker:master',
marathon_extra_config=marathon_extra_config,
masterFQDN=masterFQDN)
for i in range(num_concurrent)
]
# un comment for debugging what happens if things looks locked.
# c['www'] = {'port': 8080}
# if the masterFQDN is forced (proxy case), then we use 9989 default port
# else, we try to find a free port
if masterFQDN is not None:
c['protocols'] = {"pb": {"port": "tcp:9989"}}
else:
c['protocols'] = {"pb": {"port": "tcp:0"}}
return c
def masterConfig(num_concurrent, extra_steps=None):
if extra_steps is None:
extra_steps = []
c = {}
c['schedulers'] = [
schedulers.ForceScheduler(name="force", builderNames=["testy"])
]
triggereables = []
for i in range(num_concurrent):
c['schedulers'].append(
schedulers.Triggerable(name="trigsched" + str(i),
builderNames=["build"]))
triggereables.append("trigsched" + str(i))
f = BuildFactory()
f.addStep(steps.ShellCommand(command='echo hello'))
f.addStep(
steps.Trigger(schedulerNames=triggereables,
waitForFinish=True,
updateSourceStamp=True))
f.addStep(steps.ShellCommand(command='echo world'))
f2 = BuildFactory()
f2.addStep(steps.ShellCommand(command='echo ola'))
for step in extra_steps:
f2.addStep(step)
c['builders'] = [
BuilderConfig(name="testy", workernames=["hyper0"], factory=f),
BuilderConfig(
name="build",
workernames=["hyper" + str(i) for i in range(num_concurrent)],
factory=f2)
]
hyperconfig = workerhyper.Hyper.guess_config()
if isinstance(hyperconfig, string_types):
hyperconfig = json.load(open(hyperconfig))
hyperhost, hyperconfig = hyperconfig['clouds'].items()[0]
masterFQDN = os.environ.get('masterFQDN')
c['workers'] = [
HyperLatentWorker('hyper' + str(i),
'passwd',
hyperhost,
hyperconfig['accesskey'],
hyperconfig['secretkey'],
'buildbot/buildbot-worker:master',
masterFQDN=masterFQDN) for i in range(num_concurrent)
]
# un comment for debugging what happens if things looks locked.
# c['www'] = {'port': 8080}
# if the masterFQDN is forced (proxy case), then we use 9989 default port
# else, we try to find a free port
if masterFQDN is not None:
c['protocols'] = {"pb": {"port": "tcp:9989"}}
else:
c['protocols'] = {"pb": {"port": "tcp:0"}}
return c
def masterConfig():
c = {}
from buildbot.config import BuilderConfig
from buildbot.process.factory import BuildFactory
from buildbot.plugins import steps, schedulers
c['schedulers'] = [
schedulers.AnyBranchScheduler(name="sched", builderNames=["testy"])
]
triggereables = []
for i in range(NUM_CONCURRENT):
c['schedulers'].append(
schedulers.Triggerable(name="trigsched" + str(i),
builderNames=["build"]))
triggereables.append("trigsched" + str(i))
f = BuildFactory()
f.addStep(steps.ShellCommand(command='echo hello'))
f.addStep(
steps.Trigger(schedulerNames=triggereables,
waitForFinish=True,
updateSourceStamp=True))
f.addStep(steps.ShellCommand(command='echo world'))
f2 = BuildFactory()
f2.addStep(steps.ShellCommand(command='echo ola'))
c['builders'] = [
BuilderConfig(name="testy", workernames=["hyper0"], factory=f),
BuilderConfig(
name="build",
workernames=["hyper" + str(i) for i in range(NUM_CONCURRENT)],
factory=f2)
]
hyperconfig = workerhyper.Hyper.guess_config()
if isinstance(hyperconfig, string_types):
hyperconfig = json.load(open(hyperconfig))
hyperhost, hyperconfig = hyperconfig['clouds'].items()[0]
masterFQDN = os.environ.get('masterFQDN')
c['workers'] = [
HyperLatentWorker('hyper' + str(i),
'passwd',
hyperhost,
hyperconfig['accesskey'],
hyperconfig['secretkey'],
'buildbot/buildbot-worker:master',
masterFQDN=masterFQDN) for i in range(NUM_CONCURRENT)
]
# if the masterFQDN is forced (proxy case), then we use 9989 default port
# else, we try to find a free port
if masterFQDN is not None:
c['protocols'] = {"pb": {"port": "tcp:9989"}}
else:
c['protocols'] = {"pb": {"port": "tcp:0"}}
return c
def masterConfig(num_concurrent, extra_steps=None):
if extra_steps is None:
extra_steps = []
c = {}
c['schedulers'] = [
schedulers.ForceScheduler(name="force", builderNames=["testy"])
]
triggereables = []
for i in range(num_concurrent):
c['schedulers'].append(
schedulers.Triggerable(
name="trigsched" + str(i), builderNames=["build"]))
triggereables.append("trigsched" + str(i))
f = BuildFactory()
f.addStep(steps.ShellCommand(command='echo hello'))
f.addStep(
steps.Trigger(
schedulerNames=triggereables,
waitForFinish=True,
updateSourceStamp=True))
f.addStep(steps.ShellCommand(command='echo world'))
f2 = BuildFactory()
f2.addStep(steps.ShellCommand(command='echo ola'))
for step in extra_steps:
f2.addStep(step)
c['builders'] = [
BuilderConfig(name="testy", workernames=["kubernetes0"], factory=f),
BuilderConfig(
name="build",
workernames=["kubernetes" + str(i) for i in range(num_concurrent)],
factory=f2)
]
masterFQDN = os.environ.get('masterFQDN')
c['workers'] = [
kubernetes.KubeLatentWorker(
'kubernetes' + str(i),
'buildbot/buildbot-worker',
kube_config=kubeclientservice.KubeCtlProxyConfigLoader(
namespace=os.getenv("KUBE_NAMESPACE", "default")),
masterFQDN=masterFQDN) for i in range(num_concurrent)
]
# un comment for debugging what happens if things looks locked.
# c['www'] = {'port': 8080}
c['protocols'] = {"pb": {"port": "tcp:9989"}}
return c
def run(self):
"""
Create factory for this build.
:return: util.SUCCESS or Exception
"""
builder_steps = self.default_factory(self.build_specification, self.build.properties)
if self.build_specification.get('next_builders'):
builder_steps.append(
steps.Trigger(schedulerNames=['trigger'],
waitForFinish=False,
updateSourceStamp=False,
doStepIf=self.is_trigger_needed))
self.build.addStepsAfterCurrentStep(builder_steps)
# generator is used, because logs adding may take long time
yield self.addCompleteLog('list of steps',
str('\n'.join([str(step) for step in builder_steps])))
defer.returnValue(util.SUCCESS)
def createBuildFactory():
factory = util.BuildFactory()
buildSteps = createBuildSteps()
factory.addSteps(buildSteps)
factory.addStep(
steps.Trigger(name="Call the 'create_full_repo_all' scheduler",
schedulerNames=['create_full_repo_all_triggerable'],
waitForFinish=True,
copy_properties=[
"branch",
"repository",
"host",
"owners",
"version",
],
set_properties={
"major_ver": getMajorVersion,
"target": constructTargetString
}))
return factory
def create_factory():
factory = BuildFactory()
factory.addSteps(common.initTargetProperty())
factory.addSteps(common.initNameProperty())
factory.addStep(common.determineBuildId())
buildProperties = COMMON_PROPERTIES.copy()
buildProperties.update({
'virtual_builder_name':
util.Interpolate('Build for %(prop:box)s'),
})
factory.addStep(
steps.Trigger(
name="Call the 'build' scheduler",
schedulerNames=['build'],
waitForFinish=True,
set_properties=buildProperties,
))
runTestProperties = COMMON_PROPERTIES.copy()
runTestProperties.update({
"appendTestRunId":
util.Property("appendTestRunId"),
"backend_ssl":
util.Property("backend_ssl"),
"buildHosts": ["bb-host"],
"buildId":
util.Property("buildId"),
"test_branch":
util.Property('branch'),
"use_callgrind":
util.Property("use_callgrind"),
"use_valgrind":
util.Property("use_valgrind"),
})
factory.addStep(
ParallelRunTestTrigger(
name="Call the 'run_test' for each test set",
schedulerNames=['run_test'],
waitForFinish=True,
set_properties=runTestProperties,
))
return factory
def masterConfig(num_concurrent, extra_steps=None):
if extra_steps is None:
extra_steps = []
c = {}
c['schedulers'] = [
schedulers.ForceScheduler(name="force", builderNames=["testy"])
]
triggereables = []
for i in range(num_concurrent):
c['schedulers'].append(
schedulers.Triggerable(name="trigsched" + str(i),
builderNames=["build"]))
triggereables.append("trigsched" + str(i))
f = BuildFactory()
f.addStep(steps.ShellCommand(command='echo hello'))
f.addStep(
steps.Trigger(schedulerNames=triggereables,
waitForFinish=True,
updateSourceStamp=True))
f.addStep(steps.ShellCommand(command='echo world'))
f2 = BuildFactory()
f2.addStep(steps.ShellCommand(command='echo ola'))
for step in extra_steps:
f2.addStep(step)
c['builders'] = [
BuilderConfig(name="testy", workernames=["upcloud0"], factory=f),
BuilderConfig(
name="build",
workernames=["upcloud" + str(i) for i in range(num_concurrent)],
factory=f2)
]
creds = os.environ.get('BBTEST_UPCLOUD_CREDS')
if creds is not None:
user, password = creds.split(":")
else:
raise "Cannot run this test without credentials"
masterFQDN = os.environ.get('masterFQDN', 'localhost')
c['workers'] = []
for i in range(num_concurrent):
upcloud_host_config = {
"user_data":
"""
#!/usr/bin/env bash
groupadd -g 999 buildbot
useradd -u 999 -g buildbot -s /bin/bash -d /buildworker -m buildbot
passwd -l buildbot
apt update
apt install -y git python3 python3-dev python3-pip sudo gnupg curl
pip3 install buildbot-worker service_identity
chown -R buildbot:buildbot /buildworker
cat <<EOF >> /etc/hosts
127.0.1.1 upcloud{}
EOF
cat <<EOF >/etc/sudoers.d/buildbot
buidbot ALL=(ALL) NOPASSWD:ALL
EOF
sudo -H -u buildbot bash -c "buildbot-worker create-worker /buildworker {} upcloud{} pass"
sudo -H -u buildbot bash -c "buildbot-worker start /buildworker"
""".format(i, masterFQDN, i)
}
c['workers'].append(
UpcloudLatentWorker('upcloud' + str(i),
api_username=user,
api_password=password,
image='Debian GNU/Linux 9 (Stretch)',
hostconfig=upcloud_host_config,
masterFQDN=masterFQDN))
# un comment for debugging what happens if things looks locked.
# c['www'] = {'port': 8080}
# if the masterFQDN is forced (proxy case), then we use 9989 default port
# else, we try to find a free port
if masterFQDN is not None:
c['protocols'] = {"pb": {"port": "tcp:9989"}}
else:
c['protocols'] = {"pb": {"port": "tcp:0"}}
return c
def getGlobalBuilders(self, platforms):
ret = list()
f = util.BuildFactory()
f.workdir = ""
f.useProgress = False
f.addStep(
steps.Git(
mode="incremental",
repourl=self.giturl,
branch=self.branch,
locks=[self.lock_src.access("exclusive")],
))
if len(self.PATCHES):
f.addStep(
scummsteps.Patch(
base_dir=config.configuration_dir,
patches=self.PATCHES,
locks=[self.lock_src.access("exclusive")],
))
if self.nightly is not None:
# Trigger nightly scheduler to let it know the source stamp
f.addStep(
steps.Trigger(
name="Updating source stamp",
schedulerNames=["nightly-scheduler-{0}".format(self.name)],
set_properties={
'got_revision':
util.Property('got_revision', defaultWhenFalse=False),
},
updateSourceStamp=True,
hideStepIf=(lambda r, s: r == util.SUCCESS),
))
f.addStep(
steps.Trigger(name="Building all platforms",
schedulerNames=[self.names['sch-build']],
set_properties={
'got_revision':
util.Property('got_revision',
defaultWhenFalse=False),
'clean':
util.Property('clean', defaultWhenFalse=False),
'package':
util.Property('package', defaultWhenFalse=False)
},
updateSourceStamp=True,
waitForFinish=True))
ret.append(
util.BuilderConfig(
name=self.names['bld-fetch'],
workernames=workers.workers_by_type['fetcher'],
workerbuilddir="/data/src/{0}".format(self.name),
factory=f,
tags=["fetch", self.name],
locks=[lock_build.access('counting')],
))
if self.nightly is not None:
f = util.BuildFactory()
f.addStep(
steps.Trigger(
name="Building all platforms",
schedulerNames=[self.names['sch-build']],
updateSourceStamp=True,
waitForFinish=True,
set_properties={
'got_revision':
util.Property('got_revision', defaultWhenFalse=False),
'clean':
True,
'package':
True,
# Ensure our tag is put first and is split from the others
'owner':
' Nightly build ',
}))
ret.append(
util.BuilderConfig(
name=self.names['bld-nightly'],
# We use fetcher worker here as it will prevent building of other stuff like if a change had happened
workernames=workers.workers_by_type['fetcher'],
workerbuilddir="/data/triggers/nightly-{0}".format(
self.name),
factory=f,
tags=["nightly", self.name],
locks=[lock_build.access('counting')]))
snapshots_path = os.path.join(config.snapshots_dir, self.name)
# Builder to clean packages
f = util.BuildFactory()
f.addStep(
scummsteps.CleanupSnapshots(
dstpath=snapshots_path,
buildname=self.name,
platformnames=[
platform.name for platform in platforms
if platform.canPackage(self)
],
dry_run=util.Property("dry_run", False),
keep_builds=getattr(config, 'snapshots_keep_builds', 14),
obsolete=timedelta(
days=getattr(config, 'snapshots_obsolete_days', 30)),
cleanup_unknown=getattr(config, 'snapshots_clean_unknown',
True),
))
ret.append(
util.BuilderConfig(
name=self.names['bld-clean'],
workernames=workers.workers_by_type['fetcher'],
workerbuilddir="/data/triggers/cleanup-{0}".format(self.name),
factory=f,
tags=["cleanup", self.name],
locks=[lock_build.access('counting')]))
return ret
build_factory = util.BuildFactory()
# check out the source
checkout_step = steps.GitHub(
repourl="git://github.com/scummvm/scummvm.git",
mode="incremental",
**default_step_kwargs,
)
build_factory.addStep(checkout_step)
# run the tests (note that this will require that 'trial' is installed)
build_factory.addStep(
steps.Configure(
command=[
"./configure",
"--disable-all-engines",
"--enable-engine=director",
],
env={"CXX": "ccache g++"},
**default_step_kwargs,
))
build_factory.addStep(steps.Compile(command=["make"], **default_step_kwargs))
master_dir = os.path.dirname(os.path.dirname(__file__))
master_file = os.path.join(master_dir, "scummvm-binary")
worker_file = "scummvm"
build_factory.addStep(
steps.FileUpload(workersrc=worker_file, masterdest=master_file))
build_factory.addStep(
steps.Trigger(schedulerNames=["Director Tests"], waitForFinish=True))
