def create_two_worker_two_builder_lock_config(self, mode):
stepcontrollers = [BuildStepController(), BuildStepController()]
master_lock = util.MasterLock("lock1", maxCount=1)
config_dict = {
'builders': [
BuilderConfig(name='builder1',
workernames=['worker1'],
factory=BuildFactory([stepcontrollers[0].step]),
locks=[master_lock.access(mode)]),
BuilderConfig(name='builder2',
workernames=['worker2'],
factory=BuildFactory([stepcontrollers[1].step]),
locks=[master_lock.access(mode)]),
],
'workers': [
self.createLocalWorker('worker1'),
self.createLocalWorker('worker2'),
],
'protocols': {
'null': {}
},
'multiMaster':
True,
}
yield self.setup_master(config_dict)
builder_ids = [
(yield self.master.data.updates.findBuilderId('builder1')),
(yield self.master.data.updates.findBuilderId('builder2')),
]
return stepcontrollers, builder_ids
def __init__(self,
name,
baseurl,
branch,
*,
nightly=None,
enable_force=True,
giturl=None,
verbose_build=False,
description=None):
super().__init__(name)
if giturl is None:
giturl = baseurl + ".git"
self.baseurl = baseurl
self.giturl = giturl
self.branch = branch
self.nightly = nightly
self.enable_force = enable_force
self.verbose_build = verbose_build
self.description_ = description
if self.CONFIGURE_GENERATED_FILE is None:
raise Exception("Invalid CONFIGURE_GENERATED_FILE setting")
# Lock used to avoid writing source code when it is read by another task
self.lock_src = util.MasterLock("src-{0}".format(self.name),
maxCount=sys.maxsize)
self.buildNames()
def masterConfig():
c = {}
from buildbot.config import BuilderConfig
from buildbot.process.factory import BuildFactory
from buildbot.plugins import schedulers
from buildbot.plugins import util
lock = util.MasterLock("lock")
c['schedulers'] = [
schedulers.AnyBranchScheduler(
name="sched",
builderNames=["testy", "testy2", "testy3"]),
]
f = BuildFactory()
lockstep = LockedStep(locks=[lock.access('exclusive')])
f.addStep(lockstep)
# assert lockstep._factory.buildStep() == lockstep._factory.buildStep()
c['builders'] = [
BuilderConfig(name="testy",
workernames=["local1"],
factory=f),
BuilderConfig(name="testy2",
workernames=["local1"],
factory=f),
BuilderConfig(name="testy3",
workernames=["local1"],
factory=f)]
return c
def make_builders(repo_url,
worker_configs,
snapshots_dir=None,
snapshots_url=None,
snapshots_default_max=2):
# TODO: Use one lock per container host; for now we have only one container
# host so we just use a single lock. One lock per worker is not good enough
# since many workers may run on a single container host and this should block
# on host container resources.
# "build1" is the name used for the container host in docker-compose.yml
master_lock = util.MasterLock("build1")
builders = []
for (worker_name, worker_config) in iteritems(worker_configs):
for (builder_name,
builder_config) in iteritems(worker_config["builders"]):
# Workers with only one builder give it a blank name
if not builder_name:
builder_name = worker_name
else:
builder_name = "%s-%s" % (worker_name, builder_name)
builders.append(
make_builder_config(
repo_url=repo_url,
name=builder_name,
worker_name=worker_name,
config=builder_config,
lock=master_lock,
snapshots_dir=snapshots_dir,
snapshots_url=snapshots_url,
snapshots_default_max=snapshots_default_max))
return builders
platform.configureargs.append("--host=android-{0}".format(scummvm_target))
platform.configureargs.append("--enable-debug")
platform.packaging_cmd = "androiddistdebug"
platform.built_files = {
builds.ScummVMBuild: ["debug"],
}
platform.archiveext = "zip"
platform.testable = False
platform.description = description
platform.icon = "android"
register_platform(platform)
android.lock = util.MasterLock("android")
android(suffix="arm",
scummvm_target="arm-v7a",
ndk_target="arm-linux-androideabi",
cxx_target="armv7a-linux-androideabi",
abi_version=16,
description="Android (ARM)")
android(suffix="arm64",
scummvm_target="arm64-v8a",
ndk_target="aarch64-linux-android",
cxx_target="aarch64-linux-android",
abi_version=21,
description="Android (ARM 64\xa0bits)")
android(suffix="x86",
scummvm_target="x86",
ndk_target="i686-linux-android",
force_scheduler = schedulers.ForceScheduler(
name="force", builderNames=force_builder_names
)
c["schedulers"] = []
c["schedulers"].append(build_scheduler)
c["schedulers"].append(director_scheduler)
c["schedulers"].append(force_scheduler)
####### BUILDERS
# The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
# what steps, and which workers can execute them. Note that any particular build will
# only take place on one worker.
build_lock = util.MasterLock("Build")
c["builders"] = []
c["builders"].extend(generate_builder(target) for target in test_targets)
c["builders"].append(
util.BuilderConfig(
name="build",
workernames=["director-worker"],
factory=build_factory,
locks=[build_lock.access("exclusive")],
)
)
lingo_factory = util.BuildFactory()
lingo_factory.addStep(checkout_step)
def test_builder_lock_release_wakes_builds_for_another_builder(self):
"""
If a builder locks a master lock then the build request distributor
must retry running any buildrequests that might have been not scheduled
due to unavailability of that lock when the lock becomes available.
"""
stepcontroller1 = BuildStepController()
stepcontroller2 = BuildStepController()
master_lock = util.MasterLock("lock1", maxCount=1)
config_dict = {
'builders': [
BuilderConfig(name='builder1',
workernames=['worker1'],
factory=BuildFactory([stepcontroller1.step]),
locks=[master_lock.access('counting')]),
BuilderConfig(name='builder2',
workernames=['worker2'],
factory=BuildFactory([stepcontroller2.step]),
locks=[master_lock.access('counting')]),
],
'workers': [
self.createLocalWorker('worker1'),
self.createLocalWorker('worker2'),
],
'protocols': {
'null': {}
},
'multiMaster':
True,
}
master = yield self.getMaster(config_dict)
builder1_id = yield master.data.updates.findBuilderId('builder1')
builder2_id = yield master.data.updates.findBuilderId('builder2')
# start two builds and verify that a second build starts after the
# first is finished
yield self.createBuildrequest(master, [builder1_id])
yield self.createBuildrequest(master, [builder2_id])
builds = yield master.data.get(("builds", ))
self.assertEqual(len(builds), 1)
self.assertEqual(builds[0]['results'], None)
self.assertEqual(builds[0]['builderid'], builder1_id)
stepcontroller1.finish_step(SUCCESS)
# execute Build.releaseLocks which is called eventually
yield flushEventualQueue()
builds = yield master.data.get(("builds", ))
self.assertEqual(len(builds), 2)
self.assertEqual(builds[0]['results'], SUCCESS)
self.assertEqual(builds[1]['results'], None)
self.assertEqual(builds[1]['builderid'], builder2_id)
stepcontroller2.finish_step(SUCCESS)
builds = yield master.data.get(("builds", ))
self.assertEqual(len(builds), 2)
self.assertEqual(builds[0]['results'], SUCCESS)
self.assertEqual(builds[1]['results'], SUCCESS)
def factory(constructicon_name, builder_name, deps, commands, upload, zip,
unzip, url, resources):
deps = sorted(deps)
def work_dir_renderer(*suffix, **kwargs):
@util.renderer
def work_dir(properties):
if kwargs.get('log', False):
log.msg('properties are: ' +
pprint.pformat(properties.asDict()))
log.msg('sourcestamps are: ' + pprint.pformat(
[(i.repository, i.branch, i.revision)
for i in properties.getBuild().getAllSourceStamps()]))
sep = '/'
if all_slaves[properties['slavename']].get('platform',
0) == 'windows':
sep = '\\'
return sep.join(('..', 'constructicons', constructicon_name,
constructicon_name) + suffix)
return work_dir
result = util.BuildFactory()
def git_step(repo_url, work_dir, env):
return common.sane_step(
steps.Git,
repourl=repo_url,
codebase=repo_url,
workdir=work_dir,
mode='incremental',
env=env,
warnOnWarnings=False,
)
def extract_parameters(dict):
return {
i[len(parameter_prefix):]: str(j[0])
for i, j in dict.items() if i.startswith(parameter_prefix)
}
@util.renderer
def env(properties):
return extract_parameters(properties.asDict())
def format(command):
@util.renderer
def f(properties):
return command.format(**extract_parameters(properties.asDict()))
return f
@util.renderer
def get_command(properties):
revisions = ''
for i in properties.getBuild().getAllSourceStamps():
revision = None
if i.revision: revision = i.revision
elif i.branch: revision = i.branch
if revision: revisions += ' {}:{}'.format(i.codebase, revision)
if revisions: revisions = ' -r' + revisions
return common.constructicon_slave_go('g {}{}'.format(
builder_name,
revisions,
))
for resource in resources:
if resource not in resource_locks:
resource_locks[resource] = util.MasterLock(resource)
locks = [resource_locks[i].access('exclusive') for i in resources]
#properties, get, compile
result.addSteps([
common.sane_step(
steps.SetProperty,
name='devastator git state',
property='devastator_git_state',
value={{{devastator_git_state}}},
),
common.sane_step(
steps.SetProperty,
name='cybertron git state',
property='cybertron_git_state',
value={{{cybertron_git_state}}},
),
common.sane_step(
steps.SetProperty,
name='git state',
property='git_state',
value=global_git_states[constructicon_name],
),
git_step(global_repo_urls[constructicon_name], work_dir_renderer(),
env),
common.sane_step(
steps.ShellCommand,
name='get',
command=get_command,
workdir=work_dir_renderer(log=True),
env=env,
warnOnWarnings=False,
),
])
for command_i in range(len(commands)):
kwargs = {}
meat = commands[command_i][1]
timeout = 5 * 60
if type(meat) == str:
command = meat
else:
command = meat['command']
warning_pattern = '(.*warning[: ])'
if 'warnings' in meat:
warning_pattern = '({})'.format('|'.join(meat['warnings']))
if 'suppress_warnings' in meat:
warning_pattern = warning_pattern + '(?!{})'.format('|'.join(
meat['suppress_warnings']))
kwargs['warningPattern'] = warning_pattern
timeout = meat.get('timeout', timeout)
result.addStep(
common.sane_step(steps.Compile,
name=commands[command_i][0],
command=format(command),
workdir=work_dir_renderer(),
env=env,
locks=locks,
timeout=timeout,
maxTime=2 * 60 * 60,
**kwargs))
#upload
for i, j in upload.items(True):
zip_steps = []
upload_steps = []
unzip_steps = []
slave_src = i
master_dst_extension = ''
#zip
if i in zip:
@util.renderer
def command(properties, i=i):
return 'python -m zipfile -c {0}.zip {0}'.format(i)
zip_steps.append(
steps.ShellCommand(
command=command,
workdir=work_dir_renderer(),
alwaysRun=True,
))
slave_src += '.zip'
master_dst_extension = '.zip'
#unzip
def master_dst_function(properties,
j=j,
extension=master_dst_extension,
suffix=None):
return os.path.join(
make_full_builder_name(constructicon_name, builder_name),
str(properties['buildnumber']) + '-constructicon',
suffix if suffix else j + master_dst_extension)
@util.renderer
def master_dst_renderer(properties, f=master_dst_function):
return f(properties)
url_trim = 0
if j in unzip:
@util.renderer
def command(properties, master_dst_function=master_dst_function):
master_dst = master_dst_function(properties)
unzipped = os.path.split(master_dst)[0] or '.'
return 'python -m zipfile -e {} {}'.format(
master_dst, unzipped)
unzip_steps.append(
steps.MasterShellCommand(command=command, alwaysRun=True))
url_trim = 4
devastator_file_server_port = cybertron['devastator_file_server_port']
#upload
suffix = url.get(j, None)
@util.renderer
def url_renderer(
properties,
j=j,
suffix=suffix,
master_dst_function=master_dst_function,
devastator_file_server_port=devastator_file_server_port,
url_trim=url_trim):
return ('http://{}:{}'.format({{{devastator_host}}},
devastator_file_server_port) + '/' +
master_dst_function(properties, suffix=suffix))
upload_steps.append(
steps.FileUpload(
slavesrc=slave_src,
masterdest=master_dst_renderer,
url=url_renderer,
workdir=work_dir_renderer(),
alwaysRun=True,
))
#append
result.addSteps(zip_steps + upload_steps + unzip_steps)
return result
def getBuildersForBranch(props):
pretty_branch_name = props['branch_pretty']
deb_props = dict(props)
deb_props['image'] = random.choice({{docker_debian_worker_images}})
cent_props = dict(props)
cent_props['image'] = random.choice({{docker_centos_worker_images}})
builders = getPullRequestBuilder(props, pretty_branch_name)
#Only one maven build, per branch, at a time
branch_mvn_lock = util.MasterLock(pretty_branch_name + "mvn_lock")
for jdk in common.getJDKBuilds(props):
jdk_props = dict(props)
jdk_props['jdk'] = str(jdk)
builders.append(
util.BuilderConfig(
name=pretty_branch_name + " Build JDK " + str(jdk),
workernames=workers,
factory=build.getBuildPipeline(),
properties=jdk_props,
collapseRequests=True,
#A note on these locks: We want a single maven build per branch,
# AND a single maven build per worker
locks=[
mvn_lock.access('exclusive'),
branch_mvn_lock.access('exclusive')
]))
report_props = dict(jdk_props)
report_props['cores'] = '1'
builders.append(
util.BuilderConfig(
name=pretty_branch_name + " Reports JDK " + str(jdk),
workernames=workers,
factory=reports.getBuildPipeline(),
properties=jdk_props,
collapseRequests=True,
#A note on these locks: We want a single maven build per branch,
# AND a single maven build per worker
locks=[
mvn_lock.access('exclusive'),
branch_mvn_lock.access('exclusive')
]))
release_props = dict(props)
#We use the first listed JDK since that (should) be the lowest, most common version
release_props['jdk'] = str(common.getJDKBuilds(props)[0])
builders.append(
util.BuilderConfig(
name=pretty_branch_name + " Release",
workernames=workers,
factory=release.getBuildPipeline(),
properties=release_props,
collapseRequests=True,
#Note: We want a single maven build per worker, but since this is a release we don't
# care if there are other maven builds running elsewhere
locks=[mvn_lock.access('exclusive')]))
builders.append(
util.BuilderConfig(name=pretty_branch_name + " Markdown",
workernames=workers,
factory=markdown.getBuildPipeline(),
properties=props,
collapseRequests=True))
# builders.append(util.BuilderConfig(
# name=pretty_branch_name + " Database Tests",
# workernames=workers,
# factory=database.getBuildPipeline(),
# properties=props,
# collapseRequests=True,
# locks=[db_lock.access('exclusive')]))
builders.append(
util.BuilderConfig(name=pretty_branch_name + " Debian Packaging",
workernames=workers,
factory=debs.getBuildPipeline(),
properties=deb_props,
collapseRequests=True,
locks=[deb_lock.access('exclusive')]))
for distro in (7, 8):
el_props = dict(props)
el_props['el_version'] = distro
if 7 == distro:
el_props['image'] = f"cent{distro}"
lock = el7_lock
elif 8 == distro:
el_props['image'] = f"rocky{distro}"
lock = el8_lock
if "Develop" == pretty_branch_name:
#Set the RPM branch to master
el_props['rpmspec_override'] = "master"
#Override/set a bunch of the build props since the RPM's dont relaly have a develop...
builders.append(
util.BuilderConfig(name=pretty_branch_name +
f" el{distro} RPM Packaging",
workernames=workers,
factory=rpms.getBuildPipeline(),
properties=el_props,
collapseRequests=True,
locks=[lock.access('exclusive')]))
if props['deploy_env']:
deploy_props = dict(props)
deploy_props['deploy_suite'] = '{{ repo_deploy_suite }}'
deploy_props['package_repo_host'] = "{{ repo_host }}"
deploy_props['key_url'] = "{{ key_url }}"
deploy_props['key_id'] = "{{ key_id }}"
builders.append(
util.BuilderConfig(
name=pretty_branch_name + " Ansible Deploy",
workernames=workers,
factory=ansible.getBuildPipeline(),
properties=deploy_props,
collapseRequests=True,
#Ensure that no one is changing the package databases while we're deploying!
locks=[
deb_lock.access('exclusive'),
el7_lock.access('exclusive'),
el8_lock.access('exclusive')
]))
return builders
import reports
import markdown
import database
import debs
import rpms
import rpm_repo
import ansible
import release
#One of each of these per worker at a time
mvn_lock = util.WorkerLock("mvn_lock", maxCount=1)
db_lock = util.WorkerLock("db_lock", maxCount=1)
#These are used for the repository generation builders
#of which there must only be a single one running at a time across the whole cluster
deb_lock = util.MasterLock("deb_lock", maxCount=1)
el7_lock = util.MasterLock("el7_lock", maxCount=1)
el8_lock = util.MasterLock("el8_lock", maxCount=1)
# We're doing the filter here to remove blank entries (ie, "") since some of these lines in some cases don't yield
# results, but it's hard to keep from adding the front and end quotes in Jinja
workers = list(
filter(lambda a: a,
[{{
'\"' + groups['workers'] | map('extract', hostvars)
| selectattr('only_repo_builder', 'undefined')
| map(attribute='name') | join('\", \"') + '\"'
}},
{{
'\"' + groups['workers'] | map('extract', hostvars)
| selectattr('only_repo_builder', 'defined')
