def _ComputeSchedulers(builders):
scheduler_to_builders = {}
for category, cat_data in builders['category'].items():
scheduler_name = cat_data.get('scheduler')
if scheduler_name:
if scheduler_name not in builders['schedulers']:
raise ValueError('unknown scheduler "%s"' % scheduler_name)
for builder_name, builder_data in builders['builders'].items():
if builder_data.get(
'categories'
) is None or category in builder_data['categories']:
scheduler_to_builders.setdefault(
scheduler_name,
[]).append("%s_%s" % (builder_name, category))
schedulers = []
for scheduler_name, scheduler_values in builders['schedulers'].items():
scheduler_type = scheduler_values['type']
builder_names = scheduler_to_builders[scheduler_name]
if scheduler_type in ('git_poller', 'repo_poller'):
schedulers.append(
SingleBranchScheduler(name=scheduler_name,
branch='master',
treeStableTimer=60,
builderNames=builder_names))
elif scheduler_type == 'cron':
schedulers.append(
Nightly(name=scheduler_name,
branch='master',
minute=scheduler_values['minute'],
hour=scheduler_values['hour'],
builderNames=builder_names))
elif scheduler_type == 'adb_cron':
schedulers.append(
Nightly(name=scheduler_name,
branch='master',
hour=range(0, 24, scheduler_values['every_x_hour']),
builderNames=builder_names))
elif scheduler_type == 'emu_scheduler':
schedulers.append(
EmulatorSingleBranchScheduler(
name=scheduler_name,
treeStableTimer=1,
builderNames=builder_names,
change_filter=ChangeFilter(
project=scheduler_values['project'])))
else:
raise ValueError('unsupported scheduler type "%s"' %
scheduler_type)
return schedulers
def configure(self, config_dict):
if self.logHorizon is None:
return
logHorizon = self.logHorizon
hour = self.hour
kwargs = self.kwargs
ConfiguratorBase.configure(self, config_dict)
nightly_kwargs = {}
# we take the defaults of Nightly, except for hour
for arg in ('minute', 'dayOfMonth', 'month', 'dayOfWeek'):
if arg in kwargs:
nightly_kwargs[arg] = kwargs[arg]
self.schedulers.append(
Nightly(name=JANITOR_NAME,
builderNames=[JANITOR_NAME],
hour=hour,
**nightly_kwargs))
self.builders.append(
BuilderConfig(
name=JANITOR_NAME,
workername=JANITOR_NAME,
factory=BuildFactory(
steps=[LogChunksJanitor(logHorizon=logHorizon)])))
self.protocols.setdefault('null', {})
self.workers.append(LocalWorker(JANITOR_NAME))
def configure(self, config_dict):
steps = []
if self.logHorizon is not None:
steps.append(LogChunksJanitor(logHorizon=self.logHorizon))
if self.build_data_horizon is not None:
steps.append(BuildDataJanitor(build_data_horizon=self.build_data_horizon))
if not steps:
return
hour = self.hour
kwargs = self.kwargs
super().configure(config_dict)
nightly_kwargs = {}
# we take the defaults of Nightly, except for hour
for arg in ('minute', 'dayOfMonth', 'month', 'dayOfWeek'):
if arg in kwargs:
nightly_kwargs[arg] = kwargs[arg]
self.schedulers.append(Nightly(
name=JANITOR_NAME, builderNames=[JANITOR_NAME], hour=hour, **nightly_kwargs))
self.schedulers.append(ForceScheduler(
name=JANITOR_NAME + "_force",
builderNames=[JANITOR_NAME]))
self.builders.append(BuilderConfig(
name=JANITOR_NAME, workername=JANITOR_NAME, factory=BuildFactory(steps=steps)
))
self.protocols.setdefault('null', {})
self.workers.append(LocalWorker(JANITOR_NAME))
def _ComputeSchedulers(builders):
scheduler_to_builders = {}
for builder_name, builder_data in builders['builders'].items():
scheduler_names = builder_data.get('schedulers', [])
if 'scheduler' in builder_data:
scheduler_names.append(builder_data['scheduler'])
for scheduler_name in scheduler_names:
if scheduler_name:
if scheduler_name not in builders['schedulers']:
raise ValueError('unknown scheduler "%s"' % scheduler_name)
scheduler_to_builders.setdefault(scheduler_name,
[]).append(builder_name)
schedulers = []
for scheduler_name, scheduler_values in builders['schedulers'].items():
scheduler_type = scheduler_values['type']
builder_names = scheduler_to_builders[scheduler_name]
if scheduler_type == 'git_poller':
# git_poller pollers group changes, so we match on our specific repository
# to ensure that we only pick up changes from our poller.
schedulers.append(
SingleBranchScheduler(
name=scheduler_name,
change_filter=ChangeFilter(
repository=scheduler_values['git_repo_url'],
branch=scheduler_values.get('branch', 'master'),
),
treeStableTimer=scheduler_values.get(
'treeStableTimer', 60),
builderNames=builder_names))
elif scheduler_type == 'repo_poller':
# repo_poller pollers punt changes that use the scheduler name as their
# category (see _ComputeChangeSourceAndTagComparator). Matching on this
# ensures that we only match changes from our poller.
schedulers.append(
SingleBranchScheduler(name=scheduler_name,
change_filter=ChangeFilter(
category=str(scheduler_name), ),
treeStableTimer=scheduler_values.get(
'treeStableTimer', 60),
builderNames=builder_names))
elif scheduler_type == 'cron':
schedulers.append(
Nightly(name=scheduler_name,
branch='master',
minute=scheduler_values['minute'],
hour=scheduler_values['hour'],
builderNames=builder_names))
else:
raise ValueError('unsupported scheduler type "%s"' %
scheduler_type)
return schedulers
def configure(config):
# pollers
config['change_source'].append(
GitPoller(repositoryUri,
workdir='gitpoller-workdir',
branch='dev',
pollinterval=60,
project='toy-factory-dev'))
# schedulers
config['schedulers'].append(
SingleBranchScheduler(
name="toy-factory-dev-ci",
change_filter=filter.ChangeFilter(branch='dev'),
treeStableTimer=None,
builderNames=["toy-factory-linux-ci", "toy-factory-windows-ci"]))
config['schedulers'].append(
ForceScheduler(
name="toy-factory-dev-ci-force",
builderNames=["toy-factory-linux-ci", "toy-factory-windows-ci"]))
config['schedulers'].append(
Nightly(
name='toy-factory-dev-nightly',
branch='dev',
builderNames=["toy-factory-linux-dev", "toy-factory-windows-dev"],
hour=3,
minute=0,
onlyIfChanged=True))
config['schedulers'].append(
ForceScheduler(
name="toy-factory-dev-force",
builderNames=["toy-factory-linux-dev", "toy-factory-windows-dev"]))
# builders
config['builders'].append(
BuilderConfig(name="toy-factory-linux-ci",
slavenames=["linux"],
factory=createLinuxCIFactory()))
config['builders'].append(
BuilderConfig(name="toy-factory-windows-ci",
slavenames=["windows"],
factory=createWindowsCIFactory()))
config['builders'].append(
BuilderConfig(name="toy-factory-linux-dev",
slavenames=["linux"],
factory=createLinuxDevFactory()))
config['builders'].append(
BuilderConfig(name="toy-factory-windows-dev",
slavenames=["windows"],
factory=createWindowsDevFactory()))
def __init__(self, ssFunc, *args, **kwargs):
self.ssFunc = ssFunc
Nightly.__init__(self, *args, **kwargs)
def __init__(self, ssFunc, *args, **kwargs):
self.ssFunc = ssFunc
Nightly.__init__(self, *args, **kwargs)
# ]),
]:
schedulers.append(scheduler)
for buildslave in ['five', 'six']:
for scheduler in [
# Nightly(
# name="nightly-buildslave-cpp-%s" % buildslave,
# builderNames=["buildslave-cpp-%s" % buildslave],
# codebases=dockers_codebases,
# branch=None,
# hour=3,
# minute=0),
Nightly(name="nightly-buildslave-go-%s" % buildslave,
builderNames=["buildslave-go-%s" % buildslave],
codebases=dockers_codebases,
branch=None,
hour=3,
minute=0),
Nightly(name="nightly-buildslave-python-%s" % buildslave,
builderNames=["buildslave-python-%s" % buildslave],
codebases=dockers_codebases,
branch=None,
hour=3,
minute=30),
# Nightly(
# name="nightly-buildslave-java-%s" % buildslave,
# builderNames=["buildslave-java-%s" % buildslave],
# codebases=dockers_codebases,
# branch=None,
# hour=3,
# minute=30)
)
)
# Schedulers are a per-project entity, but need to contain the list of
# every target that they build. First we define the nightly target,
# which doesn't actually depend on anything that does any polling.
# The nightly target requires a branch so we actually have to make
# multiple nighly schedulers per project since some projects may use
# multiple branches.
for project in project_list.projects():
for target in project.targets():
c['schedulers'].append(
Nightly(
name = "sched-nightly-" + project.target_name(target),
branch = target.branch(),
builderNames = [project.target_name(target)],
hour = 00,
minute = 52
)
)
if force_list.should_force(project.name()):
c['schedulers'].append(
ForceScheduler(
name = ("force-" + project.name()).encode('ascii','ignore'),
builderNames = map(
lambda n: n.encode('ascii','ignore'),
project.all_target_names())
)
)
# Installs email handlers
def loadConfig(config):
####### CHANGESOURCES
# the 'change_source' setting tells the buildmaster how it should find out
# about source code changes. Here we point to the buildbot clone of pyflakes.
from buildbot.changes.gitpoller import GitPoller
from buildbot.changes.filter import ChangeFilter
config['change_source'].append( GitPoller(
repourl = '[email protected]:ORNL/xdd.git',
workdir='gitpoller-workdir-xdd-master',
pollinterval=120,
branch='master',
project='xdd'))
xdd_filter = ChangeFilter(
project = 'xdd',
branch = 'testing')
####### BUILDERS
# The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
# what steps, and which slaves can execute them. Note that any particular build will
# only take place on one slave.
from buildbot.process.factory import BuildFactory, GNUAutoconf
from buildbot.steps.source import Git
from buildbot.steps.shell import ShellCommand, Configure, Compile, Test
xdd_factory = BuildFactory()
# Check out the source
xdd_factory.addStep(Git(repourl='[email protected]:ORNL/xdd.git', mode='copy', branch='master'))
# Generate the test configuration
xdd_factory.addStep(ShellCommand(command=['./contrib/buildbot_gen_test_config.sh'], name="configuring"))
# Compile the code
xdd_factory.addStep(Compile(description=["compiling"]))
# Install the code
xdd_factory.addStep(ShellCommand(command=['make', 'install'], name="make install"))
# Perform make check
xdd_factory.addStep(ShellCommand(command=['make', 'check'], name="make check", maxTime=600))
# Perform make test
xdd_factory.addStep(Test(description=["make test"], maxTime=600))
# Perform cleanup
xdd_factory.addStep(ShellCommand(command=['pkill', '-f', 'xdd', '||', 'echo ""'], name='process cleanup', maxTime=60))
# Add the XDD Build factory to each of the available builders described in the master.cfg
from buildbot.config import BuilderConfig
# config['builders'].append(BuilderConfig(name="xdd-rhel5-x86_64", slavenames=["pod7"], factory=xdd_factory, env={"XDDTEST_TIMEOUT": "900"}, category='xdd'))
# config['builders'].append(BuilderConfig(name="xdd-rhel6-x86_64", slavenames=["pod9"], factory=xdd_factory, env={"XDDTEST_TIMEOUT": "900"},category='xdd'))
# config['builders'].append(BuilderConfig(name="xdd-sles10-x86_64", slavenames=["pod10"], factory=xdd_factory, env={"XDDTEST_TIMEOUT": "900"}, category='xdd'))
config['builders'].append(BuilderConfig(name="xdd-sles11-x86_64", slavenames=["pod11"], factory=xdd_factory, env={"XDDTEST_TIMEOUT": "900"}, category='xdd'))
config['builders'].append(BuilderConfig(name="xdd-osx-10-8", slavenames=["natureboy"], factory=xdd_factory, env={"XDDTEST_TIMEOUT": "900"}, category='xdd'))
# config['builders'].append(BuilderConfig(name="xdd-rhel6-ppc64", slavenames=["spry02"], factory=xdd_factory, env={"XDDTEST_TIMEOUT": "900"}, category='xdd'))
####### SCHEDULERS
# Configure the Schedulers, which decide how to react to incoming changes. In this
# case, just kick off a 'runtests' build
# Configure the nightly testing so that every test lives in the same buildset
from buildbot.schedulers.basic import SingleBranchScheduler
from buildbot.schedulers.timed import Periodic,Nightly
build_nightly_xdd=Nightly(name="xdd-nightly1",
branch = "master",
properties={'owner' : ['*****@*****.**']},
builderNames=["xdd-sles11-x86_64", "xdd-osx-10-8"],
hour = 2,
minute = 3)
config['schedulers'].append(build_nightly_xdd)
# Configure each force build seperately so that they live in differing buildsets
from buildbot.schedulers.forcesched import ForceScheduler
# config['schedulers'].append(ForceScheduler(name="xdd-force1", builderNames=["xdd-rhel5-x86_64"]))
# config['schedulers'].append(ForceScheduler(name="xdd-force2", builderNames=["xdd-rhel6-x86_64"]))
# config['schedulers'].append(ForceScheduler(name="xdd-force3", builderNames=["xdd-sles10-x86_64"]))
config['schedulers'].append(ForceScheduler(name="xdd-force4", builderNames=["xdd-sles11-x86_64"]))
config['schedulers'].append(ForceScheduler(name="xdd-force6", builderNames=["xdd-osx-10-8"]))
# config['schedulers'].append(ForceScheduler(name="xdd-force7", builderNames=["xdd-rhel6-ppc64"]))
####### STATUS TARGETS
# 'status' is a list of Status Targets. The results of each build will be
# pushed to these targets. buildbot/status/*.py has a variety to choose from,
# including web pages, email senders, and IRC bots.
from buildbot.status.mail import MailNotifier
xddMN = MailNotifier(fromaddr="*****@*****.**",
extraRecipients=['*****@*****.**'],
categories='xdd',
buildSetSummary=True,
messageFormatter=xddSummaryMail)
config['status'].append(xddMN)
deb64_release = AnyBranchScheduler(name="deb64-release",
builderNames=["release-deb-x64"])
ubu64_release = AnyBranchScheduler(name="ubu64-release",
builderNames=["release-ubu-x64"])
android_release = Dependent(name="android-release",
upstream=ubu64_release,
builderNames=["release-android"])
freebsd_release = AnyBranchScheduler(name="freebsd-release",
builderNames=["release-freebsd-x64"])
lint_release = AnyBranchScheduler(name="lint-release", builderNames=["lint"])
arm_nightly = Nightly(name="arm-nightly",
branch=None,
builderNames=["nightly-arm"],
hour=0,
minute=0)
BuildmasterConfig = {
"title":
"Dolphin Emulator",
"titleURL":
"https://github.com/dolphin-emu/dolphin.git",
"buildbotURL":
"https://buildbot.dolphin-emu.org/",
# Set low horizons to avoid IO issues.
"changeHorizon":
100,
"buildHorizon":
choices=['debug', 'release'],
default='release'),
StringParameter(name='cmake_definitions',
label='cmake definitions',
default=''),
StringParameter(name='disable_tests',
label='disable accuracy tests',
default='rgbd, shape'),
StringParameter(name='disable_perf_tests',
label='disable performance tests',
default='stereo, tracking'),
StringParameter(name='filter_tests',
label='filter accuracy tests',
default=''),
StringParameter(name='filter_perf_tests',
label='filter performance tests',
default='')
]))
for b in Codebases.branches:
bases = Codebases.genBases(branch=b)
for bdt in ['debug', 'release']:
Schedulers.append(
Nightly(name="nightly %s-%s" % (bdt, b),
hour=3,
builderNames=BuilderNames,
codebases=bases,
properties=dict(build_type=bdt,
disable_tests='rgbd, shape',
disable_perf_tests='stereo, tracking')))