def __init__(self, eventtype=None, eventtype_re=None, eventtype_fn=None, **kw):
ChangeFilter.__init__(self, **kw)
self.checks.update(self.createChecks((eventtype, eventtype_re, eventtype_fn, "prop:event.type")))
# for branch change filter, we take the real gerrit branch
# instead of the change's branch, which is also used as a grouping key
if "branch" in self.checks:
self.checks["prop:event.change.branch"] = self.checks["branch"]
del self.checks["branch"]
def _ComputeSchedulers(builders):
scheduler_to_builders = {}
for builder_name, builder_data in builders['builders'].items():
scheduler_names = builder_data.get('schedulers', [])
if 'scheduler' in builder_data:
scheduler_names.append(builder_data['scheduler'])
for scheduler_name in scheduler_names:
if scheduler_name:
if scheduler_name not in builders['schedulers']:
raise ValueError('unknown scheduler "%s"' % scheduler_name)
scheduler_to_builders.setdefault(scheduler_name,
[]).append(builder_name)
schedulers = []
for scheduler_name, scheduler_values in builders['schedulers'].items():
scheduler_type = scheduler_values['type']
builder_names = scheduler_to_builders[scheduler_name]
if scheduler_type == 'git_poller':
# git_poller pollers group changes, so we match on our specific repository
# to ensure that we only pick up changes from our poller.
schedulers.append(
SingleBranchScheduler(
name=scheduler_name,
change_filter=ChangeFilter(
repository=scheduler_values['git_repo_url'],
branch=scheduler_values.get('branch', 'master'),
),
treeStableTimer=scheduler_values.get(
'treeStableTimer', 60),
builderNames=builder_names))
elif scheduler_type == 'repo_poller':
# repo_poller pollers punt changes that use the scheduler name as their
# category (see _ComputeChangeSourceAndTagComparator). Matching on this
# ensures that we only match changes from our poller.
schedulers.append(
SingleBranchScheduler(name=scheduler_name,
change_filter=ChangeFilter(
category=str(scheduler_name), ),
treeStableTimer=scheduler_values.get(
'treeStableTimer', 60),
builderNames=builder_names))
elif scheduler_type == 'cron':
schedulers.append(
Nightly(name=scheduler_name,
branch='master',
minute=scheduler_values['minute'],
hour=scheduler_values['hour'],
builderNames=builder_names))
else:
raise ValueError('unsupported scheduler type "%s"' %
scheduler_type)
return schedulers
def __init__(self,
eventtype=None, eventtype_re=None, eventtype_fn=None, **kw):
ChangeFilter.__init__(self, **kw)
self.checks.update(
self.createChecks(
(eventtype, eventtype_re, eventtype_fn, "prop:event.type"),
))
# for branch change filter, we take the real gerrit branch
# instead of the change's branch, which is also used as a grouping key
if "branch" in self.checks:
self.checks["prop:event.change.branch"] = self.checks["branch"]
del self.checks["branch"]
def get_schedulers():
first_phase = phases[0]
last_phase = phases[-1]
# The VC scheduler initiates the first phase.
# Each phase, in turn, triggers the next phase,
# until the final phase.
for phase in phases:
my_filter = ChangeFilter(filter_fn=_tracked_projects)
phase_name = 'phase%d' % phase['number']
if phase == first_phase:
delay=120
else:
delay=15
my_filter = ChangeFilter(category = phase_name)
yield basic.AnyBranchScheduler(
name = phase_name, treeStableTimer=delay,
change_filter = my_filter,
builderNames = ['phase%d - %s' % (phase['number'], phase['name'])],
)
# Add triggers for initiating the builds in each phase.
for phase in phases:
# Split the phase builders into separate stages.
split_stages = get_phase_stages(phase)
for i, (normal, experimental) in enumerate(split_stages):
# Add the normal trigger, if used.
if normal:
yield triggerable.Triggerable(
name = 'phase%d-stage%d' % (phase['number'], i),
builderNames = [b['name'] for b in normal])
# Add the experimental trigger, if used.
if experimental:
yield triggerable.Triggerable(
name = 'phase%d-stage%d-experimental' % (phase['number'],
i),
builderNames = [b['name'] for b in experimental])
# Add a final trigger to trigger the validated build scheduler.
phase_name = 'GoodBuild'
my_filter = ChangeFilter(category = phase_name)
yield basic.AnyBranchScheduler(
name = phase_name, treeStableTimer=5,
builderNames = ['Validated Build',],
change_filter = my_filter,
)
def get_schedulers(builders):
platforms = ["linux", "osx", "win"]
schedulers = []
for platform in platforms:
platform_builders = [b for b in builders if platform in b.name]
# Unit tests for this platform
unit_builders = [b.name for b in platform_builders if "unit" in b.name]
master_unit = SingleBranchScheduler(
name="%s-master-unit" % platform,
change_filter=ChangeFilter(branch="master"),
treeStableTimer=60,
builderNames=unit_builders)
acceptance_builders = [
b.name for b in platform_builders if "acceptance" in b.name
]
master_acceptance = Dependent(name="%s-master-acceptance" % platform,
upstream=master_unit,
builderNames=acceptance_builders)
schedulers.extend([master_unit, master_acceptance])
return schedulers
def __init__(self, name, builderNames, properties={}, reason='',
createAbsoluteSourceStamps=False, onlyIfChanged=False,
branch=NoBranch, change_filter=None, fileIsImportant=None,
onlyImportant=False, **kwargs):
base.BaseScheduler.__init__(self, name, builderNames, properties,
**kwargs)
# tracking for when to start the next build
self.lastActuated = None
# A lock to make sure that each actuation occurs without interruption.
# This lock governs actuateAt, actuateAtTimer, and actuateOk
self.actuationLock = defer.DeferredLock()
self.actuateOk = False
self.actuateAt = None
self.actuateAtTimer = None
self.reason = util.ascii2unicode(reason % {'name': name})
self.branch = branch
self.change_filter = ChangeFilter.fromSchedulerConstructorArgs(change_filter=change_filter)
self.createAbsoluteSourceStamps = createAbsoluteSourceStamps
self.onlyIfChanged = onlyIfChanged
if fileIsImportant and not callable(fileIsImportant):
config.error(
"fileIsImportant must be a callable")
self.fileIsImportant = fileIsImportant
# If True, only important changes will be added to the buildset.
self.onlyImportant = onlyImportant
self._reactor = reactor # patched by tests
def getSchedulers():
return [
AnyBranchScheduler(
name="flocker",
treeStableTimer=5,
builderNames=BUILDERS,
# Only build certain branches because problems arise when we build
# many branches such as queues and request limits.
change_filter=ChangeFilter(branch_fn=build_automatically),
codebases={
"flocker": {
"repository": GITHUB + b"/flocker"
},
},
),
ForceScheduler(
name="force-flocker",
codebases=[
CodebaseParameter(
"flocker",
branch=StringParameter("branch", default="master",
size=80),
repository=FixedParameter("repository",
default=GITHUB + b"/flocker"),
),
],
properties=[
report_expected_failures_parameter,
],
builderNames=BUILDERS,
),
]
def __init__(self, name, builderNames, reason='',
createAbsoluteSourceStamps=False, onlyIfChanged=False,
branch=NoBranch, change_filter=None, fileIsImportant=None,
onlyImportant=False, **kwargs):
super().__init__(name, builderNames, **kwargs)
# tracking for when to start the next build
self.lastActuated = None
# A lock to make sure that each actuation occurs without interruption.
# This lock governs actuateAt, actuateAtTimer, and actuateOk
self.actuationLock = defer.DeferredLock()
self.actuateOk = False
self.actuateAt = None
self.actuateAtTimer = None
self.reason = util.bytes2unicode(reason % {'name': name})
self.branch = branch
self.change_filter = ChangeFilter.fromSchedulerConstructorArgs(
change_filter=change_filter)
self.createAbsoluteSourceStamps = createAbsoluteSourceStamps
self.onlyIfChanged = onlyIfChanged
if fileIsImportant and not callable(fileIsImportant):
config.error(
"fileIsImportant must be a callable")
self.fileIsImportant = fileIsImportant
# If True, only important changes will be added to the buildset.
self.onlyImportant = onlyImportant
self._reactor = reactor # patched by tests
def any_branch_scheduler():
return AnyBranchScheduler(
name=util.env.BOOTSTRAP_SCHEDULER_NAME,
treeStableTimer=5,
change_filter=ChangeFilter(branch_re='.+'), # build only branches
builderNames=[util.env.BOOTSTRAP_BUILDER_NAME],
reason='branch updated',
properties={'reason': 'branch updated'})
def getChangeFilter(self, branch, branches, change_filter, categories):
if branch is NotABranch and not change_filter:
config.error("the 'branch' argument to SingleBranchScheduler is " +
"mandatory unless change_filter is provided")
elif branches is not NotABranch:
config.error("the 'branches' argument is not allowed for " +
"SingleBranchScheduler")
return ChangeFilter.fromSchedulerConstructorArgs(
change_filter=change_filter, branch=branch, categories=categories)
def FilterNewSpec(repo, builder, branch='master'):
"""Create a new ChangeFilter that monitors the creation of new spec files.
Args:
repo: The repository to watch.
builder: The name of the cbuildbot config to watch.
branch: The branch that the specified builder is building on.
"""
prefix = 'Automatic: Start %s %s ' % (builder, branch)
return ChangeFilter(lambda change: change.comments.startswith(prefix),
repository=repo)
def watch_paths(self, paths):
"""
Start the build if an incoming change-set contains files that begin with the given
directory names.
"""
BuildmasterConfig['schedulers'].append(SingleBranchScheduler(
builderNames=[self.name],
change_filter=ChangeFilter(filter_fn=partial(is_dir_in_change, paths)),
name=scheduler_name(self, 'filter-' + hashify(''.join(paths))),
treeStableTimer=60))
def Update(c):
c['schedulers'].append(
SingleBranchScheduler(name='chromium_scheduler',
change_filter=ChangeFilter(project='chromium',
branch='master'),
treeStableTimer=60,
builderNames=[
'Win Builder',
'Mac Builder',
'Linux Builder',
]), )
specs = [
{
'name': 'Win Builder',
'category': 'win'
},
{
'name': 'Win7 Tester',
'category': 'win'
},
{
'name': 'Win8 Tester',
'category': 'win'
},
{
'name': 'Win10 Tester',
'category': 'win'
},
{
'name': 'Mac Builder',
'category': 'mac'
},
{
'name': 'Mac Tester',
'category': 'mac'
},
{
'name': 'Linux Builder',
'category': 'linux'
},
{
'name': 'Linux Tester',
'category': 'linux'
},
]
c['builders'].extend([{
'name': spec['name'],
'factory': m_remote_run('chromium'),
'category': spec['category'],
'notify_on_missing': True,
} for spec in specs])
def getChangeFilter(self, branch, branches, change_filter, categories):
if branch is NotABranch and not change_filter:
config.error(
"the 'branch' argument to SingleBranchScheduler is " +
"mandatory unless change_filter is provided")
elif branches is not NotABranch:
config.error(
"the 'branches' argument is not allowed for " +
"SingleBranchScheduler")
return ChangeFilter.fromSchedulerConstructorArgs(
change_filter=change_filter, branch=branch,
categories=categories)
def watch(self, url, branch):
repo_url = '%s/%s' % (url, branch)
if not has_svn_change_source(repo_url):
BuildmasterConfig['change_source'].append(SVNPoller(
svnurl=repo_url,
pollInterval=300 + randrange(0, 60),
histmax=5))
BuildmasterConfig['schedulers'].append(SingleBranchScheduler(
name=scheduler_name(self, 'svn-' + hashify(repo_url)),
treeStableTimer=60,
builderNames=[self.name],
change_filter=ChangeFilter(repository=repo_url)))
def FilterNewSpec(repo, builder, branch='master'):
"""Create a new ChangeFilter that monitors the creation of new spec files.
Args:
repo: The repository to watch.
builder: The name of the cbuildbot config to watch.
branch: The branch that the specified builder is building on.
"""
def checkChange(change):
for line in change.comments.splitlines():
if line.startswith('Automatic: Start %s %s ' % (builder, branch)):
return True
return False
return ChangeFilter(checkChange, repository=repo)
lan_buildslaves.append("anerist-slave")
all_publican_builders = []
all_translation_builders = {}
c['schedulers'] = []
c['change_source'] = []
c['builders'] = []
defined_factories = {}
pp = {
'name': 'Placeholder Project',
'uri': 'https://gitlab.randomuser.org/pete/placeholder.git',
'command': all
'branches' = ChangeFilter(
branch_fn = [
'master'
]
)
}
defined_factories['pp_maker_factory'] = BuildFactory(
_make_factory_step_generator(pp['name'], pp['uri'], pp['command'] )
)
c['change_source'].append(GitPoller(
pp['uri'],
workdir='/srv/buildbot',
branches=pp['branches'],
pollinterval=random.randint(300,600)
))
c['builders'].append(
BuilderConfig(
def ros_testbuild(c,
job_name,
url,
branch,
distro,
arch,
rosdistro,
machines,
othermirror,
keys,
token=None):
# Change source is either GitPoller or GitPRPoller
# TODO: make this configurable for svn/etc
project_name = ''
if token:
project_name = '_'.join([job_name, rosdistro, 'prtestbuild'])
c['change_source'].append(
GitPRPoller(
name=rosdistro + "_pr_poller",
repourl=url, # this may pose some problems
project=project_name,
token=token,
pollInterval=15))
# parse repo_url git@github:author/repo.git to repoOwner, repoName
r_owner, r_name = (url.split(':')[1])[:-4].split('/')
c['status'].append(
status.GitHubStatus(token=token,
repoOwner=r_owner,
repoName=r_name))
else:
project_name = '_'.join([job_name, rosdistro, 'testbuild'])
c['change_source'].append(
NamedGitPoller(repourl=url,
name=rosdistro,
branch=branch,
project=project_name))
c['schedulers'].append(
basic.SingleBranchScheduler(
name=project_name,
builderNames=[
project_name,
],
change_filter=ChangeFilter(project=project_name)))
# Directory which will be bind-mounted
binddir = '/tmp/' + project_name
f = BuildFactory()
# Remove any old crud in /tmp folder
f.addStep(ShellCommand(command=['rm', '-rf', binddir], hideStepIf=success))
# Check out repository (to /tmp)
f.addStep(
Git(repourl=util.Property('repository', default=url),
branch=util.Property('branch', default=branch),
alwaysUseLatest=True,
mode='full',
workdir=binddir + '/src/' + job_name))
# Download testbuild.py script from master
f.addStep(
FileDownload(name=job_name + '-grab-script',
mastersrc='scripts/testbuild.py',
slavedest=Interpolate('%(prop:workdir)s/testbuild.py'),
hideStepIf=success))
# Update the cowbuilder
f.addStep(
ShellCommand(command=['cowbuilder-update.py', distro, arch] + keys,
hideStepIf=success))
# Make and run tests in a cowbuilder
f.addStep(
TestBuild(name=job_name + '-build',
command=[
'sudo', 'cowbuilder', '--execute',
Interpolate('%(prop:workdir)s/testbuild.py'),
'--distribution', distro, '--architecture', arch,
'--bindmounts', binddir, '--basepath',
'/var/cache/pbuilder/base-' + distro + '-' + arch +
'.cow', '--override-config', '--othermirror',
othermirror, '--', binddir, rosdistro
],
logfiles={'tests': binddir + '/testresults'},
descriptionDone=['make and test', job_name]))
c['builders'].append(
BuilderConfig(name=project_name, slavenames=machines, factory=f))
# return the name of the job created
return project_name
def Update(c):
hourly_builders = [
'Android Builder',
'Android Builder (dbg)',
'Android Builder ARM64 (dbg)',
'Linux Builder',
'Mac Builder',
]
win_builders = [
'Win Builder',
]
all_builders = hourly_builders + win_builders
c['schedulers'].extend([
SingleBranchScheduler(name='webrtc_scheduler',
change_filter=ChangeFilter(project='webrtc',
branch='master'),
treeStableTimer=0,
builderNames=all_builders),
Periodic(name='hourly_periodic_scheduler',
periodicBuildTimer=60 * 60,
builderNames=hourly_builders),
Periodic(name='4hours_periodic_scheduler',
periodicBuildTimer=4 * 60 * 60,
builderNames=win_builders),
])
specs = [
{
'name': 'Win Builder',
'category': 'win'
},
{
'name': 'Win7 Tester',
'category': 'win'
},
{
'name': 'Win8 Tester',
'category': 'win'
},
{
'name': 'Win10 Tester',
'category': 'win'
},
{
'name': 'Mac Builder',
'category': 'mac'
},
{
'name': 'Mac Tester',
'category': 'mac'
},
{
'name': 'Linux Builder',
'category': 'linux'
},
{
'name': 'Linux Tester',
'category': 'linux'
},
{
'name': 'Android Builder',
'category': 'android'
},
{
'name': 'Android Builder (dbg)',
'category': 'android'
},
{
'name': 'Android Builder ARM64 (dbg)',
'category': 'android',
'slavebuilddir': 'android_arm64',
},
{
'name': 'Android Tests (dbg) (K Nexus5)',
'category': 'android'
},
{
'name': 'Android Tests (dbg) (L Nexus5)',
'category': 'android'
},
{
'name': 'Android Tests (dbg) (L Nexus6)',
'category': 'android'
},
{
'name': 'Android Tests (dbg) (L Nexus7.2)',
'category': 'android'
},
{
'name': 'Android Tests (dbg) (L Nexus9)',
'category': 'android'
},
]
for spec in specs:
builder_dict = {
'name': spec['name'],
'factory': m_remote_run('chromium'),
'category': spec['category'],
'notify_on_missing': True,
}
if 'slavebuilddir' in spec:
builder_dict['slavebuilddir'] = spec['slavebuilddir']
c['builders'].append(builder_dict)
def Update(self, c):
global_schedulers = dict((s.name, s) for s in c['schedulers']
if s.name.startswith('global_'))
assert not (set(global_schedulers) & set(self._schedulers))
for builder in self._builders:
# Update the schedulers with the builder.
schedulers = builder['schedulers']
if schedulers:
for scheduler in schedulers:
if scheduler in global_schedulers:
global_schedulers[scheduler].builderNames.append(builder['name'])
else:
self._schedulers[scheduler]['builders'].append(builder['name'])
# Construct the category.
categories = []
if builder.get('category', None):
categories.append(builder['category'])
if builder.get('gatekeeper', None):
categories.extend(builder['gatekeeper'].split('|'))
category = '|'.join(categories)
# Append the builder to the list.
new_builder = {'name': builder['name'],
'factory': self._factories[builder['factory']],
'category': category,
'auto_reboot': builder['auto_reboot']}
if builder['builddir']:
new_builder['builddir'] = builder['builddir']
if builder['slavebuilddir']:
new_builder['slavebuilddir'] = builder['slavebuilddir']
c['builders'].append(new_builder)
# Process the main schedulers.
for s_name in self._schedulers:
scheduler = self._schedulers[s_name]
if scheduler['type'] == 'Scheduler':
instance = Scheduler(name=s_name,
branch=scheduler['branch'],
treeStableTimer=scheduler['treeStableTimer'],
builderNames=scheduler['builders'],
categories=scheduler['categories'])
scheduler['instance'] = instance
c['schedulers'].append(instance)
elif scheduler['type'] == 'URLScheduler':
ident = str(uuid.uuid4())
c['change_source'].append(
URLPoller(changeurl=scheduler['url'],
pollInterval=scheduler['pollInterval'],
category=ident,
include_revision=scheduler['include_revision']))
instance = Scheduler(name=s_name,
change_filter=ChangeFilter(category=ident),
builderNames=scheduler['builders'])
scheduler['instance'] = instance
c['schedulers'].append(instance)
# Process the dependent schedulers.
for s_name in self._schedulers:
scheduler = self._schedulers[s_name]
if scheduler['type'] == 'Dependent':
c['schedulers'].append(
Dependent(s_name,
self._schedulers[scheduler['parent']]['instance'],
scheduler['builders']))
# Process the triggerable schedulers.
for s_name in self._schedulers:
scheduler = self._schedulers[s_name]
if scheduler['type'] == 'Triggerable':
c['schedulers'].append(Triggerable(s_name,
scheduler['builders']))
# Process the periodic schedulers.
for s_name in self._schedulers:
scheduler = self._schedulers[s_name]
if scheduler['type'] == 'Periodic':
c['schedulers'].append(
Periodic(s_name,
periodicBuildTimer=scheduler['periodicBuildTimer'],
builderNames=scheduler['builders']))
# Process the nightly schedulers.
for s_name in self._schedulers:
scheduler = self._schedulers[s_name]
if scheduler['type'] == 'Nightly':
c['schedulers'].append(Nightly(s_name,
branch=scheduler['branch'],
hour=scheduler['hour'],
builderNames=scheduler['builders']))
Args:
change: a buildbot Change object.
"""
if change.branch != 'trunk':
return False
for path in change.files:
if path.startswith('syzygy/binaries/'):
return True
return False
# Binaries scheduler for Syzygy.
binaries_scheduler = SingleBranchScheduler('syzygy_binaries',
treeStableTimer=0,
change_filter=ChangeFilter(
filter_fn=_BinariesFilter),
builderNames=['Syzygy Smoke Test'])
# Windows binaries smoke-test builder for Syzygy.
smoke_test_factory = win().SyzygySmokeTestFactory()
smoke_test_builder = {
'name': 'Syzygy Smoke Test',
'factory': smoke_test_factory,
'schedulers': 'syzygy_binaries',
'auto_reboot': False,
'category': 'official',
}
def isImportantWin_Server_BusinessWindows(change):
is_important = False
for each_file in change.files:
each_file = each_file.replace("\\","/")
if each_file.startswith("/"):
each_file = each_file[1:]
if each_file.endswith("/"):
each_file = each_file[:-1]
#if each_file.startswith("DVDFab/test.txt"):
if each_file.startswith("monitor_file_path"):
is_important = True
break
return is_important
scheduler_win_Server_Business_build = SingleBranchScheduler(name = "win_Server_Business",
change_filter = ChangeFilter(branch_fn=dev_branch_fn_git),
treeStableTimer = 5,
builderNames = ["win_Server_Business"],
fileIsImportant = isImportantWin_Server_BusinessWindows,
onlyImportant = True,
properties={'owner':['*****@*****.**']})
Win_Server_Business_time = timed.Nightly(
name = 'Win_Server_Business',
builderNames = ["win_Server_Business"],
branch = "trunk/goland",
hour = [200],
minute = 200,
dayOfWeek = [0, 1, 2, 3, 4])
from buildbot.schedulers.basic import AnyBranchScheduler
from buildbot.schedulers.triggerable import Triggerable
from buildbot.changes.filter import ChangeFilter
from buildbot.process.factory import BuildFactory
from buildbot.steps.master import MasterShellCommand
from buildscripts import steps as buildsteps
project = __opts__['project']
c['schedulers'].append(
AnyBranchScheduler(name=project,
change_filter=ChangeFilter(project=project,
category='default'),
builderNames=['{0} source'.format(project)]))
c["schedulers"].append(
Triggerable(name="{0} packaging".format(project),
builderNames=["deb_packaging", "rpm_packaging"]))
def push_to_github(__opts__):
cwd = 'sandboxes/{0}/svn2git'.format(project)
return [
MasterShellCommand(command="""
cd sandboxes/{0}/svn2git
svn2git --rebase --verbose
git push origin master""".format(project),
description='Pushing commit to GitHub',
descriptionDone='Push commit to GitHub (trunk)'),
]
c['properties'] = {
'version': SCHAT_VERSION,
'version_legacy': SCHAT_VERSION_LEGACY,
'suffix': '-dev',
}
c['schedulers'] = [
ForceScheduler(name='force',
builderNames=[
'lucid', 'lucid64', 'win32', 'win32-legacy', 'macosx',
'macosx-legacy', 'source', 'ppa', 'ppa-dev', 'obs',
'obs-dev', 'precise', 'precise64', 'release', 'beta'
]),
SingleBranchScheduler(name='trunk',
change_filter=ChangeFilter(branch=None),
treeStableTimer=60,
builderNames=[
'lucid', 'lucid64', 'win32', 'source', 'precise',
'precise64'
])
]
svn_co = [
SVN(mode='full',
method='clobber',
repourl='https://schat.googlecode.com/svn/trunk/'),
]
svn_co_legacy = [
SVN(mode='full',
def ros_testbuild(c, job_name, url, branch, distro, arch, rosdistro, machines, othermirror, keys):
# Change source is simply a GitPoller
# TODO: make this configurable for svn/etc
c['change_source'].append(
NamedGitPoller(
repourl = url,
name = rosdistro,
branch = branch,
project = job_name+'_'+rosdistro+'_testbuild'
)
)
c['schedulers'].append(
basic.SingleBranchScheduler(
name = job_name+'_'+rosdistro+'_testbuild',
builderNames = [job_name+'_'+rosdistro+'_testbuild',],
change_filter = ChangeFilter(project = job_name+'_'+rosdistro+'_testbuild')
)
)
# Directory which will be bind-mounted
binddir = '/tmp/'+job_name+'_'+rosdistro+'_testbuild'
f = BuildFactory()
# Remove any old crud in /tmp folder
f.addStep(
ShellCommand(
command = ['rm', '-rf', binddir],
hideStepIf = success
)
)
# Check out repository (to /tmp)
f.addStep(
Git(
repourl = url,
branch = branch,
alwaysUseLatest = True,
mode = 'full',
workdir = binddir+'/src/'+job_name
)
)
# Download testbuild.py script from master
f.addStep(
FileDownload(
name = job_name+'-grab-script',
mastersrc = 'scripts/testbuild.py',
slavedest = Interpolate('%(prop:workdir)s/testbuild.py'),
hideStepIf = success
)
)
# Update the cowbuilder
f.addStep(
ShellCommand(
command = ['cowbuilder-update.py', distro, arch] + keys,
hideStepIf = success
)
)
# Make and run tests in a cowbuilder
f.addStep(
TestBuild(
name = job_name+'-build',
command = ['cowbuilder', '--execute', Interpolate('%(prop:workdir)s/testbuild.py'),
'--distribution', distro, '--architecture', arch,
'--bindmounts', binddir,
'--basepath', '/var/cache/pbuilder/base-'+distro+'-'+arch+'.cow',
'--override-config', '--othermirror', othermirror,
'--', binddir, rosdistro],
logfiles = {'tests' : binddir+'/testresults' },
env = {'DIST': distro},
descriptionDone = ['make and test', job_name]
)
)
c['builders'].append(
BuilderConfig(
name = job_name+'_'+rosdistro+'_testbuild',
slavenames = machines,
factory = f
)
)
# return the name of the job created
return job_name+'_'+rosdistro+'_testbuild'
def getGlobalSchedulers(self, platforms):
ret = list()
change_filter = ChangeFilter(repository=self.baseurl,
branch=self.branch)
# Fetch scheduler (triggered by event source)
ret.append(
SingleBranchScheduler(name="fetch-{0}".format(self.name),
change_filter=change_filter,
treeStableTimer=5,
builderNames=["fetch-{0}".format(self.name)
]))
# Nightly scheduler (started by time)
# It's triggered after regular builds to take note of the last fetched source
# Note that build is not started by trigger
if self.nightly is not None:
ret.append(
NightlyTriggerable(
name="nightly-{0}".format(self.name),
branch=self.branch,
builderNames=["nightly-{0}".format(self.name)],
hour=self.nightly[0],
minute=self.nightly[1],
onlyIfChanged=True))
# All compiling builders
comp_builders = [
"{0}-{1}".format(self.name, p.name) for p in platforms
if p.canBuild(self)
]
# Global build scheduler (triggered by fetch build)
ret.append(Triggerable(name=self.name, builderNames=comp_builders))
# Force schedulers
if self.enable_force:
ret.append(
ForceScheduler(
name="force-scheduler-{0}-fetch".format(self.name),
reason=StringParameter(name="reason",
label="Reason:",
required=True,
size=80),
builderNames=["fetch-{0}".format(self.name)],
codebases=[CodebaseParameter(codebase='', hide=True)],
properties=[
BooleanParameter(name="clean",
label="Clean",
default=False),
BooleanParameter(name="package",
label="Package",
default=False),
]))
ret.append(
ForceScheduler(
name="force-scheduler-{0}-build".format(self.name),
reason=StringParameter(name="reason",
label="Reason:",
required=True,
size=80),
builderNames=comp_builders,
codebases=[CodebaseParameter(codebase='', hide=True)],
properties=[
BooleanParameter(name="clean",
label="Clean",
default=False),
BooleanParameter(name="package",
label="Package",
default=False),
]))
return ret
def _addGateKeepers(self):
"""
Parse configuration for gatekeepers.
"""
poll_branches = []
default = self._parent.getDefaultGateKeeperData()
default.update(self._default.get('gatekeepers', {}))
default.update(self._raw['gatekeepers'].get(DEFAULT, {}))
for keeper, keeper_data in self._raw['gatekeepers'].items():
if keeper == DEFAULT:
continue
data = default.copy()
data.update(keeper_data)
scheduler = data['scheduler']
slaves = self._parent.getSlaves(data['environment'])
steps = data['steps']
gatekeeper_properties = {}
builder_name = '%s-gk-%s' % (self._name, keeper)
if isinstance(scheduler, basestring):
poll_branches.append(scheduler)
stable_timer = data.get('stable_timer', STABLE_TIMER)
self._parent.addScheduler(
SingleBranchScheduler(
name=builder_name,
change_filter=ChangeFilter(
project=self._name,
branch=scheduler,
),
treeStableTimer=stable_timer,
builderNames=[builder_name],
))
else:
# Default to TRY scheduler.
self._parent.addTryTarget(builder_name)
send_github_status = data.get('github_send_status', False)
if send_github_status:
parts = self._github_slug.split('/', 1)
gatekeeper_properties.update({
"github_repo_owner": parts[0],
"github_repo_name": parts[1],
})
step_environment = self._parent.getStepEnvironment(
data['environment'])
self._parent.addBuilder(
BuilderConfig(
name=builder_name,
slavenames=slaves,
nextSlave=self._nextSlave,
factory=RunStepsFactory(
project=self,
steps=data['steps'],
environment=step_environment,
),
category=self._name,
properties=gatekeeper_properties,
))
self._parent.addNotifications(builder=builder_name,
configuration=data.get(
'notifications',
{},
))
self._addChangeSource(
project=self._name,
repo=self._repo,
branches=poll_branches,
)
from buildbot.changes.pb import PBChangeSource
from buildbot.config import BuilderConfig
from buildbot.process.factory import BuildFactory
from buildbot.schedulers.basic import AnyBranchScheduler
from buildbot.schedulers.forcesched import ForceScheduler
from buildbot.steps.shell import ShellCommand
from buildbot.worker import Worker
c['workers'] = [Worker("local1", "localpw")]
c['protocols'] = {'pb': {'port': 'tcp:0'}}
c['change_source'] = []
c['change_source'] = PBChangeSource()
c['schedulers'] = []
c['schedulers'].append(
AnyBranchScheduler(name="all",
change_filter=ChangeFilter(project_re='^testy/'),
treeStableTimer=1 * 60,
builderNames=[
'testy',
]))
c['schedulers'].append(ForceScheduler(name="force", builderNames=["testy"]))
f1 = BuildFactory()
f1.addStep(ShellCommand(command='echo hi'))
c['builders'] = []
c['builders'].append(
BuilderConfig(name="testy", workernames=["local1"], factory=f1))
c['status'] = []
c['title'] = "test"
c['titleURL'] = "test"
c['buildbotURL'] = "http://localhost:8010/"
c['db'] = {'db_url': "sqlite:///state.sqlite"}
def GetChangeFilter(self):
"""Returns (ChangeFilter): A BuildBot ChangeFilter for matching changes."""
return ChangeFilter(lambda change: self._CheckCommitLines(
*change.comments.splitlines()),
repository=self._repo)
def loadConfig(config):
####### CHANGESOURCES
# the 'change_source' setting tells the buildmaster how it should find out
# about source code changes. Here we point to the buildbot clone of pyflakes.
from buildbot.changes.gitpoller import GitPoller
from buildbot.changes.filter import ChangeFilter
config['change_source'].append( GitPoller(
repourl = '[email protected]:ORNL/xdd.git',
workdir='gitpoller-workdir-xdd-master',
pollinterval=120,
branch='master',
project='xdd'))
xdd_filter = ChangeFilter(
project = 'xdd',
branch = 'testing')
####### BUILDERS
# The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
# what steps, and which slaves can execute them. Note that any particular build will
# only take place on one slave.
from buildbot.process.factory import BuildFactory, GNUAutoconf
from buildbot.steps.source import Git
from buildbot.steps.shell import ShellCommand, Configure, Compile, Test
xdd_factory = BuildFactory()
# Check out the source
xdd_factory.addStep(Git(repourl='[email protected]:ORNL/xdd.git', mode='copy', branch='master'))
# Generate the test configuration
xdd_factory.addStep(ShellCommand(command=['./contrib/buildbot_gen_test_config.sh'], name="configuring"))
# Compile the code
xdd_factory.addStep(Compile(description=["compiling"]))
# Install the code
xdd_factory.addStep(ShellCommand(command=['make', 'install'], name="make install"))
# Perform make check
xdd_factory.addStep(ShellCommand(command=['make', 'check'], name="make check", maxTime=600))
# Perform make test
xdd_factory.addStep(Test(description=["make test"], maxTime=600))
# Perform cleanup
xdd_factory.addStep(ShellCommand(command=['pkill', '-f', 'xdd', '||', 'echo ""'], name='process cleanup', maxTime=60))
# Add the XDD Build factory to each of the available builders described in the master.cfg
from buildbot.config import BuilderConfig
# config['builders'].append(BuilderConfig(name="xdd-rhel5-x86_64", slavenames=["pod7"], factory=xdd_factory, env={"XDDTEST_TIMEOUT": "900"}, category='xdd'))
# config['builders'].append(BuilderConfig(name="xdd-rhel6-x86_64", slavenames=["pod9"], factory=xdd_factory, env={"XDDTEST_TIMEOUT": "900"},category='xdd'))
# config['builders'].append(BuilderConfig(name="xdd-sles10-x86_64", slavenames=["pod10"], factory=xdd_factory, env={"XDDTEST_TIMEOUT": "900"}, category='xdd'))
config['builders'].append(BuilderConfig(name="xdd-sles11-x86_64", slavenames=["pod11"], factory=xdd_factory, env={"XDDTEST_TIMEOUT": "900"}, category='xdd'))
config['builders'].append(BuilderConfig(name="xdd-osx-10-8", slavenames=["natureboy"], factory=xdd_factory, env={"XDDTEST_TIMEOUT": "900"}, category='xdd'))
# config['builders'].append(BuilderConfig(name="xdd-rhel6-ppc64", slavenames=["spry02"], factory=xdd_factory, env={"XDDTEST_TIMEOUT": "900"}, category='xdd'))
####### SCHEDULERS
# Configure the Schedulers, which decide how to react to incoming changes. In this
# case, just kick off a 'runtests' build
# Configure the nightly testing so that every test lives in the same buildset
from buildbot.schedulers.basic import SingleBranchScheduler
from buildbot.schedulers.timed import Periodic,Nightly
build_nightly_xdd=Nightly(name="xdd-nightly1",
branch = "master",
properties={'owner' : ['*****@*****.**']},
builderNames=["xdd-sles11-x86_64", "xdd-osx-10-8"],
hour = 2,
minute = 3)
config['schedulers'].append(build_nightly_xdd)
# Configure each force build seperately so that they live in differing buildsets
from buildbot.schedulers.forcesched import ForceScheduler
# config['schedulers'].append(ForceScheduler(name="xdd-force1", builderNames=["xdd-rhel5-x86_64"]))
# config['schedulers'].append(ForceScheduler(name="xdd-force2", builderNames=["xdd-rhel6-x86_64"]))
# config['schedulers'].append(ForceScheduler(name="xdd-force3", builderNames=["xdd-sles10-x86_64"]))
config['schedulers'].append(ForceScheduler(name="xdd-force4", builderNames=["xdd-sles11-x86_64"]))
config['schedulers'].append(ForceScheduler(name="xdd-force6", builderNames=["xdd-osx-10-8"]))
# config['schedulers'].append(ForceScheduler(name="xdd-force7", builderNames=["xdd-rhel6-ppc64"]))
####### STATUS TARGETS
# 'status' is a list of Status Targets. The results of each build will be
# pushed to these targets. buildbot/status/*.py has a variety to choose from,
# including web pages, email senders, and IRC bots.
from buildbot.status.mail import MailNotifier
xddMN = MailNotifier(fromaddr="*****@*****.**",
extraRecipients=['*****@*****.**'],
categories='xdd',
buildSetSummary=True,
messageFormatter=xddSummaryMail)
config['status'].append(xddMN)
def getChangeFilter(self, branch, branches, change_filter, categories):
assert branch is NotABranch
return ChangeFilter.fromSchedulerConstructorArgs(
change_filter=change_filter, branch=branches,
categories=categories)
def _VersionFileFilter(change):
"""A change filter function that disregards all changes that don't
touch src/syzygy/SYZYGY_VERSION.
Args:
change: a buildbot Change object.
"""
return change.branch == 'master' and 'syzygy/SYZYGY_VERSION' in change.files
# Official build scheduler for Syzygy
official_scheduler = SingleBranchScheduler(
'syzygy_version',
treeStableTimer=0,
change_filter=ChangeFilter(filter_fn=_VersionFileFilter),
builderNames=['Syzygy Official'])
# Windows official Release builder
official_builder = {
'name': 'Syzygy Official',
'factory': AF.BaseFactory(recipe='syzygy/continuous'),
'schedulers': 'syzygy_version',
'auto_reboot': False,
'category': 'official',
}
def Update(config, active_master, c):
c['schedulers'].append(official_scheduler)
c['builders'].append(official_builder)
def getChangeFilter(self, branch, branches, change_filter, categories):
assert branch is NotABranch
return ChangeFilter.fromSchedulerConstructorArgs(
change_filter=change_filter,
branch=branches,
categories=categories)
from buildbot.schedulers.basic import SingleBranchScheduler
from buildbot.schedulers.forcesched import ForceScheduler, FixedParameter, StringParameter
from buildbot.schedulers.timed import Periodic
from buildbot.schedulers.trysched import Try_Userpass
from buildbot.changes.filter import ChangeFilter
from config.builders import builder_names, polled_builder_names
all_builder_names = builder_names + polled_builder_names
schedulers = []
manual_builder = ChangeFilter(project=builder_names, branch='master')
schedulers.append(
SingleBranchScheduler(name="all",
treeStableTimer=15,
change_filter=manual_builder,
builderNames=builder_names))
polled_builder = ChangeFilter(project=polled_builder_names, branch='master')
schedulers.append(
SingleBranchScheduler(name="mongodb",
treeStableTimer=15,
change_filter=polled_builder,
builderNames=polled_builder_names))
schedulers.append(
ForceScheduler(name="force",
repository=FixedParameter(name="repository"),
branch=StringParameter(name="branch", default="master"),
