def __init__(self, platform, branch, **kwargs):
self.platform = platform
filter = ChangeFilter(branch=[branch, None], filter_fn=self.filter)
AnyBranchScheduler.__init__(self,
name=platform,
change_filter=filter,
**kwargs)
def _ComputeSchedulers(builders):
scheduler_to_builders = {}
for category, cat_data in builders['category'].items():
scheduler_name = cat_data.get('scheduler')
if scheduler_name:
if scheduler_name not in builders['schedulers']:
raise ValueError('unknown scheduler "%s"' % scheduler_name)
for builder_name, builder_data in builders['builders'].items():
if builder_data.get(
'categories'
) is None or category in builder_data['categories']:
scheduler_to_builders.setdefault(
scheduler_name,
[]).append("%s_%s" % (builder_name, category))
schedulers = []
for scheduler_name, scheduler_values in builders['schedulers'].items():
scheduler_type = scheduler_values['type']
builder_names = scheduler_to_builders[scheduler_name]
if scheduler_type in ('git_poller', 'repo_poller'):
schedulers.append(
SingleBranchScheduler(name=scheduler_name,
branch='master',
treeStableTimer=60,
builderNames=builder_names))
elif scheduler_type == 'cron':
schedulers.append(
Nightly(name=scheduler_name,
branch='master',
minute=scheduler_values['minute'],
hour=scheduler_values['hour'],
builderNames=builder_names))
elif scheduler_type == 'adb_cron':
schedulers.append(
Nightly(name=scheduler_name,
branch='master',
hour=range(0, 24, scheduler_values['every_x_hour']),
builderNames=builder_names))
elif scheduler_type == 'emu_scheduler':
schedulers.append(
EmulatorSingleBranchScheduler(
name=scheduler_name,
treeStableTimer=1,
builderNames=builder_names,
change_filter=ChangeFilter(
project=scheduler_values['project'])))
else:
raise ValueError('unsupported scheduler type "%s"' %
scheduler_type)
return schedulers
def getSingleBranchSchedulers(builders, schedulers, **kwargs):
"""
I'm taking over all of not yet assigned builders with the
declared source code dependencies, and automatically generate
a minimum set of SingleBranchSchedulers to handle all the declared
source code dependency combinations.
"""
# Prepare a list of builders which already have been assigned to schedulers.
builders_with_schedulers = {b for s in schedulers for b in s.builderNames}
builders_with_automatic_schedulers = []
for builder in builders:
# Only for the builders created with LLVMBuildFactory or similar.
if getattr(builder['factory'], 'depends_on_projects', None):
# And only if this builder does not yet have an assigned scheduler.
if builder['name'] not in builders_with_schedulers:
# This builder is a candidate for an automatic scheduler.
builders_with_automatic_schedulers.append(builder)
automatic_schedulers = []
# Do we have any to take care of?
if builders_with_automatic_schedulers:
# Let's reconsile first to get a unique set of dependencies.
# We need a set of unique sets of dependent projects.
set_of_dependencies = set([
frozenset(getattr(b['factory'], 'depends_on_projects'))
for b in builders_with_automatic_schedulers
])
treeStableTimer = kwargs.get('treeStableTimer', 2 * 60)
automatic_schedulers = []
for projects in set_of_dependencies:
sch_builders = [
b['name']
for b in builders_with_automatic_schedulers if frozenset(
getattr(b['factory'], 'depends_on_projects')) == projects
]
automatic_scheduler_name = ",".join(sorted(projects))
projects_to_filter = getProjectsToFilter(projects)
automatic_schedulers.append(
SingleBranchScheduler(
name=automatic_scheduler_name,
treeStableTimer=treeStableTimer,
builderNames=sch_builders,
change_filter=ChangeFilter(project=projects_to_filter)))
log.msg(
"Generated SingleBranchScheduler: { name='%s'" %
automatic_scheduler_name, ", builderNames=", sch_builders,
", change_filter=", projects_to_filter, "}")
return automatic_schedulers
def _getSingleBranchAutomaticSchedulers(builders_with_automatic_schedulers,
filter_branch, treeStableTimer):
automatic_schedulers = []
# Do we have any to take care of?
if builders_with_automatic_schedulers:
# Let's reconsile first to get a unique set of dependencies.
# We need a set of unique sets of dependent projects.
set_of_dependencies = set([
frozenset(getattr(b['factory'], 'depends_on_projects'))
for b in builders_with_automatic_schedulers
])
for projects in set_of_dependencies:
sch_builders = [
b['name']
for b in builders_with_automatic_schedulers if frozenset(
getattr(b['factory'], 'depends_on_projects')) == projects
]
automatic_scheduler_name = filter_branch + ":" + ",".join(
sorted(projects))
projects_to_filter = getProjectsToFilter(projects)
automatic_schedulers.append(
SingleBranchScheduler(name=automatic_scheduler_name,
treeStableTimer=treeStableTimer,
builderNames=sch_builders,
change_filter=ChangeFilter(
project=projects_to_filter,
branch=filter_branch)))
log.msg(
"Generated SingleBranchScheduler: { name='%s'" %
automatic_scheduler_name, ", builderNames=", sch_builders,
", change_filter=", projects_to_filter,
" (branch: %s)" % filter_branch,
", treeStableTimer=%s" % treeStableTimer, "}")
return automatic_schedulers
def make_change_filter(project):
return ChangeFilter(
repository_fn=lambda url: any(r.match_url(url) for r in repositories),
branch='master')
else:
from buildbot.buildslave import BuildSlave
from buildbot.scheduler import AnyBranchScheduler, Triggerable, Nightly
from buildbot.schedulers.forcesched import FixedParameter, ForceScheduler, StringParameter, BooleanParameter
from buildbot.schedulers.filter import ChangeFilter
from buildbot.process import buildstep, factory, properties
from factories import *
import re
import json
import operator
import wkbuild
trunk_filter = ChangeFilter(branch=["trunk", None])
buildbot_identifiers_re = re.compile('^[a-zA-Z_-][a-zA-Z0-9_-]*$')
BUILDER_NAME_LENGTH_LIMIT = 70
STEP_NAME_LENGTH_LIMIT = 50
def pickLatestBuild(builder, requests):
return max(requests, key=operator.attrgetter("submittedAt"))
def loadBuilderConfig(c, is_test_mode_enabled=False):
# FIXME: These file handles are leaked.
if is_test_mode_enabled:
passwords = {}
else:
def define_travis_builder(self,
name,
repository,
branch=None,
vcs_type=None,
username=None,
password=None):
job_name = "%s-job" % name
spawner_name = name
if not repository.endswith("/"):
repository = repository + "/"
if not vcs_type:
if repository.startswith("https://svn."):
vcs_type = "svn"
elif repository.startswith("git://github.com/"):
vcs_type = "git"
if not username and not password:
p = urlparse.urlparse(repository)
k = (p.scheme, p.netloc)
if k in self.passwords:
username, password = self.passwords[k]
# Define the builder for the main job
self.config['builders'].append(
BuilderConfig(
name=job_name,
slavenames=self.get_runner_slaves(),
properties=self.properties,
#mergeRequests = mergeRequests,
mergeRequests=False,
factory=TravisFactory(
repository=repository,
branch=branch,
vcs_type=vcs_type,
username=username,
password=password,
),
))
self.config['schedulers'].append(Triggerable(job_name, [job_name]))
# Define the builder for a spawer
self.config['builders'].append(
BuilderConfig(
name=spawner_name,
nextBuild=nextBuild,
slavenames=self.get_spawner_slaves(),
properties=self.properties,
category="spawner",
factory=TravisSpawnerFactory(
repository=repository,
branch=branch,
scheduler=job_name,
vcs_type=vcs_type,
username=username,
password=password,
),
))
SchedulerKlass = {
True: SingleBranchScheduler,
False: AnyBranchScheduler
}[bool(branch)]
self.config['schedulers'].append(
SchedulerKlass(
name=spawner_name,
builderNames=[spawner_name],
change_filter=ChangeFilter(project=name),
onlyImportant=True,
fileIsImportant=fileIsImportant,
))
setup_poller = dict(git=self.setup_git_poller,
svn=self.setup_svn_poller)[vcs_type]
setup_poller(repository, branch, name, username, password)
def filter_change(self, change):
print "Trying to filter %r with %r" % (change, self)
return ChangeFilter.filter_change(self, change)
def __init__(self, builders, **kwargs): self._builders = builders ChangeFilter.__init__(self, **kwargs)
def filter_change(self, change):
print "Trying to filter %r with %r" % (change, self)
return ChangeFilter.filter_change(self, change)
def define_travis_builder(self, name, repository, branch=None, vcs_type=None, username=None, password=None, subrepos=None):
job_name = "%s-job" % name
spawner_name = name
if not repository.endswith("/"):
repository = repository + "/"
if not username and not password:
p = urlparse.urlparse(repository)
k = (p.scheme, p.netloc)
if k in self.passwords:
username, password = self.passwords[k]
codebases = {spawner_name: {'repository': repository}}
if subrepos:
for subrepo in subrepos:
codebases[subrepo['project']] = {'repository': subrepo['repository']}
# Define the builder for the main job
self.config['builders'].append(BuilderConfig(
name = job_name,
slavenames = self.get_runner_slaves(),
properties = self.properties,
#mergeRequests = mergeRequests,
mergeRequests = False,
env = dict(
DEBIAN_FRONTEND = "noninteractive",
CI = "true",
TRAVIS = "true",
HAS_JOSH_K_SEAL_OF_APPROVAL = "true",
LANG = "en_GB.UTF-8",
LC_ALL = "en_GB.UTF-8",
),
factory = TravisFactory(
projectname = spawner_name,
repository = repository,
branch = branch,
vcs_type = vcs_type,
username = username,
password = password,
subrepos = subrepos,
),
))
self.config['schedulers'].append(Triggerable(
name=job_name,
builderNames=[job_name],
codebases=codebases,
))
# Define the builder for a spawer
self.config['builders'].append(BuilderConfig(
name = spawner_name,
nextBuild = nextBuild,
slavenames = self.get_spawner_slaves(),
properties = self.properties,
category = "spawner",
factory = TravisSpawnerFactory(
projectname = spawner_name,
repository = repository,
branch = branch,
scheduler = job_name,
vcs_type = vcs_type,
username = username,
password = password,
),
))
SchedulerKlass = {True:SingleBranchScheduler, False:AnyBranchScheduler}[bool(branch)]
self.config['schedulers'].append(SchedulerKlass(
name = spawner_name,
builderNames = [spawner_name],
change_filter = ChangeFilter(project=name),
onlyImportant = True,
fileIsImportant = fileIsImportant,
codebases=codebases,
))
self.setup_poller(repository, vcs_type, branch, name, username, password)