def addChange(self, change):
"""Call Scheduler.addChange only if the branch name (eg. project name
in your case) is in the repository url"""
if isinstance(change.branch, basestring):
if change.branch in self.repository:
change.branch = None
Scheduler.addChange(self, change)
def addChange(self, change):
"""Call Scheduler.addChange only if the branch name (eg. project name
in your case) is in the repository url"""
if 'repository' in change.properties:
if change.properties['repository'] == self.repository:
change.branch = None
Scheduler.addChange(self, change)
def __init__(self, name, builderNames, repository):
"""Override Scheduler.__init__
Add a new parameter : repository
"""
Scheduler.__init__(self, name, None, 120,
builderNames, fileIsImportant=None)
self.repository = repository
def addChange(self, change):
"""Call Scheduler.addChange only if the branch name (eg. project name
in your case) is in the repository url"""
if isinstance(change.branch, basestring):
if self.repository.endswith(change.branch):
self.branch = change.branch
Scheduler.addChange(self, change)
def __init__(self, name, project, builderNames, properties={}):
Scheduler.__init__(self, name, branch=None, treeStableTimer=180,
builderNames=builderNames, properties=properties)
self.project = project
self.importantChanges = []
self.unimportantChanges = []
self.nextBuildTime = None
self.timer = None
def __init__(self, name, project, builderNames, properties={}):
Scheduler.__init__(self, name, branch=None, treeStableTimer=180,
builderNames=builderNames, properties=properties)
self.project = project
self.importantChanges = []
self.unimportantChanges = []
self.nextBuildTime = None
self.timer = None
def addChange(self, change):
""" for some vcs, e.g. git, the default branch cannot be determined
leading to "ignored off-branch changes" in the log. this can be
fixed by explicitly setting a branch, which is made a little
clearer here """
if not self.branch:
log.msg('%s ignoring change due to unknown default branch. '
'please set one using `branch = ...`' % self)
Scheduler.addChange(self, change)
def addChange(self, change):
""" the documentation states that setting `branch` to `None` should
make the scheduler only consider changes on the default branch;
as there seems to be neither support for determining that default
branch nor for checking for `None`, this gets fixed by always
adding the change if `branch` was set to `None` """
if self.branch is None and change.branch is not None:
change.branch = None
Scheduler.addChange(self, change)
def __init__(
self, chooserFunc, prettyNames=None, unittestPrettyNames=None, unittestSuites=None,
talosSuites=None, buildbotBranch=None, buildersWithSetsMap=None, **kwargs):
self.chooserFunc = chooserFunc
self.prettyNames = prettyNames
self.unittestPrettyNames = unittestPrettyNames
self.unittestSuites = unittestSuites
self.talosSuites = talosSuites
self.buildbotBranch = buildbotBranch
self.buildersWithSetsMap = buildersWithSetsMap
Scheduler.__init__(self, **kwargs)
def __init__(
self, chooserFunc, prettyNames=None, unittestPrettyNames=None, unittestSuites=None,
talosSuites=None, buildbotBranch=None, buildersWithSetsMap=None, **kwargs):
self.chooserFunc = chooserFunc
self.prettyNames = prettyNames
self.unittestPrettyNames = unittestPrettyNames
self.unittestSuites = unittestSuites
self.talosSuites = talosSuites
self.buildbotBranch = buildbotBranch
self.buildersWithSetsMap = buildersWithSetsMap
Scheduler.__init__(self, **kwargs)
def decide_and_remove_changes(self, t, important, unimportant):
"""
Based on Scheduler.decide_and_remove_changes.
If we have n or more important changes, we should trigger jobs.
If more than idleTimeout has elapsed since the last change, we should trigger jobs.
"""
if not important:
return None
nImportant = len(important)
if nImportant < self.n:
if not self.idleTimeout:
log.msg("%s: skipping with %i/%i important changes since no idle timeout" %
(self.name, nImportant, self.n))
return
most_recent = max([c.when for c in important])
elapsed = int(now() - most_recent)
if self.idleTimeout and elapsed < self.idleTimeout:
# Haven't hit the timeout yet, so let's wait more
log.msg("%s: skipping with %i/%i important changes since only %i/%is have elapsed" %
(self.name, nImportant, self.n, elapsed, self.idleTimeout))
return now() + (self.idleTimeout - elapsed)
log.msg("%s: triggering with %i/%i important changes since %is have elapsed" % (self.name, nImportant, self.n, elapsed))
else:
log.msg("%s: triggering since we have %i/%i important changes" % (self.name, nImportant, self.n))
return Scheduler.decide_and_remove_changes(self, t, important, unimportant)
def decide_and_remove_changes(self, t, important, unimportant):
"""
Based on Scheduler.decide_and_remove_changes.
If we have n or more important changes, we should trigger jobs.
If more than idleTimeout has elapsed since the last change, we should trigger jobs.
"""
if not important:
return None
nImportant = len(important)
if nImportant < self.n:
if not self.idleTimeout:
log.msg("%s: skipping with %i/%i important changes since no idle timeout" %
(self.name, nImportant, self.n))
return
oldest = min([c.when for c in important])
elapsed = int(now() - oldest)
if self.idleTimeout and elapsed < self.idleTimeout:
# Haven't hit the timeout yet, so let's wait more
log.msg("%s: skipping with %i/%i important changes since only %i/%is have elapsed" %
(self.name, nImportant, self.n, elapsed, self.idleTimeout))
return now() + (self.idleTimeout - elapsed)
log.msg("%s: triggering with %i/%i important changes since %is have elapsed" % (self.name, nImportant, self.n, elapsed))
else:
log.msg("%s: triggering since we have %i/%i important changes" % (self.name, nImportant, self.n))
return Scheduler.decide_and_remove_changes(self, t, important, unimportant)
def Update(self, c):
for builder in self._builders:
# Update the schedulers with the builder.
schedulers = builder['schedulers']
if schedulers:
for scheduler in schedulers:
self._schedulers[scheduler]['builders'].append(
builder['name'])
# Construct the category.
categories = []
if builder.get('category', None):
categories.append(builder['category'])
if builder.get('gatekeeper', None):
categories.extend(builder['gatekeeper'].split('|'))
category = '|'.join(categories)
# Append the builder to the list.
new_builder = {
'name': builder['name'],
'factory': self._factories[builder['factory']],
'category': category,
'auto_reboot': builder['auto_reboot']
}
if builder['builddir']:
new_builder['builddir'] = builder['builddir']
c['builders'].append(new_builder)
# Process the main schedulers
for s_name in self._schedulers:
scheduler = self._schedulers[s_name]
if scheduler['type'] == 'Scheduler':
instance = Scheduler(
name=s_name,
branch=scheduler['branch'],
treeStableTimer=scheduler['treeStableTimer'],
builderNames=scheduler['builders'],
categories=scheduler['categories'])
scheduler['instance'] = instance
c['schedulers'].append(instance)
# Process the dependent schedulers
for s_name in self._schedulers:
scheduler = self._schedulers[s_name]
if scheduler['type'] == 'Dependent':
c['schedulers'].append(
Dependent(
s_name,
self._schedulers[scheduler['parent']]['instance'],
scheduler['builders']))
# Process the treggerable schedulers
for s_name in self._schedulers:
scheduler = self._schedulers[s_name]
if scheduler['type'] == 'Triggerable':
c['schedulers'].append(
Triggerable(s_name, scheduler['builders']))
def getSchedulers(builders):
post_commit_builders = [
b['name'] for b in builders if b['name'].startswith('post-commit-')
] + ['page-layout-test-mac-os-x']
post_commit_builders.sort()
periodic_builders = [
b['name'] for b in builders if b['name'].startswith('periodic-')
]
periodic_builders.sort()
post_commit = Scheduler(name="post-commit",
branch=None,
treeStableTimer=90,
builderNames=post_commit_builders)
periodic = Periodic("periodic", periodic_builders, 6 * 60 * 60)
return [post_commit, periodic]
def _create_schedulers(self):
"""Create all the schedulers, and add them to the config.
"""
from buildbot.scheduler import Scheduler, Dependent
scheds = {}
for name in self.sched_order:
item = self.scheduler_builders[name]
kwargs = item[0]
builders = item[1:]
if "depends" in kwargs:
upstream = kwargs['depends']
if upstream not in scheds:
raise ConfigError(
"Scheduler name %r used in depends before being defined"
% name)
upstream = scheds[upstream]
sched = Dependent(name, upstream, builders)
else:
sched = Scheduler(name=name, builderNames=builders, **kwargs)
scheds[name] = sched
self.c['schedulers'].append(sched)
def Update(self, c):
global_schedulers = dict((s.name, s) for s in c['schedulers']
if s.name.startswith('global_'))
assert not (set(global_schedulers) & set(self._schedulers))
for builder in self._builders:
# Update the schedulers with the builder.
schedulers = builder['schedulers']
if schedulers:
for scheduler in schedulers:
if scheduler in global_schedulers:
global_schedulers[scheduler].builderNames.append(
builder['name'])
else:
self._schedulers[scheduler]['builders'].append(
builder['name'])
# Construct the category.
categories = []
if builder.get('category', None):
categories.append(builder['category'])
if builder.get('gatekeeper', None):
categories.extend(builder['gatekeeper'].split('|'))
category = '|'.join(categories)
# Append the builder to the list.
new_builder = {
'name': builder['name'],
'factory': self._factories[builder['factory']],
'category': category,
'auto_reboot': builder['auto_reboot']
}
if builder['builddir']:
new_builder['builddir'] = builder['builddir']
c['builders'].append(new_builder)
# Process the main schedulers.
for s_name in self._schedulers:
scheduler = self._schedulers[s_name]
if scheduler['type'] == 'Scheduler':
instance = Scheduler(
name=s_name,
branch=scheduler['branch'],
treeStableTimer=scheduler['treeStableTimer'],
builderNames=scheduler['builders'],
categories=scheduler['categories'])
scheduler['instance'] = instance
c['schedulers'].append(instance)
# Process the dependent schedulers.
for s_name in self._schedulers:
scheduler = self._schedulers[s_name]
if scheduler['type'] == 'Dependent':
c['schedulers'].append(
Dependent(
s_name,
self._schedulers[scheduler['parent']]['instance'],
scheduler['builders']))
# Process the triggerable schedulers.
for s_name in self._schedulers:
scheduler = self._schedulers[s_name]
if scheduler['type'] == 'Triggerable':
c['schedulers'].append(
Triggerable(s_name, scheduler['builders']))
# Process the periodic schedulers.
for s_name in self._schedulers:
scheduler = self._schedulers[s_name]
if scheduler['type'] == 'Periodic':
c['schedulers'].append(
Periodic(
s_name,
periodicBuildTimer=scheduler['periodicBuildTimer'],
builderNames=scheduler['builders']))
# Process the nightly schedulers.
for s_name in self._schedulers:
scheduler = self._schedulers[s_name]
if scheduler['type'] == 'Nightly':
c['schedulers'].append(
Nightly(s_name,
branch=scheduler['branch'],
hour=scheduler['hour'],
builderNames=scheduler['builders']))
def __init__(self, numberOfBuildsToTrigger=3, **kwargs):
self.numberOfBuildsToTrigger = numberOfBuildsToTrigger
Scheduler.__init__(self, **kwargs)
def SetupBuildersAndSchedulers(c, builders, slaves, ActiveMaster):
"""Set up builders and schedulers for the build master."""
# List of dicts for every builder.
builder_dicts = []
# Builder names by scheduler.
builders_by_scheduler = {s: [] for s in SCHEDULERS}
# Maps a triggering builder to its triggered builders.
triggered_builders = collections.defaultdict(list)
def process_builder(builder, is_trybot=False):
"""Create a dict for the given builder and place its name in the
appropriate scheduler list.
"""
builder_name = builder['name']
if is_trybot:
builder_name = builder_name_schema.TrybotName(builder_name)
# Categorize the builder based on its role.
try:
category = builder_name_schema.DictForBuilderName(
builder_name)['role']
subcategory = builder_name.split(
builder_name_schema.BUILDER_NAME_SEP)[1]
category = '|'.join((category, subcategory))
except ValueError:
# Assume that all builders whose names don't play by our rules are named
# upstream and are therefore canaries.
category = builder_name_schema.BUILDER_ROLE_CANARY
builder_dict = {
'name': builder_name,
'gatekeeper': builder.get('gatekeeper_categories', ''),
'auto_reboot': builder.get('auto_reboot', DEFAULT_AUTO_REBOOT),
'slavenames': slaves.GetSlavesName(builder=builder['name']),
'category': category,
'recipe': builder.get('recipe', DEFAULT_RECIPE),
}
builder_dicts.append(builder_dict)
parent_builder = builder.get('triggered_by')
if parent_builder is not None:
assert builder.get('scheduler') is None
if is_trybot:
parent_builder = builder_name_schema.TrybotName(parent_builder)
triggered_builders[parent_builder].append(builder_name)
elif is_trybot:
builders_by_scheduler[TRY_SCHEDULER_NAME].append(builder_name)
else:
scheduler = builder.get('scheduler', PERCOMMIT_SCHEDULER_NAME)
builders_by_scheduler[scheduler].append(builder_name)
# Create builders and trybots.
for builder in builders:
process_builder(builder)
if builder.get('do_trybot', DEFAULT_DO_TRYBOT):
process_builder(builder, is_trybot=True)
# Verify that all parent builders exist.
all_nontriggered_builders = set(
builders_by_scheduler[PERCOMMIT_SCHEDULER_NAME]).union(
set(builders_by_scheduler[TRY_SCHEDULER_NAME]))
trigger_parents = set(triggered_builders.keys())
nonexistent_parents = trigger_parents - all_nontriggered_builders
if nonexistent_parents:
raise Exception('Could not find parent builders: %s' %
', '.join(nonexistent_parents))
# Create the schedulers.
def trigger_name(parent_builder):
"""Given a parent builder name, return a triggerable scheduler name."""
return 'triggers_%s' % parent_builder
c['schedulers'] = []
s = Scheduler(name=PERCOMMIT_SCHEDULER_NAME,
branch=POLLING_BRANCH,
treeStableTimer=60,
builderNames=builders_by_scheduler[PERCOMMIT_SCHEDULER_NAME])
c['schedulers'].append(s)
s = timed.Nightly(
name=PERIODIC_15MINS_SCHEDULER_NAME,
branch=POLLING_BRANCH,
builderNames=builders_by_scheduler[PERIODIC_15MINS_SCHEDULER_NAME],
minute=[i * 15 for i in xrange(60 / 15)],
hour='*',
dayOfMonth='*',
month='*',
dayOfWeek='*')
c['schedulers'].append(s)
for parent, builders_to_trigger in triggered_builders.iteritems():
c['schedulers'].append(
Triggerable(name=trigger_name(parent),
builderNames=builders_to_trigger))
pools = BuildersPools(TRY_SCHEDULER_NAME)
pools[TRY_SCHEDULER_NAME].extend(builders_by_scheduler[TRY_SCHEDULER_NAME])
c['schedulers'].append(
TryJobRietveld(name=TRY_SCHEDULER_NAME,
code_review_sites={
TRY_SCHEDULER_PROJECT: ActiveMaster.code_review_site
},
pools=pools,
project=TRY_SCHEDULER_PROJECT,
filter_master=True))
# Create the BuildFactorys.
annotator = annotator_factory.AnnotatorFactory()
for builder_dict in builder_dicts:
triggers = ([trigger_name(builder_dict['name'])]
if builder_dict['name'] in triggered_builders else None)
builder_dict['factory'] = annotator.BaseFactory(builder_dict['recipe'],
triggers=triggers)
# Finished!
c['builders'] = builder_dicts
def Update(self, c):
for builder in self._builders:
# Update the schedulers with the builder.
schedulers = builder['schedulers']
if schedulers:
for scheduler in schedulers:
self._schedulers[scheduler]['builders'].append(builder['name'])
# Construct the category.
categories = []
if builder.get('category', None):
categories.append(builder['category'])
if builder.get('gatekeeper', None):
categories.extend(builder['gatekeeper'].split('|'))
category = '|'.join(categories)
# Append the builder to the list.
new_builder = {'name': builder['name'],
'factory': self._factories[builder['factory']],
'category': category,
'auto_reboot': builder['auto_reboot']}
if builder['builddir']:
new_builder['builddir'] = builder['builddir']
c['builders'].append(new_builder)
c['builders'].sort(key=lambda builder: builder['name'])
# Process the main schedulers.
for s_name in self._schedulers:
scheduler = self._schedulers[s_name]
if scheduler['type'] == 'Scheduler':
def filter_fn(change, builders):
"""Filters out if change.comments contains certain keywords.
The change is filtered out if the commit message contains:
* SKIP_BUILDBOT_SUBSTRING or
* RUN_BUILDERS_REGEX when the scheduler does not contain any of the
specified builders
Args:
change: An instance of changes.Change.
builders: Sequence of strings. The builders that are run by this
scheduler.
Returns:
If the change should be filtered out (i.e. not run by the buildbot
code) then False is returned else True is returned.
"""
if SKIP_BUILDBOT_SUBSTRING in change.comments:
return False
match_obj = RUN_BUILDERS_RE_COMPILED.search(change.comments)
if builders and match_obj:
for builder_to_run in match_obj.group(1).split(','):
if builder_to_run.strip() in builders:
break
else:
return False
return True
skia_change_filter = SkiaChangeFilter(
builders=scheduler['builders'],
branch=skia_vars.GetGlobalVariable('master_branch_name'),
filter_fn=filter_fn)
instance = Scheduler(name=s_name,
treeStableTimer=scheduler['treeStableTimer'],
builderNames=scheduler['builders'],
change_filter=skia_change_filter)
c['schedulers'].append(instance)
self._schedulers[s_name]['instance'] = instance
# Process the periodic schedulers.
for s_name in self._schedulers:
scheduler = self._schedulers[s_name]
if scheduler['type'] == 'PeriodicScheduler':
instance = timed.Nightly(
name=s_name,
branch=skia_vars.GetGlobalVariable('master_branch_name'),
builderNames=scheduler['builders'],
minute=scheduler['minute'],
hour=scheduler['hour'],
dayOfMonth=scheduler['dayOfMonth'],
month=scheduler['month'],
dayOfWeek=scheduler['dayOfWeek'])
c['schedulers'].append(instance)
self._schedulers[s_name]['instance'] = instance
# Process the Rietveld-based try schedulers.
for s_name in self._schedulers:
scheduler = self._schedulers[s_name]
if scheduler['type'] == 'TryJobRietveld':
pools = BuildersPools(s_name)
pools[s_name].extend(scheduler['builders'])
instance = try_job_rietveld.TryJobRietveld(
name=s_name,
pools=pools,
last_good_urls={'skia': None},
code_review_sites={'skia': config_private.CODE_REVIEW_SITE},
project='skia')
c['schedulers'].append(instance)
self._schedulers[s_name]['instance'] = instance
# Process the svn-based try schedulers.
for s_name in self._schedulers:
scheduler = self._schedulers[s_name]
if scheduler['type'] == 'TryJobSubversion':
pools = BuildersPools(s_name)
pools[s_name].extend(scheduler['builders'])
instance = try_job_svn.TryJobSubversion(
name=s_name,
svn_url=TRY_SVN_BASEURL,
last_good_urls={'skia': None},
code_review_sites={'skia': config_private.CODE_REVIEW_SITE},
pools=pools)
c['schedulers'].append(instance)
self._schedulers[s_name]['instance'] = instance
# Process the dependent schedulers.
for s_name in self._schedulers:
scheduler = self._schedulers[s_name]
if scheduler['type'] == 'Dependent':
instance = Dependent(
s_name,
self._schedulers[scheduler['parent']]['instance'],
scheduler['builders'])
c['schedulers'].append(instance)
self._schedulers[s_name]['instance'] = instance
def __init__(self, name, **kwargs):
self.unimportantChanges = []
self.keeper = kwargs.pop('keeper', None)
Scheduler.__init__(self, name=name, **kwargs)
def _VersionFileFilter(change):
"""A change filter function that disregards all changes that don't
touch src/syzygy/VERSION.
Args:
change: a buildbot Change object.
"""
return change.branch == 'trunk' and 'syzygy/VERSION' in change.files
#
# Official build scheduler for Syzygy
#
official_scheduler = Scheduler(
'syzygy_version',
treeStableTimer=0,
change_filter=ChangeFilter(filter_fn=_VersionFileFilter),
builderNames=['Syzygy Official'])
#
# Windows official Release builder
#
official_factory = win().SyzygyFactory(official_release=True)
official_builder = {
'name': 'Syzygy Official',
'factory': official_factory,
'schedulers': 'syzygy_version',
'auto_reboot': False,
'category': 'official',
}
def SetupBuildersAndSchedulers(c, builders, slaves, ActiveMaster):
"""Set up builders and schedulers for the build master."""
# List of dicts for every builder.
builder_dicts = []
# Builder names by scheduler.
builders_by_scheduler = {s: [] for s in SCHEDULERS}
# Maps a triggering builder to its triggered builders.
triggered_builders = collections.defaultdict(list)
def process_builder(builder, is_trybot=False):
"""Create a dict for the given builder and place its name in the
appropriate scheduler list.
"""
builder_name = builder['name']
if is_trybot:
builder_name = builder_name_schema.TrybotName(builder_name)
# Categorize the builder based on its role.
try:
category = builder_name_schema.DictForBuilderName(
builder_name)['role']
subcategory = builder_name.split(
builder_name_schema.BUILDER_NAME_SEP)[1]
category = '|'.join((category, subcategory))
except ValueError:
# Assume that all builders whose names don't play by our rules are named
# upstream and are therefore canaries.
category = builder_name_schema.BUILDER_ROLE_CANARY
properties = builder.get('properties', {})
cc = builder.get('cc')
if cc:
if isinstance(cc, basestring):
cc = [cc]
properties['owners'] = cc
builder_dict = {
'name':
builder_name,
'auto_reboot':
builder.get('auto_reboot', DEFAULT_AUTO_REBOOT),
'slavenames':
slaves.GetSlavesName(builder=builder['name']),
'category':
category,
'recipe':
builder.get('recipe', DEFAULT_RECIPE),
'properties':
properties,
'mergeRequests':
builder.get('can_merge_requests', CanMergeBuildRequests),
'slavebuilddir':
SLAVE_WORKDIR,
}
builder_dicts.append(builder_dict)
parent_builder = builder.get('triggered_by')
if parent_builder is not None:
assert builder.get('scheduler') is None
if is_trybot:
parent_builder = builder_name_schema.TrybotName(parent_builder)
triggered_builders[parent_builder].append(builder_name)
else:
scheduler = builder.get('scheduler', BUILDBUCKET_SCHEDULER_NAME)
# Setting the scheduler to BUILDBUCKET_SCHEDULER_NAME indicates that
# BuildBucket is the only way to schedule builds for this bot; just
# pretend to add a scheduler in those cases.
builders_by_scheduler[scheduler].append(builder_name)
# Create builders and trybots.
for builder in builders:
if builder.get('trybot_only', DEFAULT_TRYBOT_ONLY):
# trybot_only=True should only be used in combination with do_trybot=True
# Also, the buildername then needs to already have the '-Trybot' suffix.
assert builder.get('do_trybot', DEFAULT_DO_TRYBOT)
assert builder['name'] == builder_name_schema.TrybotName(
builder['name'])
else:
process_builder(builder)
if builder.get('do_trybot', DEFAULT_DO_TRYBOT):
process_builder(builder, is_trybot=True)
# Verify that all parent builders exist.
all_nontriggered_builders = set(
builders_by_scheduler[BUILDBUCKET_SCHEDULER_NAME])
trigger_parents = set(triggered_builders.keys())
nonexistent_parents = trigger_parents - all_nontriggered_builders
if nonexistent_parents:
raise Exception('Could not find parent builders: %s' %
', '.join(nonexistent_parents))
# Create the schedulers.
infra_change_filter = change_filter.ChangeFilter(
project='buildbot', repository=global_constants.INFRA_REPO)
skia_master_only_change_filter = change_filter.ChangeFilter(
project='skia', repository=ActiveMaster.repo_url, branch=MASTER_BRANCH)
c['schedulers'] = []
s = Scheduler(
name=MASTER_ONLY_SCHEDULER_NAME,
treeStableTimer=60,
change_filter=skia_master_only_change_filter,
builderNames=builders_by_scheduler[MASTER_ONLY_SCHEDULER_NAME])
c['schedulers'].append(s)
s = timed.Nightly(
name=PERIODIC_15MINS_SCHEDULER_NAME,
branch=MASTER_BRANCH,
builderNames=builders_by_scheduler[PERIODIC_15MINS_SCHEDULER_NAME],
minute=[i * 15 for i in xrange(60 / 15)],
hour='*',
dayOfMonth='*',
month='*',
dayOfWeek='*')
c['schedulers'].append(s)
s = timed.Nightly(
name=NIGHTLY_SCHEDULER_NAME,
branch=MASTER_BRANCH,
builderNames=builders_by_scheduler[NIGHTLY_SCHEDULER_NAME],
minute=0,
hour=22,
dayOfMonth='*',
month='*',
dayOfWeek='*')
c['schedulers'].append(s)
s = timed.Nightly(
name=WEEKLY_SCHEDULER_NAME,
branch=MASTER_BRANCH,
builderNames=builders_by_scheduler[WEEKLY_SCHEDULER_NAME],
minute=0,
hour=0,
dayOfMonth='*',
month='*',
dayOfWeek=6) # Sunday (Monday = 0).
c['schedulers'].append(s)
s = AnyBranchScheduler(
name=INFRA_PERCOMMIT_SCHEDULER_NAME,
treeStableTimer=0,
change_filter=infra_change_filter,
builderNames=builders_by_scheduler[INFRA_PERCOMMIT_SCHEDULER_NAME])
c['schedulers'].append(s)
# Don't add triggerable schedulers for triggered_builders; triggers are now
# handled on the slave-side through buildbucket.
# Create the BuildFactorys.
annotator = annotator_factory.AnnotatorFactory(ActiveMaster)
for builder_dict in builder_dicts:
factory = annotator.BaseFactory(builder_dict['recipe'], timeout=2400)
factory.properties.update(builder_dict['properties'], 'BuildFactory')
builder_dict['factory'] = factory
# Finished!
c['builders'] = builder_dicts
def __init__(self, repo_names, *args, **kwargs):
Scheduler.__init__(self, *args, **kwargs)
self.repo_names = repo_names
def __init__(self, name, **kwargs):
self.unimportantChanges = []
self.keeper = kwargs.pop('keeper', None)
Scheduler.__init__(self, name=name, **kwargs)
def AddSchedulersAndTriggers(buildmaster_config=None,
slave_list=None,
scheduler_name=None,
branch=None):
"""Adds schedulers and triggers to the BuildmasterConfig based on
the contents of slaves.cfg, passed in as a slave_list.
This function relies on certain structure in the slaves.cfg, in
particular the custom 'triggered_by' property, which is not yet
commonly used to define triggers.
Returns a dictionary mapping builder name, for those builders which
invoke triggers, to the (synthesized) name of the trigger.
TODO(kbr): this function does not yet support builders with
multiple slaves behind them, but could be updated to do so.
Throws an Exception if a non-existent builder is mentioned in
another builder's 'triggered_by' property.
Arguments:
buildmaster_config: a BuildmasterConfig into which the
'schedulers' property will be defined.
slave_list: a SlavesList constructed from slaves.cfg.
scheduler_name: the name of the Scheduler for the polling (not
triggered) builders.
"""
c = buildmaster_config
polling_builders = []
# Maps the parent builder to a set of the names of the builders it triggers.
trigger_map = collections.defaultdict(list)
# Maps the name of the parent builder to the (synthesized) name of its
# trigger, wrapped in a list.
trigger_name_map = {}
next_group_id = 0
for slave in slave_list.slaves:
builder = slave['builder']
parent_builder = slave.get('triggered_by')
if parent_builder is not None:
if slave_list.GetSlave(builder=parent_builder) is None:
raise Exception(
'Could not find parent builder %s for builder %s' %
(parent_builder, builder))
trigger_map[parent_builder].append(builder)
if parent_builder not in trigger_name_map:
trigger_name_map[
parent_builder] = 'trigger_group_%d' % next_group_id
next_group_id += 1
else:
polling_builders.append(builder)
s_gpu = Scheduler(name=scheduler_name,
branch=branch,
treeStableTimer=60,
builderNames=polling_builders)
c['schedulers'] = [s_gpu]
for name, builders in trigger_map.iteritems():
c['schedulers'].append(
Triggerable(name=trigger_name_map[name], builderNames=builders))
return trigger_name_map
def addChange(self, change):
if (not hasattr(change, "repo_name") or
change.repo_name in self.repo_names):
return Scheduler.addChange(self, change)
def __init__(self, name, n, idleTimeout=None, **kwargs):
self.n = n
self.idleTimeout = idleTimeout
Scheduler.__init__(self, name=name, **kwargs)
def Update(_config, active_master, c):
c['schedulers'].extend([
Scheduler(name='linux_memory_rel',
branch='master',
treeStableTimer=60,
builderNames=[
'Linux ASan LSan Builder',
'Linux ChromiumOS MSan Builder',
'Linux MSan Builder',
'Linux TSan Builder',
]),
Triggerable(name='linux_asan_rel_trigger',
builderNames=[
'Linux ASan LSan Tests (1)',
'Linux ASan Tests (sandboxed)',
]),
Triggerable(name='linux_chromiumos_msan_rel_trigger',
builderNames=['Linux ChromiumOS MSan Tests']),
Triggerable(name='linux_msan_rel_trigger',
builderNames=['Linux MSan Tests']),
Triggerable(name='linux_tsan_rel_trigger',
builderNames=['Linux TSan Tests']),
])
specs = [
{
'name': 'Linux ASan LSan Builder',
'triggers': ['linux_asan_rel_trigger'],
'category': '1linux asan lsan msan',
},
{
'name': 'Linux ASan LSan Tests (1)',
'category': '1linux asan lsan msan',
},
{
'name': 'Linux ASan Tests (sandboxed)',
'category': '1linux asan lsan msan',
},
{
'name': 'Linux ChromiumOS MSan Builder',
'triggers': ['linux_chromiumos_msan_rel_trigger'],
'category': '1linux asan lsan msan',
},
{
'name': 'Linux ChromiumOS MSan Tests',
'category': '1linux asan lsan msan',
},
{
'name': 'Linux MSan Builder',
'triggers': ['linux_msan_rel_trigger'],
'category': '1linux asan lsan msan',
},
{
'name': 'Linux MSan Tests',
'category': '1linux asan lsan msan',
},
{
'name': 'Linux TSan Builder',
'triggers': ['linux_tsan_rel_trigger'],
'category': '10TSan v2|compile',
},
{
'name': 'Linux TSan Tests',
'category': '10TSan v2',
},
]
c['builders'].extend([{
'name':
spec['name'],
'factory':
m_remote_run('chromium', triggers=spec.get('triggers')),
'notify_on_missing':
True,
'category':
spec['category'],
} for spec in specs])
def __init__(self, name, n, idleTimeout=None, **kwargs):
self.n = n
self.idleTimeout = idleTimeout
Scheduler.__init__(self, name=name, **kwargs)
def __init__(self, name, builderNames, repository):
Scheduler.__init__(
self, name, None, 120, builderNames, fileIsImportant=None)
self.repository = repository
def __init__(self, numberOfBuildsToTrigger=3, **kwargs):
self.numberOfBuildsToTrigger = numberOfBuildsToTrigger
Scheduler.__init__(self, **kwargs)
def __init__(self, name, branch, treeStableTimer, builderNames,
fileIsImportant=None, properties={}):
Scheduler.__init__(self, name=name, branch=branch, treeStableTimer=treeStableTimer, builderNames=builderNames,
fileIsImportant=fileIsImportant, properties=properties)
