def fireTimer(self):
# clear out our state
self.timer = None
self.nextBuildTime = None
# Only build changes that are deemed important by self.fileIsImportant
changes = self.importantChanges
self.importantChanges = []
self.unimportantChanges = []
# Only add builders to the buildset that are currently online
buildset_builderNames = []
for builder_name in self.builderNames:
# Get a builder from the BotMaster:
try:
builder = self.parent.botmaster.builders.get(builder_name)
except AttributeError:
# when this scheduler is used by the AnyBranchScheduler, botmaster is one level up from normal
builder = self.parent.parent.botmaster.builders.get(builder_name)
if builder.builder_status.getState()[0] == 'idle':
buildset_builderNames.append(builder_name)
# create a BuildSet, submit it to the BuildMaster
bs = buildset.BuildSet(buildset_builderNames,
SourceStamp(changes=changes),
properties=self.properties)
self.submitBuildSet(bs)
def startService(self):
BaseUpstreamScheduler.startService(self)
log.msg("starting l10n scheduler")
if self.inipath is None:
# testing, don't trigger tree builds
return
# trigger tree builds for our trees, clear() first
cp = ConfigParser()
cp.read(self.inipath)
self.trees.clear()
_ds = []
for tree in cp.sections():
# create a BuildSet, submit it to the BuildMaster
props = properties.Properties()
props.update({
'tree': tree,
'l10nbuilds': self.inipath,
},
"Scheduler")
bs = buildset.BuildSet([self.treebuilder],
SourceStamp(),
properties=props)
self.submitBuildSet(bs)
_ds.append(bs.waitUntilFinished())
d = defer.DeferredList(_ds)
d.addCallback(self.onTreesBuilt)
self.waitOnTree = d
def perspective_try(self, branch, revision, patch, builderNames, properties={}):
log.msg("user %s requesting build on builders %s" % (self.username,
builderNames))
for b in builderNames:
if not b in self.parent.builderNames:
log.msg("%s got job with builder %s" % (self, b))
log.msg(" but that wasn't in our list: %s"
% (self.parent.builderNames,))
return
ss = SourceStamp(branch, revision, patch)
reason = "'try' job from user %s" % self.username
# roll the specified props in with our inherited props
combined_props = Properties()
combined_props.updateFromProperties(self.parent.properties)
combined_props.update(properties, "try build")
bs = buildset.BuildSet(builderNames,
ss,
reason=reason,
properties=combined_props)
self.parent.submitBuildSet(bs)
# return a remotely-usable BuildSetStatus object
from buildbot.status.client import makeRemote
return makeRemote(bs.status)
def trigger(self, ss):
"""Trigger this scheduler. Returns a deferred that will fire when the
buildset is finished.
"""
bs = buildset.BuildSet(self.builderNames, ss)
d = bs.waitUntilFinished()
self.submit(bs)
return d
def doPeriodicBuild(self):
# Schedule the next run
self.setTimer()
# And trigger a build
bs = buildset.BuildSet(self.builderNames,
SourceStamp(branch=self.branch), self.reason)
self.submit(bs)
def fireTimer(self):
# clear out our state
self.timer = None
self.nextBuildTime = None
changes = self.importantChanges + self.unimportantChanges
self.importantChanges = []
self.unimportantChanges = []
# create a BuildSet, submit it to the BuildMaster
bs = buildset.BuildSet(self.builderNames, SourceStamp(changes=changes))
self.submit(bs)
def fireTimer(self):
self.timer = None
self.nextBuildTime = None
changes = self.importantChanges + self.unimportantChanges
self.importantChanges = []
self.unimportantChanges = []
# submit
ss = noMergeSourceStamp(changes=changes)
bs = buildset.BuildSet(self.builderNames, ss)
self.submit(bs)
def testGetBuildSets(self):
# validate IStatus.getBuildSets
s = status.builder.Status(None, ".")
bs1 = buildset.BuildSet(["a", "b"],
sourcestamp.SourceStamp(),
reason="one",
bsid="1")
s.buildsetSubmitted(bs1.status)
self.failUnlessEqual(s.getBuildSets(), [bs1.status])
bs1.status.notifyFinishedWatchers()
self.failUnlessEqual(s.getBuildSets(), [])
def fireTimer(self):
self.timer = None
self.nextBuildTime = None
changes = self.importantChanges + self.unimportantChanges
self.importantChanges = []
self.unimportantChanges = []
# submit
for i in range(0, self.numberOfBuildsToTrigger):
bs = buildset.BuildSet(self.builderNames,
SourceStamp(changes=changes))
self.submit(bs)
def trigger(self, ss, set_props=None):
"""Trigger this scheduler. Returns a deferred that will fire when the
buildset is finished.
"""
# properties for this buildset are composed of our own properties,
# potentially overridden by anything from the triggering build
props = Properties()
props.updateFromProperties(self.properties)
if set_props: props.updateFromProperties(set_props)
bs = buildset.BuildSet(self.builderNames, ss, properties=props)
d = bs.waitUntilFinished()
self.submitBuildSet(bs)
return d
def perspective_try(self, branch, revision, patch, builderNames):
log.msg("user %s requesting build on builders %s" %
(self.username, builderNames))
for b in builderNames:
if not b in self.parent.builderNames:
log.msg("%s got job with builder %s" % (self, b))
log.msg(" but that wasn't in our list: %s" %
(self.parent.builderNames, ))
return
ss = SourceStamp(branch, revision, patch)
reason = "'try' job from user %s" % self.username
bs = buildset.BuildSet(builderNames, ss, reason=reason)
self.parent.submitBuildSet(bs)
# return a remotely-usable BuildSetStatus object
from buildbot.status.client import makeRemote
return makeRemote(bs.status)
def fireTimer(self):
# clear out our state
self.timer = None
self.nextBuildTime = None
changes = self.unimportantChanges + self.importantChanges
self.importantChanges = []
self.unimportantChanges = []
buildset_builderNames = []
# There may be multiple build requests, only process the very last one (latest)
change = changes[-1]
# Only add builders that are:
# 1) dependent builder is listed in the change request, this means that the upstream builder was online
# 2) builder is online
# determine the priority based on branch
# note that this is a list comprehension, so we need to "unpack" with
# [0] at end
priority = [p[1] for p in self.priorities if p[0] == change.branch][0]
f = open("%s/change-%s.%s" % (self.changeDir, change.revision, priority))
phase1Builders = []
for line in f.readlines():
if line.startswith("builders:"):
phase1Builders = line[line.find(":")+1:].strip().split()
break
# builders is a list of builders that built the request in Phase1
for p1Builder in phase1Builders:
# Get the depenent builder
for dependent in self.builderDependencies:
if p1Builder == dependent[1]:
p2Builder = dependent[0]
# See if the builder is online
# Get a builder from the BotMaster:
builder = self.parent.botmaster.builders.get(p2Builder)
# Add the builder to the set if it is idle (not building and not offline)
if builder.builder_status.getState()[0] == 'idle':
buildset_builderNames.append(p2Builder)
# create a BuildSet, submit it to the BuildMaster
bs = buildset.BuildSet(buildset_builderNames,
SourceStamp(changes=changes),
properties=self.properties)
self.submitBuildSet(bs)
def submitBuildsets(self):
for tpl, changes in self.pendings.iteritems():
tree, locale = tpl
_t = self.trees[tree]
props = properties.Properties()
# figure out the latest change
try:
when = timeHelper(max(filter(None, (c.when for c in changes))))
from life.models import Push
except (ValueError, ImportError):
when = None
for k, v in _t.branches.iteritems():
props.setProperty(k+"_branch", v, "Scheduler")
_r = "default"
if when is not None:
if k == 'l10n':
repo = '%s/%s' % (v, locale)
else:
repo = v
q = Push.objects.filter(repository__name=repo,
push_date__lte=when,
changesets__branch__name='default')
try:
# get the latest changeset on the 'default' branch
# not strictly .tip, for pushes with heads on
# multiple branches (bug 602182)
_c = q.order_by('-pk')[0].changesets.order_by('-pk')
_r = str(_c.filter(branch__name='default')[0].shortrev)
except IndexError:
# no pushes, update to empty repo 000000000000
_r = "default"
props.setProperty(k+"_revision", _r, "Scheduler")
props.update({"tree": tree,
"locale": locale,
"l10n.ini": _t.l10ninis[_t.branches['en']][0],
"revisions": sorted(_t.branches.keys()),
},
"Scheduler")
bs = buildset.BuildSet(self.builderNames,
SourceStamp(changes=changes),
properties=props)
self.submitBuildSet(bs)
self.dSubmitBuildsets = None
self.pendings.clear()
def queueBuild(self, locale, change):
props = properties.Properties()
props.update({'locale': locale,
'tree': self.tree,
'branch': self.branch,
'repourl': self.repourl,
'refpath': self.enBranch + '/en-US',
'en_revision': 'default',
'en_branch': self.enBranch + '/en-US',
'l10npath': self.branch + '/' + locale,
'l10n_revision': 'default',
'l10n_branch': self.branch,
},
'Scheduler')
ss = SourceStamp(changes=[change])
bs = buildset.BuildSet(self.builderNames, ss,
reason = "%s %s" % (self.tree, locale),
properties = props)
self.submitBuildSet(bs)
def fireTimer(self):
# clear out our state
self.timer = None
self.nextBuildTime = None
changes = self.unimportantChanges + self.importantChanges
self.importantChanges = []
self.unimportantChanges = []
# Only add builders to the buildset that are currently online
buildset_builderNames = []
for builder_name in self.builderNames:
# Get a builder from the BotMaster:
builder = self.parent.botmaster.builders.get(builder_name)
if builder.builder_status.getState()[0] == 'idle':
buildset_builderNames.append(builder_name)
# create a BuildSet, submit it to the BuildMaster
bs = buildset.BuildSet(buildset_builderNames,
SourceStamp(changes=changes),
properties=self.properties)
self.submitBuildSet(bs)
def messageReceived(self, filename):
md = os.path.join(self.parent.basedir, self.jobdir)
if runtime.platformType == "posix":
# open the file before moving it, because I'm afraid that once
# it's in cur/, someone might delete it at any moment
path = os.path.join(md, "new", filename)
f = open(path, "r")
os.rename(os.path.join(md, "new", filename),
os.path.join(md, "cur", filename))
else:
# do this backwards under windows, because you can't move a file
# that somebody is holding open. This was causing a Permission
# Denied error on bear's win32-twisted1.3 buildslave.
os.rename(os.path.join(md, "new", filename),
os.path.join(md, "cur", filename))
path = os.path.join(md, "cur", filename)
f = open(path, "r")
try:
builderNames, ss, bsid = self.parseJob(f)
except BadJobfile:
log.msg("%s reports a bad jobfile in %s" % (self, filename))
log.err()
return
# compare builderNames against self.builderNames
# TODO: think about this some more.. why bother restricting it?
# perhaps self.builderNames should be used as the default list
# instead of being used as a restriction?
for b in builderNames:
if not b in self.builderNames:
log.msg("%s got jobfile %s with builder %s" % (self,
filename, b))
log.msg(" but that wasn't in our list: %s"
% (self.builderNames,))
return
reason = "'try' job"
bs = buildset.BuildSet(builderNames, ss, reason=reason,
bsid=bsid, properties=self.properties)
self.submitBuildSet(bs)
def submitBuildsets(self):
connection.close_if_unusable_or_obsolete()
log.msg('submitting %d pending buildsets' % len(self.pendings))
for tpl, changes in self.pendings.iteritems():
tree, locale = tpl
_t = self.trees[tree]
props = properties.Properties()
# figure out the latest change
try:
when = timeHelper(max(filter(None, (c.when for c in changes))))
except (ValueError, ImportError):
when = None
revisions = sorted(_t.branches.keys())
for k, v in _t.branches.iteritems():
_r = "000000000000"
if k == 'l10n':
repo = '%s/%s' % (v, locale)
else:
repo = v
try:
repo = Repository.objects.get(name=repo)
except Repository.DoesNotExist:
log.msg('Repository %s does not exist, skipping' % repo)
revisions.remove(k)
continue
q = Push.objects.filter(repository=repo,
changesets__branch__name='default')
if when is not None:
q = q.filter(push_date__lte=when)
try:
# get the latest changeset on the 'default' branch
# not strictly .tip, for pushes with heads on
# multiple branches (bug 602182)
_p = q.order_by('-pk')[0]
if _p.push_date:
if not when:
when = _p.push_date
else:
when = max(when, _p.push_date)
_c = _p.changesets.order_by('-pk')
_r = str(_c.filter(branch__name='default')[0].revision)
except IndexError:
# no pushes, try to get a good Changeset.
# this is guaranteed to at least return the null changeset
_r = str(
repo.changesets
.filter(branch__name='default')
.order_by('-pk')
.values_list('revision', flat=True)[0])
relpath = repo.relative_path()
props.setProperty(k+"_branch", relpath,
"Scheduler")
if relpath != repo.name:
props.setProperty("local_" + repo.name, relpath,
"Scheduler")
props.setProperty(k+"_revision", _r, "Scheduler")
_f = Forest.objects.get(name=_t.branches['l10n'])
# use the relative path of the en repo we got above
inipath = '{}/{}'.format(
props['en_branch'],
_t.l10ninis[_t.branches['en']][0])
props.update({"tree": tree,
"l10nbase": _f.relative_path(),
"locale": locale,
"inipath": inipath,
"srctime": when,
"revisions": revisions,
},
"Scheduler")
bs = buildset.BuildSet(self.builderNames,
SourceStamp(changes=changes),
properties=props)
self.submitBuildSet(bs)
log.msg('one buildset successfully submitted')
self.dSubmitBuildsets = None
self.pendings.clear()
def addChange(self, change):
'''Main entry point for the scheduler, this is called by the
buildmaster.
'''
log.msg("addChange appscheduler, %s" % str(self.waitOnTree))
if self.waitOnTree is not None:
# a tree build is currently running, wait with this
# until we're done with it
self.pendingChanges.append(change)
return
# fixup change.locale if property is given
if not hasattr(change, 'locale') or not change.locale:
if 'locale' in change.properties:
change.locale = change.properties['locale']
log.msg("locale: %s" % getattr(change, 'locale', 'none'))
if not hasattr(change, 'locale') or not change.locale:
# check branch, l10n.inis
# if l10n.inis are found, callback to all-locales, locales/en-US
# otherwise just check those straight away
if change.branch not in self.branches:
log.msg('not our branches')
return
tree_triggers = set()
branchdata = self.branches[change.branch]
for f in change.files:
if f in branchdata.inis:
tree_triggers.update(branchdata.inis[f])
if tree_triggers:
# trigger tree builds, wait for them to finish
# and check the change for en-US builds
_ds = []
for _n in tree_triggers:
props = properties.Properties()
props.update({
'tree': _n,
'l10nbuilds': self.inipath,
},
"Scheduler")
bs = buildset.BuildSet([self.treebuilder],
SourceStamp(branch=change.branch,
changes=[change]),
properties=props)
self.submitBuildSet(bs)
_ds.append(bs.waitUntilFinished())
d = defer.DeferredList(_ds)
d.addCallback(self.onTreesBuilt,
branchdata=branchdata, change=change)
self.waitOnTree = d
return
self.checkEnUS(None, branchdata, change)
return
# check l10n changesets
log.msg('my branch: %s, in? %s' %
(change.branch, ','.join(sorted(self.l10nbranches.keys()))))
if change.branch not in self.l10nbranches:
return
l10ndirs = self.l10nbranches[change.branch]
log.msg('yes, dirs: %s' % ','.join(sorted(l10ndirs)))
trees = set()
for f in change.files:
for mod, _trees in l10ndirs.iteritems():
if f.startswith(mod):
trees |= _trees
for _n in trees:
if change.locale in self.trees[_n].locales:
self.compareBuild(_n, change.locale, [change])
else:
log.msg('%s not in tree %s, needs %s' % (
change.locale,
_n,
','.join(sorted(self.trees[_n].locales))))
return
def upstreamBuilt(self, ss):
bs = buildset.BuildSet(self.builderNames, ss)
self.submit(bs)
def upstreamBuilt(self, ss):
bs = buildset.BuildSet(self.builderNames, SourceStamp(branch=self.branch))
self.submitBuildSet(bs)
def upstreamBuilt(self, ss):
# Only add builders to the buildset that are currently online and if the dependent builder
# also built this change.
# If a fileIsImportant function is defined, check the files before determining buildsets
if self.fileIsImportant:
startThisBuildSet = False
# SourceStamp provides us with a tuple of changes, loop through and check them all
for change in ss.changes:
if self.fileIsImportant(change):
# If any file is important, we start the buildSet
startThisBuildSet = True
break
if not startThisBuildSet:
# no important files found, do not continue with this buildset
return
if self.source_stamp == None:
# This is the first time that the ss has been seen, this is NOT a callback
self.source_stamp = ss
else:
# This can either be a callback or a new ss passed in from upstream builder
try:
revision_prev = int(self.source_stamp.revision)
revision_curr = int(ss.revision)
except ValueError:
errmsg = "BuilderDependent.upstreamBuilt has an unknown ss.revison: %s" % (ss.revision)
assert False, errmsg
# ss.revision is an OLDER REV than self.source_stamp.revision
if revision_curr < revision_prev:
# This means we are a callback, and during sleep period a new build request
# has come in and is now in control, we can just stop running.
return
# ss.revision is a NEWER REV than self.source_stamp.revision
elif revision_curr > revision_prev:
# This would indicate a new build and we need to merge ss and self.source_stamp
ss.mergeWith([self.source_stamp])
self.source_stamp = ss
# Revisions are the same so this is the callback
# else revision_curr == revision_prev
buildset_builderNames = []
for builder_name in self.builderNames:
# Get the depenent builder
for dependent in self.builderDependencies:
if builder_name == dependent[0]:
dependent_builder = dependent[1]
break
# Get the dependent builder
dep_builder = self.parent.getStatus().getBuilder(dependent_builder)
# Determine the revision of the last build for the dependent builder
# need to make sure that the builder has at least one previous build
try:
dep_ss = dep_builder.getLastFinishedBuild().getSourceStamp()
except AttributeError:
dep_ss = SourceStamp()
# Only continue the check if the dependent builder built the sourceStamp,
if ss.revision == dep_ss.revision:
# Get a builder from the BotMaster:
builder = self.parent.botmaster.builders.get(builder_name)
if builder.builder_status.getState()[0] == 'building':
# There is a builder in this scheduler that is active so we need
# to NOT start the build yet, but instead callback in X seconds
# to see if all of the builders are available.
self.timer = reactor.callLater(self.callbackInterval, self.upstreamBuilt, ss)
return
# Add the builder to the set if it is idle (not building and not offline)
if builder.builder_status.getState()[0] == 'idle':
buildset_builderNames.append(builder_name)
bs = buildset.BuildSet(buildset_builderNames, ss,
properties=self.properties)
self.submitBuildSet(bs)
# Clear the tracked source stamp
self.source_stamp = None
def doPeriodicBuild(self):
bs = buildset.BuildSet(self.builderNames,
SourceStamp(branch=self.branch),
self.reason,
properties=self.properties)
self.submitBuildSet(bs)
def upstreamBuilt(self, ss):
bs = buildset.BuildSet(self.builderNames, ss,
properties=self.properties)
self.submitBuildSet(bs)
