def fireTimer(self): # clear out our state self.timer = None self.nextBuildTime = None # Only build changes that are deemed important by self.fileIsImportant changes = self.importantChanges self.importantChanges = [] self.unimportantChanges = [] # Only add builders to the buildset that are currently online buildset_builderNames = [] for builder_name in self.builderNames: # Get a builder from the BotMaster: try: builder = self.parent.botmaster.builders.get(builder_name) except AttributeError: # when this scheduler is used by the AnyBranchScheduler, botmaster is one level up from normal builder = self.parent.parent.botmaster.builders.get(builder_name) if builder.builder_status.getState()[0] == 'idle': buildset_builderNames.append(builder_name) # create a BuildSet, submit it to the BuildMaster bs = buildset.BuildSet(buildset_builderNames, SourceStamp(changes=changes), properties=self.properties) self.submitBuildSet(bs)
def startService(self): BaseUpstreamScheduler.startService(self) log.msg("starting l10n scheduler") if self.inipath is None: # testing, don't trigger tree builds return # trigger tree builds for our trees, clear() first cp = ConfigParser() cp.read(self.inipath) self.trees.clear() _ds = [] for tree in cp.sections(): # create a BuildSet, submit it to the BuildMaster props = properties.Properties() props.update({ 'tree': tree, 'l10nbuilds': self.inipath, }, "Scheduler") bs = buildset.BuildSet([self.treebuilder], SourceStamp(), properties=props) self.submitBuildSet(bs) _ds.append(bs.waitUntilFinished()) d = defer.DeferredList(_ds) d.addCallback(self.onTreesBuilt) self.waitOnTree = d
def perspective_try(self, branch, revision, patch, builderNames, properties={}): log.msg("user %s requesting build on builders %s" % (self.username, builderNames)) for b in builderNames: if not b in self.parent.builderNames: log.msg("%s got job with builder %s" % (self, b)) log.msg(" but that wasn't in our list: %s" % (self.parent.builderNames,)) return ss = SourceStamp(branch, revision, patch) reason = "'try' job from user %s" % self.username # roll the specified props in with our inherited props combined_props = Properties() combined_props.updateFromProperties(self.parent.properties) combined_props.update(properties, "try build") bs = buildset.BuildSet(builderNames, ss, reason=reason, properties=combined_props) self.parent.submitBuildSet(bs) # return a remotely-usable BuildSetStatus object from buildbot.status.client import makeRemote return makeRemote(bs.status)
def trigger(self, ss): """Trigger this scheduler. Returns a deferred that will fire when the buildset is finished. """ bs = buildset.BuildSet(self.builderNames, ss) d = bs.waitUntilFinished() self.submit(bs) return d
def doPeriodicBuild(self): # Schedule the next run self.setTimer() # And trigger a build bs = buildset.BuildSet(self.builderNames, SourceStamp(branch=self.branch), self.reason) self.submit(bs)
def fireTimer(self): # clear out our state self.timer = None self.nextBuildTime = None changes = self.importantChanges + self.unimportantChanges self.importantChanges = [] self.unimportantChanges = [] # create a BuildSet, submit it to the BuildMaster bs = buildset.BuildSet(self.builderNames, SourceStamp(changes=changes)) self.submit(bs)
def fireTimer(self): self.timer = None self.nextBuildTime = None changes = self.importantChanges + self.unimportantChanges self.importantChanges = [] self.unimportantChanges = [] # submit ss = noMergeSourceStamp(changes=changes) bs = buildset.BuildSet(self.builderNames, ss) self.submit(bs)
def testGetBuildSets(self): # validate IStatus.getBuildSets s = status.builder.Status(None, ".") bs1 = buildset.BuildSet(["a", "b"], sourcestamp.SourceStamp(), reason="one", bsid="1") s.buildsetSubmitted(bs1.status) self.failUnlessEqual(s.getBuildSets(), [bs1.status]) bs1.status.notifyFinishedWatchers() self.failUnlessEqual(s.getBuildSets(), [])
def fireTimer(self): self.timer = None self.nextBuildTime = None changes = self.importantChanges + self.unimportantChanges self.importantChanges = [] self.unimportantChanges = [] # submit for i in range(0, self.numberOfBuildsToTrigger): bs = buildset.BuildSet(self.builderNames, SourceStamp(changes=changes)) self.submit(bs)
def trigger(self, ss, set_props=None): """Trigger this scheduler. Returns a deferred that will fire when the buildset is finished. """ # properties for this buildset are composed of our own properties, # potentially overridden by anything from the triggering build props = Properties() props.updateFromProperties(self.properties) if set_props: props.updateFromProperties(set_props) bs = buildset.BuildSet(self.builderNames, ss, properties=props) d = bs.waitUntilFinished() self.submitBuildSet(bs) return d
def perspective_try(self, branch, revision, patch, builderNames): log.msg("user %s requesting build on builders %s" % (self.username, builderNames)) for b in builderNames: if not b in self.parent.builderNames: log.msg("%s got job with builder %s" % (self, b)) log.msg(" but that wasn't in our list: %s" % (self.parent.builderNames, )) return ss = SourceStamp(branch, revision, patch) reason = "'try' job from user %s" % self.username bs = buildset.BuildSet(builderNames, ss, reason=reason) self.parent.submitBuildSet(bs) # return a remotely-usable BuildSetStatus object from buildbot.status.client import makeRemote return makeRemote(bs.status)
def fireTimer(self): # clear out our state self.timer = None self.nextBuildTime = None changes = self.unimportantChanges + self.importantChanges self.importantChanges = [] self.unimportantChanges = [] buildset_builderNames = [] # There may be multiple build requests, only process the very last one (latest) change = changes[-1] # Only add builders that are: # 1) dependent builder is listed in the change request, this means that the upstream builder was online # 2) builder is online # determine the priority based on branch # note that this is a list comprehension, so we need to "unpack" with # [0] at end priority = [p[1] for p in self.priorities if p[0] == change.branch][0] f = open("%s/change-%s.%s" % (self.changeDir, change.revision, priority)) phase1Builders = [] for line in f.readlines(): if line.startswith("builders:"): phase1Builders = line[line.find(":")+1:].strip().split() break # builders is a list of builders that built the request in Phase1 for p1Builder in phase1Builders: # Get the depenent builder for dependent in self.builderDependencies: if p1Builder == dependent[1]: p2Builder = dependent[0] # See if the builder is online # Get a builder from the BotMaster: builder = self.parent.botmaster.builders.get(p2Builder) # Add the builder to the set if it is idle (not building and not offline) if builder.builder_status.getState()[0] == 'idle': buildset_builderNames.append(p2Builder) # create a BuildSet, submit it to the BuildMaster bs = buildset.BuildSet(buildset_builderNames, SourceStamp(changes=changes), properties=self.properties) self.submitBuildSet(bs)
def submitBuildsets(self): for tpl, changes in self.pendings.iteritems(): tree, locale = tpl _t = self.trees[tree] props = properties.Properties() # figure out the latest change try: when = timeHelper(max(filter(None, (c.when for c in changes)))) from life.models import Push except (ValueError, ImportError): when = None for k, v in _t.branches.iteritems(): props.setProperty(k+"_branch", v, "Scheduler") _r = "default" if when is not None: if k == 'l10n': repo = '%s/%s' % (v, locale) else: repo = v q = Push.objects.filter(repository__name=repo, push_date__lte=when, changesets__branch__name='default') try: # get the latest changeset on the 'default' branch # not strictly .tip, for pushes with heads on # multiple branches (bug 602182) _c = q.order_by('-pk')[0].changesets.order_by('-pk') _r = str(_c.filter(branch__name='default')[0].shortrev) except IndexError: # no pushes, update to empty repo 000000000000 _r = "default" props.setProperty(k+"_revision", _r, "Scheduler") props.update({"tree": tree, "locale": locale, "l10n.ini": _t.l10ninis[_t.branches['en']][0], "revisions": sorted(_t.branches.keys()), }, "Scheduler") bs = buildset.BuildSet(self.builderNames, SourceStamp(changes=changes), properties=props) self.submitBuildSet(bs) self.dSubmitBuildsets = None self.pendings.clear()
def queueBuild(self, locale, change): props = properties.Properties() props.update({'locale': locale, 'tree': self.tree, 'branch': self.branch, 'repourl': self.repourl, 'refpath': self.enBranch + '/en-US', 'en_revision': 'default', 'en_branch': self.enBranch + '/en-US', 'l10npath': self.branch + '/' + locale, 'l10n_revision': 'default', 'l10n_branch': self.branch, }, 'Scheduler') ss = SourceStamp(changes=[change]) bs = buildset.BuildSet(self.builderNames, ss, reason = "%s %s" % (self.tree, locale), properties = props) self.submitBuildSet(bs)
def fireTimer(self): # clear out our state self.timer = None self.nextBuildTime = None changes = self.unimportantChanges + self.importantChanges self.importantChanges = [] self.unimportantChanges = [] # Only add builders to the buildset that are currently online buildset_builderNames = [] for builder_name in self.builderNames: # Get a builder from the BotMaster: builder = self.parent.botmaster.builders.get(builder_name) if builder.builder_status.getState()[0] == 'idle': buildset_builderNames.append(builder_name) # create a BuildSet, submit it to the BuildMaster bs = buildset.BuildSet(buildset_builderNames, SourceStamp(changes=changes), properties=self.properties) self.submitBuildSet(bs)
def messageReceived(self, filename): md = os.path.join(self.parent.basedir, self.jobdir) if runtime.platformType == "posix": # open the file before moving it, because I'm afraid that once # it's in cur/, someone might delete it at any moment path = os.path.join(md, "new", filename) f = open(path, "r") os.rename(os.path.join(md, "new", filename), os.path.join(md, "cur", filename)) else: # do this backwards under windows, because you can't move a file # that somebody is holding open. This was causing a Permission # Denied error on bear's win32-twisted1.3 buildslave. os.rename(os.path.join(md, "new", filename), os.path.join(md, "cur", filename)) path = os.path.join(md, "cur", filename) f = open(path, "r") try: builderNames, ss, bsid = self.parseJob(f) except BadJobfile: log.msg("%s reports a bad jobfile in %s" % (self, filename)) log.err() return # compare builderNames against self.builderNames # TODO: think about this some more.. why bother restricting it? # perhaps self.builderNames should be used as the default list # instead of being used as a restriction? for b in builderNames: if not b in self.builderNames: log.msg("%s got jobfile %s with builder %s" % (self, filename, b)) log.msg(" but that wasn't in our list: %s" % (self.builderNames,)) return reason = "'try' job" bs = buildset.BuildSet(builderNames, ss, reason=reason, bsid=bsid, properties=self.properties) self.submitBuildSet(bs)
def submitBuildsets(self): connection.close_if_unusable_or_obsolete() log.msg('submitting %d pending buildsets' % len(self.pendings)) for tpl, changes in self.pendings.iteritems(): tree, locale = tpl _t = self.trees[tree] props = properties.Properties() # figure out the latest change try: when = timeHelper(max(filter(None, (c.when for c in changes)))) except (ValueError, ImportError): when = None revisions = sorted(_t.branches.keys()) for k, v in _t.branches.iteritems(): _r = "000000000000" if k == 'l10n': repo = '%s/%s' % (v, locale) else: repo = v try: repo = Repository.objects.get(name=repo) except Repository.DoesNotExist: log.msg('Repository %s does not exist, skipping' % repo) revisions.remove(k) continue q = Push.objects.filter(repository=repo, changesets__branch__name='default') if when is not None: q = q.filter(push_date__lte=when) try: # get the latest changeset on the 'default' branch # not strictly .tip, for pushes with heads on # multiple branches (bug 602182) _p = q.order_by('-pk')[0] if _p.push_date: if not when: when = _p.push_date else: when = max(when, _p.push_date) _c = _p.changesets.order_by('-pk') _r = str(_c.filter(branch__name='default')[0].revision) except IndexError: # no pushes, try to get a good Changeset. # this is guaranteed to at least return the null changeset _r = str( repo.changesets .filter(branch__name='default') .order_by('-pk') .values_list('revision', flat=True)[0]) relpath = repo.relative_path() props.setProperty(k+"_branch", relpath, "Scheduler") if relpath != repo.name: props.setProperty("local_" + repo.name, relpath, "Scheduler") props.setProperty(k+"_revision", _r, "Scheduler") _f = Forest.objects.get(name=_t.branches['l10n']) # use the relative path of the en repo we got above inipath = '{}/{}'.format( props['en_branch'], _t.l10ninis[_t.branches['en']][0]) props.update({"tree": tree, "l10nbase": _f.relative_path(), "locale": locale, "inipath": inipath, "srctime": when, "revisions": revisions, }, "Scheduler") bs = buildset.BuildSet(self.builderNames, SourceStamp(changes=changes), properties=props) self.submitBuildSet(bs) log.msg('one buildset successfully submitted') self.dSubmitBuildsets = None self.pendings.clear()
def addChange(self, change): '''Main entry point for the scheduler, this is called by the buildmaster. ''' log.msg("addChange appscheduler, %s" % str(self.waitOnTree)) if self.waitOnTree is not None: # a tree build is currently running, wait with this # until we're done with it self.pendingChanges.append(change) return # fixup change.locale if property is given if not hasattr(change, 'locale') or not change.locale: if 'locale' in change.properties: change.locale = change.properties['locale'] log.msg("locale: %s" % getattr(change, 'locale', 'none')) if not hasattr(change, 'locale') or not change.locale: # check branch, l10n.inis # if l10n.inis are found, callback to all-locales, locales/en-US # otherwise just check those straight away if change.branch not in self.branches: log.msg('not our branches') return tree_triggers = set() branchdata = self.branches[change.branch] for f in change.files: if f in branchdata.inis: tree_triggers.update(branchdata.inis[f]) if tree_triggers: # trigger tree builds, wait for them to finish # and check the change for en-US builds _ds = [] for _n in tree_triggers: props = properties.Properties() props.update({ 'tree': _n, 'l10nbuilds': self.inipath, }, "Scheduler") bs = buildset.BuildSet([self.treebuilder], SourceStamp(branch=change.branch, changes=[change]), properties=props) self.submitBuildSet(bs) _ds.append(bs.waitUntilFinished()) d = defer.DeferredList(_ds) d.addCallback(self.onTreesBuilt, branchdata=branchdata, change=change) self.waitOnTree = d return self.checkEnUS(None, branchdata, change) return # check l10n changesets log.msg('my branch: %s, in? %s' % (change.branch, ','.join(sorted(self.l10nbranches.keys())))) if change.branch not in self.l10nbranches: return l10ndirs = self.l10nbranches[change.branch] log.msg('yes, dirs: %s' % ','.join(sorted(l10ndirs))) trees = set() for f in change.files: for mod, _trees in l10ndirs.iteritems(): if f.startswith(mod): trees |= _trees for _n in trees: if change.locale in self.trees[_n].locales: self.compareBuild(_n, change.locale, [change]) else: log.msg('%s not in tree %s, needs %s' % ( change.locale, _n, ','.join(sorted(self.trees[_n].locales)))) return
def upstreamBuilt(self, ss): bs = buildset.BuildSet(self.builderNames, ss) self.submit(bs)
def upstreamBuilt(self, ss): bs = buildset.BuildSet(self.builderNames, SourceStamp(branch=self.branch)) self.submitBuildSet(bs)
def upstreamBuilt(self, ss): # Only add builders to the buildset that are currently online and if the dependent builder # also built this change. # If a fileIsImportant function is defined, check the files before determining buildsets if self.fileIsImportant: startThisBuildSet = False # SourceStamp provides us with a tuple of changes, loop through and check them all for change in ss.changes: if self.fileIsImportant(change): # If any file is important, we start the buildSet startThisBuildSet = True break if not startThisBuildSet: # no important files found, do not continue with this buildset return if self.source_stamp == None: # This is the first time that the ss has been seen, this is NOT a callback self.source_stamp = ss else: # This can either be a callback or a new ss passed in from upstream builder try: revision_prev = int(self.source_stamp.revision) revision_curr = int(ss.revision) except ValueError: errmsg = "BuilderDependent.upstreamBuilt has an unknown ss.revison: %s" % (ss.revision) assert False, errmsg # ss.revision is an OLDER REV than self.source_stamp.revision if revision_curr < revision_prev: # This means we are a callback, and during sleep period a new build request # has come in and is now in control, we can just stop running. return # ss.revision is a NEWER REV than self.source_stamp.revision elif revision_curr > revision_prev: # This would indicate a new build and we need to merge ss and self.source_stamp ss.mergeWith([self.source_stamp]) self.source_stamp = ss # Revisions are the same so this is the callback # else revision_curr == revision_prev buildset_builderNames = [] for builder_name in self.builderNames: # Get the depenent builder for dependent in self.builderDependencies: if builder_name == dependent[0]: dependent_builder = dependent[1] break # Get the dependent builder dep_builder = self.parent.getStatus().getBuilder(dependent_builder) # Determine the revision of the last build for the dependent builder # need to make sure that the builder has at least one previous build try: dep_ss = dep_builder.getLastFinishedBuild().getSourceStamp() except AttributeError: dep_ss = SourceStamp() # Only continue the check if the dependent builder built the sourceStamp, if ss.revision == dep_ss.revision: # Get a builder from the BotMaster: builder = self.parent.botmaster.builders.get(builder_name) if builder.builder_status.getState()[0] == 'building': # There is a builder in this scheduler that is active so we need # to NOT start the build yet, but instead callback in X seconds # to see if all of the builders are available. self.timer = reactor.callLater(self.callbackInterval, self.upstreamBuilt, ss) return # Add the builder to the set if it is idle (not building and not offline) if builder.builder_status.getState()[0] == 'idle': buildset_builderNames.append(builder_name) bs = buildset.BuildSet(buildset_builderNames, ss, properties=self.properties) self.submitBuildSet(bs) # Clear the tracked source stamp self.source_stamp = None
def doPeriodicBuild(self): bs = buildset.BuildSet(self.builderNames, SourceStamp(branch=self.branch), self.reason, properties=self.properties) self.submitBuildSet(bs)
def upstreamBuilt(self, ss): bs = buildset.BuildSet(self.builderNames, ss, properties=self.properties) self.submitBuildSet(bs)