def testNoRevision(self):
c = Change(who="catlee",
files=["foo"],
comments="",
branch="b1",
revision=None)
ss = SourceStamp(changes=[c])
self.assertEquals(ss.revision, None)
def done(self, res):
if not self.repository:
self.repository = self.treetop
# TODO: figure out the branch and project too
ss = SourceStamp(self.branch,
self.baserev,
self.patch,
repository=self.repository)
return ss
def _triggerUpstream(self, res):
log.msg("trigger upstream")
ss = SourceStamp()
upstream = [
s for s in self.master.allSchedulers() if s.name == 's_upstream'
][0]
d = upstream.trigger(ss)
d.addCallback(self._gotBuild)
return d
def parseJob(self, f):
# jobfiles are serialized build requests. Each is a list of
# serialized netstrings, in the following order:
# "1", the version number of this format
# buildsetID, arbitrary string, used to find the buildSet later
# branch name, "" for default-branch
# base revision, "" for HEAD
# patchlevel, usually "1"
# patch
# builderNames...
p = JobFileScanner()
p.dataReceived(f.read())
if p.error:
raise BadJobfile("unable to parse netstrings")
s = p.strings
ver = s.pop(0)
if ver == "1":
buildsetID, branch, baserev, patchlevel, diff = s[:5]
builderNames = s[5:]
if branch == "":
branch = None
if baserev == "":
baserev = None
patchlevel = int(patchlevel)
patch = (patchlevel, diff)
ss = SourceStamp("Old client", branch, baserev, patch)
elif ver == "2": # introduced the repository and project property
buildsetID, branch, baserev, patchlevel, diff, repository, project = s[:
7]
builderNames = s[7:]
if branch == "":
branch = None
if baserev == "":
baserev = None
patchlevel = int(patchlevel)
patch = (patchlevel, diff)
ss = SourceStamp(branch,
baserev,
patch,
repository=repository,
project=project)
else:
raise BadJobfile("unknown version '%s'" % ver)
return builderNames, ss, buildsetID
def testAsDictEmpty(self):
EXPECTED = {
'revision': None,
'branch': None,
'hasPatch': False,
'changes': [],
'project': '',
'repository': '',
}
self.assertEqual(EXPECTED, SourceStamp().asDict())
def requestBuild(self, builder, locale):
# returns a Deferred that fires with an IBuildStatus object when the
# build is finished
props = Properties()
props.setProperty('locale', locale, 'scheduler')
props.setProperty('tree', 'app', 'scheduler')
req = BuildRequest("forced build", SourceStamp(), builder,
properties=props)
self.control.getBuilder(builder).requestBuild(req)
return req.waitUntilFinished()
def doBuild(self, buildername, reason="forced"):
# we need to prevent these builds from being merged, so we create
# each of them with a different revision specifier. The revision is
# ignored because our build process does not have a source checkout
# step.
self.revision += 1
br = BuildRequest(reason, SourceStamp(revision=self.revision))
d = br.waitUntilFinished()
self.control.getBuilder(buildername).requestBuild(br)
return d
def doPeriodicBuild(self):
# Schedule the next run
self.setTimer()
# And trigger a build
bs = buildset.BuildSet(self.builderNames,
SourceStamp(branch=self.branch),
self.reason,
properties=self.properties)
self.submitBuildSet(bs)
def _add_build_and_remove_changes(self, t, important, unimportant):
all_changes = sorted(important + unimportant, key=lambda c : c.number)
db = self.parent.db
for i in range(self.numberOfBuildsToTrigger):
if self.treeStableTimer is None:
# each *important* Change gets a separate build. Unimportant
# builds get ignored.
for c in important:
ss = SourceStamp(changes=[c])
ssid = db.get_sourcestampid(ss, t)
self.create_buildset(ssid, "scheduler", t)
else:
ss = SourceStamp(changes=all_changes)
ssid = db.get_sourcestampid(ss, t)
self.create_buildset(ssid, "scheduler", t)
# and finally retire the changes from scheduler_changes
changeids = [c.number for c in all_changes]
db.scheduler_retire_changes(self.schedulerid, changeids, t)
def _testRequest_1(self, res):
c = interfaces.IControl(self.master)
req = base.BuildRequest("I was bored", SourceStamp(), 'test_builder')
builder_control = c.getBuilder("force")
d = defer.Deferred()
req.subscribe(d.callback)
builder_control.requestBuild(req)
d.addCallback(self._testRequest_2)
# we use the same check-the-results code as testForce
return d
def submitBuild(self):
ss = SourceStamp()
br = BuildRequest("forced build", ss, "dummy")
self.control.getBuilder("dummy").requestBuild(br)
d = defer.Deferred()
def _started(bc):
br.unsubscribe(_started)
d.callback(bc)
br.subscribe(_started)
return d
def makeBuildStatus(self, branches):
"""
Make an IBuildStatus that has source stamps for the given branches.
"""
sourceStamps = [SourceStamp(branch=branch) for branch in branches]
# This is the bare minimum we can specify to construct a
# `BuildStatus`.
build = BuildStatus(BuilderStatus(None, None, None, None), None, None)
build.setSourceStamps(sourceStamps)
return build
def testPriority(self):
self.rmtree("basedir")
os.mkdir("basedir")
self.master.loadConfig(config_priority)
self.master.readConfig = True
self.master.startService()
# Our fake source stamp
# we override canBeMergedWith so that our requests don't get merged together
ss = SourceStamp()
ss.canBeMergedWith = lambda x: False
# Send 10 requests to alternating builders
# We fudge the submittedAt field after submitting since they're all
# getting submitted so close together according to time.time()
# and all we care about is what order they're run in.
reqs = []
self.start_order = []
for i in range(10):
req = BuildRequest(str(i), ss, "test_builder")
j = i % 2 + 1
self.master.botmaster.builders['quick%i' %
j].submitBuildRequest(req)
req.submittedAt = i
# Keep track of what order the builds start in
def append(build):
self.start_order.append(int(build.reason))
req.subscribe(append)
reqs.append(req.waitUntilFinished())
dl = defer.DeferredList(reqs)
dl.addCallback(self._all_finished)
def _delay(res):
d1 = defer.Deferred()
reactor.callLater(0.5, d1.callback, None)
# this test depends upon this 0.5s delay landing us in the middle
# of one of the builds.
return d1
def _connect(res, i):
return self.connectSlave(slavename="bot%i" % i,
builders=["quick1", "quick2"])
# Now add the slaves
d = self.connectSlave(slavename="bot0", builders=["quick1", "quick2"])
for i in range(1, 5):
d.addCallback(_delay)
d.addCallback(_connect, i)
d.addCallback(lambda x: dl)
return d
def testAsDictBranch(self):
EXPECTED = {
'revision': 'Rev',
'branch': 'Br',
'hasPatch': False,
'changes': [],
'project': '',
'repository': '',
}
self.assertEqual(EXPECTED,
SourceStamp(branch='Br', revision='Rev').asDict())
def ssFunc(scheduler, t):
#### NOTE: called in a thread!
db = scheduler.parent.db
# Look back 24 hours for a good revision to build
start = time.time()
rev = lastGoodRev(db, t, branch, builderNames, start - (24 * 3600),
start)
end = time.time()
log.msg("lastGoodRev: took %.2f seconds to run; returned %s" %
(end - start, rev))
if rev is None:
# Check if there are any recent l10n changes
if l10nBranch:
lastL10nChange = lastChange(db, t, l10nBranch)
if lastL10nChange:
lastL10nChange = lastL10nChange.when
else:
lastL10nChange = 0
else:
lastL10nChange = 0
# If there are no recent l10n changes, and we don't want to trigger
# builds if nothing has changed in the past 24 hours, then return
# None, indicating that no build should be scheduled
if not triggerBuildIfNoChanges:
if l10nBranch:
if (start - lastL10nChange) > (24 * 3600):
return None
else:
return None
# Couldn't find a good revision. Fall back to using the latest
# revision on this branch
c = lastChange(db, t, branch)
if c:
rev = c.revision
log.msg("lastChange returned %s" % (rev))
# Find the last revisions our scheduler's builders have built. This can
# include forced builds.
last_built_revs = getLastBuiltRevisions(db, t, branch,
scheduler.builderNames)
log.msg("lastNightlyRevisions: %s" % last_built_revs)
if last_built_revs:
# Make sure that rev is newer than the last revision we built.
later_rev = getLatestRev(db, t, branch, [rev] + last_built_revs)
if later_rev != rev:
log.msg("lastGoodRev: Building %s since it's newer than %s" %
(later_rev, rev))
rev = later_rev
return SourceStamp(branch=scheduler.branch, revision=rev)
def fireTimer(self):
# clear out our state
self.timer = None
self.nextBuildTime = None
changes = self.importantChanges + self.unimportantChanges
self.importantChanges = []
self.unimportantChanges = []
# create a BuildSet, submit it to the BuildMaster
bs = buildset.BuildSet(self.builderNames, SourceStamp(changes=changes))
self.submit(bs)
def _send(res):
# send some build requests
reqs = []
ss = SourceStamp()
for i in range(5):
req = BuildRequest(str(i), ss, "dummy")
self.master.botmaster.builders['dummy'].submitBuildRequest(req)
reqs.append(req.waitUntilFinished())
dl = defer.DeferredList(reqs)
dl.addCallback(check)
return dl
def perspective_requestBuild(self,
buildername,
reason,
branch,
revision,
properties={}):
c = interfaces.IControl(self.master)
bc = c.getBuilder(buildername)
ss = SourceStamp(branch, revision)
bpr = Properties()
bpr.update(properties, "remote requestBuild")
return bc.submitBuildRequest(ss, reason, bpr)
def force(self, req, auth_ok=False):
name = req.args.get("username", ["<unknown>"])[0]
reason = req.args.get("comments", ["<no reason specified>"])[0]
branch = req.args.get("branch", [""])[0]
revision = req.args.get("revision", [""])[0]
r = "The web-page 'force build' button was pressed by '%s': %s\n" \
% (html.escape(name), html.escape(reason))
log.msg("web forcebuild of builder '%s', branch='%s', revision='%s'"
" by user '%s'" %
(self.builder_status.getName(), branch, revision, name))
# check if this is allowed
if not auth_ok:
if not self.getAuthz(req).actionAllowed('forceBuild', req,
self.builder_status):
log.msg("..but not authorized")
return Redirect(path_to_authfail(req))
# keep weird stuff out of the branch revision, and property strings.
# TODO: centralize this somewhere.
if not re.match(r'^[\w\.\-\/]*$', branch):
log.msg("bad branch '%s'" % branch)
return Redirect(path_to_builder(req, self.builder_status))
if not re.match(r'^[\w\.\-\/]*$', revision):
log.msg("bad revision '%s'" % revision)
return Redirect(path_to_builder(req, self.builder_status))
properties = getAndCheckProperties(req)
if properties is None:
return Redirect(path_to_builder(req, self.builder_status))
if not branch:
branch = None
if not revision:
revision = None
# TODO: if we can authenticate that a particular User pushed the
# button, use their name instead of None, so they'll be informed of
# the results.
# TODO2: we can authenticate that a particular User pushed the button
# now, so someone can write this support. but it requires a
# buildbot.changes.changes.Change instance which is tedious at this
# stage to compute
s = SourceStamp(branch=branch, revision=revision)
try:
c = interfaces.IControl(self.getBuildmaster(req))
bc = c.getBuilder(self.builder_status.getName())
bc.submitBuildRequest(s, r, properties, now=True)
except interfaces.NoSlaveError:
# TODO: tell the web user that their request could not be
# honored
pass
# send the user back to the builder page
return Redirect(path_to_builder(req, self.builder_status))
def addRebuild(t):
c3 = Change(who='me!',
branch='b1',
revision='r1',
files=['http://path/to/build'],
comments='really important',
properties={'buildid': '20110214000001'},
when=6)
self.dbc.addChangeToDatabase(c3)
ss = SourceStamp(branch='b1', changes=[c3], revision='r1')
ssid = self.dbc.get_sourcestampid(ss, t)
self.dbc.create_buildset(ssid, "rebuild", Properties(), ["b1"], t)
def fireTimer(self):
self.timer = None
self.nextBuildTime = None
changes = self.importantChanges + self.unimportantChanges
self.importantChanges = []
self.unimportantChanges = []
# submit
for i in range(0, self.numberOfBuildsToTrigger):
bs = buildset.BuildSet(self.builderNames,
SourceStamp(changes=changes))
self.submit(bs)
def testAsDictChanges(self):
changes = [
Change('nobody', [], 'Comment', branch='br2', revision='rev2'),
Change('nob', ['file2', 'file3'],
'Com',
branch='br3',
revision='rev3'),
]
s = SourceStamp(branch='Br',
revision='Rev',
patch=(1, 'Pat'),
changes=changes)
r = s.asDict()
del r['changes'][0]['when']
del r['changes'][1]['when']
EXPECTED = {
'revision':
'rev3',
'branch':
'br3',
'hasPatch':
True,
'project':
'',
'repository':
'',
'changes': [{
'branch': 'br2',
'category': None,
'comments': 'Comment',
'files': [],
'number': None,
'properties': [],
'revision': 'rev2',
'revlink': '',
'project': '',
'repository': '',
'who': 'nobody'
}, {
'branch': 'br3',
'category': None,
'comments': 'Com',
'files': ['file2', 'file3'],
'number': None,
'properties': [],
'revision': 'rev3',
'revlink': '',
'who': 'nob',
'project': '',
'repository': '',
}],
}
self.assertEqual(EXPECTED, r)
def perspective_forcewait(self,
builder='build',
reason='',
branch='',
revision='',
pdict={}):
log.msg('forcewait called')
branch_validate = self.master.config.validation['branch']
revision_validate = self.master.config.validation['revision']
pname_validate = self.master.config.validation['property_name']
pval_validate = self.master.config.validation['property_value']
if not branch_validate.match(branch):
log.msg("bad branch '%s'" % branch)
return
if not revision_validate.match(revision):
log.msg("bad revision '%s'" % revision)
return
properties = Properties()
if pdict:
for prop in pdict:
pname = prop
pvalue = pdict[prop]
if not pname_validate.match(pname) or \
not pval_validate.match(pvalue):
log.msg("bad property name='%s', value='%s'" %
(pname, pvalue))
return
log.msg('set property %s %s' % (pname, pvalue))
properties.setProperty(pname, pvalue, "Force Build PB")
c = interfaces.IControl(self.master)
b = c.getBuilder(builder)
ss = SourceStamp(branch=branch, revision=revision)
dr = defer.Deferred()
def started(s):
log.msg('force started')
dr.callback(s.getNumber())
def requested(breq):
log.msg('force requested')
breq.subscribe(started)
d2 = b.submitBuildRequest(ss, reason, props=properties.asDict())
d2.addCallback(requested)
d2.addErrback(log.err, "while forcing a build")
return dr
def _testDontClaimPingingSlave_0(self, ign):
self.master.botmaster.builders["b1"].CHOOSE_SLAVES_RANDOMLY = False
timers = []
self.slaves['bot1'].debugOpts["stallPings"] = (10, timers)
bss = self.control.submitBuildSet(["b1"], SourceStamp(), "forced")
s1 = bss.getBuildRequests()[0]
d1 = bss.waitUntilFinished()
# give it a chance to start pinging
d2 = defer.Deferred()
d2.addCallback(self._testDontClaimPingingSlave_1, d1, s1, timers)
reactor.callLater(1, d2.callback, None)
return d2
def testReconfig(self):
# reconfiguring a Builder should not interrupt any running Builds. No
# queued BuildRequests should be lost. The next Build started should
# use the new process.
slave1 = self.slaves['bot1']
bot1 = slave1.getServiceNamed('bot')
sb1 = bot1.builders['b1']
self.failUnless(isinstance(sb1, bot.SlaveBuilder))
self.failUnless(sb1.running)
b1 = self.master.botmaster.builders['b1']
self.orig_b1 = b1
self.d1 = d1 = defer.Deferred()
self.d2 = defer.Deferred()
self.d3, self.d4 = defer.Deferred(), defer.Deferred()
self.d5, self.d6 = defer.Deferred(), defer.Deferred()
self.build1_started = False
self.build2_started = False
self.build3_started = False
waitCommandRegistry[("one", "build1")] = self._one_started
waitCommandRegistry[("two", "build2")] = self._two_started
waitCommandRegistry[("three", "build3")] = self._three_started
# use different branches to make sure these cannot be merged
bss1 = self.control.submitBuildSet(["b1"], SourceStamp(branch="1"),
"build1")
bss2 = self.control.submitBuildSet(["b1"], SourceStamp(branch="2"),
"build2")
bss3 = self.control.submitBuildSet(["b1"], SourceStamp(branch="3"),
"build3")
self.bsses = (bss1, bss2, bss3)
self.requests = (bss1.getBuildRequests()[0],
bss2.getBuildRequests()[0],
bss3.getBuildRequests()[0])
# all three are now in the queue
# wait until the first one has started
d1.addCallback(self._testReconfig_2)
return d1
def testMultipleStepInstances(self):
steps = [
(source.CVS, {'cvsroot': "root", 'cvsmodule': "module"}),
(shell.Configure, {'command': "./configure"}),
(shell.Compile, {'command': "make"}),
(shell.Compile, {'command': "make more"}),
(shell.Compile, {'command': "make evenmore"}),
(shell.Test, {'command': "make test"}),
(shell.Test, {'command': "make testharder"}),
]
f = factory.ConfigurableBuildFactory(steps)
req = base.BuildRequest("reason", SourceStamp())
b = f.newBuild([req])
def getBuildRequestStatus(id):
brstatus = BuildRequestStatus(builder.builder_status.name, id, self.master_status)
brstatus._buildrequest = mock.Mock()
brstatus.getSubmitTime = lambda: 1418823086
brstatus.getResults = lambda : -1
brstatus.getReason = lambda: 'because'
brstatus.getPriority = lambda : 50
ss = SourceStamp(branch='b', sourcestampsetid=1, repository='z')
brstatus.getSourceStamps = lambda: {}
brstatus.getSourceStamp = lambda: ss
brstatus.getBuildProperties = lambda: Properties()
return brstatus
def send(res):
# send some build requests
ss = SourceStamp()
for i in range(5):
req = BuildRequest(str(i), ss, "dummy")
self.master.botmaster.builders['dummy'].submitBuildRequest(req)
req.submittedAt = i
def append(build):
start_order.append(int(build.reason))
req.subscribe(append)
reqs.append(req.waitUntilFinished())
def force(self, req):
"""
Custom properties can be passed from the web form. To do
this, subclass this class, overriding the force() method. You
can then determine the properties (usually from form values,
by inspecting req.args), then pass them to this superclass
force method.
"""
name = req.args.get("username", ["<unknown>"])[0]
reason = req.args.get("comments", ["<no reason specified>"])[0]
branch = req.args.get("branch", [""])[0]
revision = req.args.get("revision", [""])[0]
r = "The web-page 'force build' button was pressed by '%s': %s\n" \
% (name, reason)
log.msg("web forcebuild of builder '%s', branch='%s', revision='%s'" %
(self.builder_status.getName(), branch, revision))
if not self.builder_control:
# TODO: tell the web user that their request was denied
log.msg("but builder control is disabled")
return Redirect("..")
# keep weird stuff out of the branch and revision strings. TODO:
# centralize this somewhere.
if not re.match(r'^[\w\.\-\/]*$', branch):
log.msg("bad branch '%s'" % branch)
return Redirect("..")
if not re.match(r'^[\w\.\-\/]*$', revision):
log.msg("bad revision '%s'" % revision)
return Redirect("..")
if not branch:
branch = None
if not revision:
revision = None
# TODO: if we can authenticate that a particular User pushed the
# button, use their name instead of None, so they'll be informed of
# the results.
s = SourceStamp(branch=branch, revision=revision)
req = BuildRequest(r, s, builderName=self.builder_status.getName())
try:
self.builder_control.requestBuildSoon(req)
except interfaces.NoSlaveError:
# TODO: tell the web user that their request could not be
# honored
pass
# send the user back to the builder page
return Redirect(".")
def setUp(self):
self.builder = FakeBuilder()
self.builder_status = builder.BuilderStatus("fakebuilder")
self.builder_status.basedir = "test_properties"
self.builder_status.nextBuildNumber = 5
rmdirRecursive(self.builder_status.basedir)
os.mkdir(self.builder_status.basedir)
self.build_status = self.builder_status.newBuild()
req = base.BuildRequest("reason",
SourceStamp(branch="branch2", revision=1234))
self.build = base.Build([req])
self.build.setBuilder(self.builder)
self.build.setupStatus(self.build_status)
self.build.setupSlaveBuilder(FakeSlaveBuilder())
