def test_reclaimAllBuilds(self):
wfd = defer.waitForDeferred(
self.makeBuilder())
yield wfd
wfd.getResult()
claims = []
def fakeClaimBRs(*args):
claims.append(args)
return defer.succeed(None)
self.bldr.master.db.buildrequests.claimBuildRequests = fakeClaimBRs
self.bldr.master.db.buildrequests.reclaimBuildRequests = fakeClaimBRs
def mkbld(brids):
bld = mock.Mock(name='Build')
bld.requests = []
for brid in brids:
br = mock.Mock(name='BuildRequest %d' % brid)
br.id = brid
bld.requests.append(br)
return bld
old = mkbld([15]) # keep a reference to the "old" build
self.bldr.old_building[old] = None
self.bldr.building.append(mkbld([10,11,12]))
wfd = defer.waitForDeferred(
self.bldr.reclaimAllBuilds())
yield wfd
wfd.getResult()
self.assertEqual(claims, [ (set([10,11,12,15]),) ])
def http_PROPFIND(self, request):
"""
Respond to a PROPFIND request. (RFC 2518, section 8.1)
"""
if not self.exists():
log.err("File not found: %s" % (self,))
raise HTTPError(responsecode.NOT_FOUND)
#
# Check authentication and access controls
#
x = waitForDeferred(self.authorize(request, (davxml.Read(),)))
yield x
x.getResult()
#
# Read request body
#
try:
doc = waitForDeferred(davXMLFromStream(request.stream))
yield doc
doc = doc.getResult()
except ValueError, e:
log.err("Error while handling PROPFIND body: %s" % (e,))
raise HTTPError(StatusResponse(responsecode.BAD_REQUEST, str(e)))
def produceAllMetrics(host, port, protocol, timeout=5.0, delay=0.0):
"""Produce ALL metrics of ALL metricID's to the endpoint with some
delay in between.
@param host (string)
@param port (int)
@param protocol (class) - twisted.internet.protocol.Protocol
@param timeout (int|float) - timeout parameter for the protocol
@param delay (int|float) - delay between sends
@callback (int) - how many metrics were sent
@errback N/A
@return defer.Deferred()
"""
result = 0
for obj in TimeSeriesData.objects:
while obj.pending:
try:
d = obj.produce_all(host, port, protocol, timeout=timeout)
wfd = defer.waitForDeferred(d)
yield wfd
result += wfd.getResult()
if delay:
d = defer.Deferred()
config.reactor.callLater(delay, d.callback, None)
wfd = defer.waitForDeferred(d)
yield wfd
wfd.getResult()
except: pass #swallow errors
yield result
def test_stopService_flushes(self):
wfd = defer.waitForDeferred(
self.makeBuilder())
yield wfd
wfd.getResult()
# just check that stopService calls this and waits
# for the deferred to fire
events = []
long_d = defer.Deferred()
long_d.addCallback(lambda _ : events.append('long_d'))
self.bldr.maybeStartBuild = lambda : long_d
stop_d = self.bldr.stopService()
stop_d.addCallback(lambda _ : events.append('stop_d'))
# nothing should have happened yet
self.assertEqual(events, [])
# finish the maybeStartBuild invocation..
long_d.callback(None)
wfd = defer.waitForDeferred(stop_d)
yield wfd
wfd.getResult()
# and then check that things happened in the right order
self.assertEqual(events, [ 'long_d', 'stop_d' ])
def produce(self):
"""Produce entries."""
while not self._paused:
wfd = defer.waitForDeferred(self._write_pending())
yield wfd
wfd.getResult()
if self.finished:
self.finishProducing()
return
first = self._last_fetching
while (self._currently_fetching <= self._max_currently_fetching and
self._last_fetching <= self._end and
self._currently_stored <= FLAGS.entries_buffer):
last = min(self._last_fetching + self._batch_size - 1, self._end,
self._last_fetching + self._max_currently_fetching
- self._currently_fetching + 1)
self._batches.put(self._create_fetch_deferred(first, last))
self._currently_fetching += last - first + 1
first = last + 1
self._last_fetching = first
wfd = defer.waitForDeferred(self._batches.get())
# Pause here until the body of the response is available.
yield wfd
# The producer may have been paused while waiting for the response,
# or errored out upon receiving it: do not write the entries out
# until after the next self._paused check.
self._pending = wfd.getResult()
def scanExistingClassifiedChanges(self):
# call gotChange for each classified change. This is called at startup
# and is intended to re-start the treeStableTimer for any changes that
# had not yet been built when the scheduler was stopped.
# NOTE: this may double-call gotChange for changes that arrive just as
# the scheduler starts up. In practice, this doesn't hurt anything.
wfd = defer.waitForDeferred(
self.master.db.schedulers.getChangeClassifications(
self.schedulerid))
yield wfd
classifications = wfd.getResult()
# call gotChange for each change, after first fetching it from the db
for changeid, important in classifications.iteritems():
wfd = defer.waitForDeferred(
self.master.db.changes.getChange(changeid))
yield wfd
chdict = wfd.getResult()
if not chdict:
continue
wfd = defer.waitForDeferred(
changes.Change.fromChdict(self.master, chdict))
yield wfd
change = wfd.getResult()
wfd = defer.waitForDeferred(
self.gotChange(change, important))
yield wfd
wfd.getResult()
def stableTimerFired(self, timer_name):
# if the service has already been stoppd then just bail out
if not self._stable_timers[timer_name]:
return
# delete this now-fired timer
del self._stable_timers[timer_name]
wfd = defer.waitForDeferred(
self.getChangeClassificationsForTimer(self.schedulerid,
timer_name))
yield wfd
classifications = wfd.getResult()
# just in case: databases do weird things sometimes!
if not classifications: # pragma: no cover
return
changeids = sorted(classifications.keys())
wfd = defer.waitForDeferred(
self.addBuildsetForChanges(reason='scheduler',
changeids=changeids))
yield wfd
wfd.getResult()
max_changeid = changeids[-1] # (changeids are sorted)
wfd = defer.waitForDeferred(
self.master.db.schedulers.flushChangeClassifications(
self.schedulerid, less_than=max_changeid+1))
yield wfd
wfd.getResult()
def http_MKCOL(self, request):
"""
Respond to a MKCOL request. (RFC 2518, section 8.3)
"""
parent = waitForDeferred(request.locateResource(parentForURL(request.uri)))
yield parent
parent = parent.getResult()
x = waitForDeferred(parent.authorize(request, (davxml.Bind(),)))
yield x
x.getResult()
if self.exists():
log.err("Attempt to create collection where file exists: %s"
% (self,))
raise HTTPError(responsecode.NOT_ALLOWED)
if not parent.isCollection():
log.err("Attempt to create collection with non-collection parent: %s"
% (self,))
raise HTTPError(StatusResponse(
responsecode.CONFLICT,
"Parent resource is not a collection."
))
#
# Read request body
#
x = waitForDeferred(noDataFromStream(request.stream))
yield x
try:
x.getResult()
except ValueError, e:
log.err("Error while handling MKCOL body: %s" % (e,))
raise HTTPError(responsecode.UNSUPPORTED_MEDIA_TYPE)
def deleteResource(request, resource, resource_uri, depth="0"):
"""
Handle a resource delete with proper quota etc updates
"""
if not resource.exists():
log.error("File not found: %s" % (resource,))
raise HTTPError(responsecode.NOT_FOUND)
# Do quota checks before we start deleting things
myquota = waitForDeferred(resource.quota(request))
yield myquota
myquota = myquota.getResult()
if myquota is not None:
old_size = waitForDeferred(resource.quotaSize(request))
yield old_size
old_size = old_size.getResult()
else:
old_size = 0
# Do delete
x = waitForDeferred(delete(resource_uri, resource.fp, depth))
yield x
result = x.getResult()
# Adjust quota
if myquota is not None:
d = waitForDeferred(resource.quotaSizeAdjust(request, -old_size))
yield d
d.getResult()
yield result
def test_reconfig_bad_config(self):
reactor = self.make_reactor()
self.master.reconfigService = mock.Mock(
side_effect=lambda n : defer.succeed(None))
wfd = defer.waitForDeferred(
self.master.startService(_reactor=reactor))
yield wfd
wfd.getResult()
# reset, since startService called reconfigService
self.master.reconfigService.reset_mock()
# reconfig, with a failure
self.patch_loadConfig_fail()
wfd = defer.waitForDeferred(
self.master.reconfig())
yield wfd
wfd.getResult()
self.master.stopService()
self.assertLogged("reconfig aborted without")
self.failIf(self.master.reconfigService.called)
def test_reconfigService_default_changed(self):
config = mock.Mock()
config.slavePortnum = '9876'
self.attachChangeSource(pb.PBChangeSource())
self.startChangeSource()
wfd = defer.waitForDeferred(
self.changesource.reconfigService(config))
yield wfd
wfd.getResult()
self.assertRegistered('9876', 'change', 'changepw')
config.slavePortnum = '1234'
wfd = defer.waitForDeferred(
self.changesource.reconfigService(config))
yield wfd
wfd.getResult()
self.assertUnregistered('9876', 'change', 'changepw')
self.assertRegistered('1234', 'change', 'changepw')
wfd = defer.waitForDeferred(
self.stopChangeSource())
yield wfd
wfd.getResult()
self.assertUnregistered('1234', 'change', 'changepw')
def test_brdictToBuildRequest(self):
self.makeBuilder()
# set up all of the data required for a BuildRequest object
wfd = defer.waitForDeferred(
self.db.insertTestData([
fakedb.SourceStamp(id=234),
fakedb.Buildset(id=30, sourcestampid=234, reason='foo',
submitted_at=1300305712, results=-1),
fakedb.BuildRequest(id=19, buildsetid=30, buildername='bldr',
priority=13, submitted_at=1300305712, results=-1),
]))
yield wfd
wfd.getResult()
wfd = defer.waitForDeferred(
self.db.buildrequests.getBuildRequest(19))
yield wfd
brdict = wfd.getResult()
wfd = defer.waitForDeferred(
self.bldr._brdictToBuildRequest(brdict))
yield wfd
br = wfd.getResult()
# just check that the BuildRequest looks reasonable -
# test_process_buildrequest checks the whole thing
self.assertEqual(br.reason, 'foo')
# and check that the cross-pointers are correct
self.assertIdentical(br.brdict, brdict)
self.assertIdentical(brdict['brobj'], br)
self.bldr._breakBrdictRefloops([brdict])
def full(self, _):
if self.method == 'clobber':
wfd = defer.waitForDeferred(self.clobber())
yield wfd
wfd.getResult()
return
elif self.method in ['copy', 'export']:
wfd = defer.waitForDeferred(self.copy())
yield wfd
wfd.getResult()
return
wfd = defer.waitForDeferred(self._sourcedirIsUpdatable())
yield wfd
updatable = wfd.getResult()
if not updatable:
d = self._dovccmd(['checkout', self.svnurl, '.'])
elif self.method == 'clean':
d = self.clean()
elif self.method == 'fresh':
d = self.fresh()
wfd = defer.waitForDeferred(d)
yield wfd
wfd.getResult()
def _mergeRequests(self, breq, unclaimed_requests, mergeRequests_fn):
"""Use C{mergeRequests_fn} to merge C{breq} against
C{unclaimed_requests}, where both are build request dictionaries"""
# short circuit if there is no merging to do
if not mergeRequests_fn or len(unclaimed_requests) == 1:
yield [ breq ]
return
# we'll need BuildRequest objects, so get those first
wfd = defer.waitForDeferred(
defer.gatherResults(
[ self._brdictToBuildRequest(brdict)
for brdict in unclaimed_requests ]))
yield wfd
unclaimed_request_objects = wfd.getResult()
breq_object = unclaimed_request_objects.pop(
unclaimed_requests.index(breq))
# gather the mergeable requests
merged_request_objects = [breq_object]
for other_breq_object in unclaimed_request_objects:
wfd = defer.waitForDeferred(
defer.maybeDeferred(lambda :
mergeRequests_fn(self, breq_object, other_breq_object)))
yield wfd
if wfd.getResult():
merged_request_objects.append(other_breq_object)
# convert them back to brdicts and return
merged_requests = [ br.brdict for br in merged_request_objects ]
yield merged_requests
def start(self):
args = self.args
# args['dir'] is relative to Builder directory, and is required.
assert args["dir"] is not None
dirnames = args["dir"]
self.timeout = args.get("timeout", 120)
self.maxTime = args.get("maxTime", None)
self.rc = 0
if type(dirnames) is list:
assert len(dirnames) != 0
for dirname in dirnames:
wfd = defer.waitForDeferred(self.removeSingleDir(dirname))
yield wfd
res = wfd.getResult()
# Even if single removal of single file/dir consider it as
# failure of whole command, but continue removing other files
# Send 'rc' to master to handle failure cases
if res != 0:
self.rc = res
else:
wfd = defer.waitForDeferred(self.removeSingleDir(dirnames))
yield wfd
self.rc = wfd.getResult()
self.sendStatus({"rc": self.rc})
def full(self):
if self.method == 'clobber':
wfd = defer.waitForDeferred(self.clobber())
yield wfd
wfd.getResult()
return
elif self.method == 'copy':
wfd = defer.waitForDeferred(self.copy())
yield wfd
wfd.getResult()
return
wfd = defer.waitForDeferred(self._sourcedirIsUpdatable())
yield wfd
updatable = wfd.getResult()
if not updatable:
log.msg("No git repo present, making full clone")
d = self._doFull()
elif self.method == 'clean':
d = self.clean()
elif self.method == 'fresh':
d = self.fresh()
else:
raise ValueError("Unknown method, check your configuration")
wfd = defer.waitForDeferred(d)
yield wfd
wfd.getResult()
def asDict_async(self):
result = {}
wfd = defer.waitForDeferred(
self.getSourceStamp())
yield wfd
ss = wfd.getResult()
result['source'] = ss.asDict()
result['builderName'] = self.getBuilderName()
wfd = defer.waitForDeferred(
self.getSubmitTime())
yield wfd
submittedAt = wfd.getResult()
result['submittedAt'] = submittedAt
wfd = defer.waitForDeferred(
self.getBuilds())
yield wfd
builds = wfd.getResult()
result['builds'] = [ build.asDict() for build in builds ]
yield result
def process_all(self):
"""
Process data when no virtual server and no real server are provided.
"""
# Retrieve all data
for oid in self.oids:
w = defer.waitForDeferred(self.proxy.walk(self.oids[oid]))
yield w
w.getResult()
# For each virtual server, build it
for ov in self.cache('ltmVirtualServAddrType'):
# Grab HTTP class
try:
classes = self.cache(('ltmVsHttpClassProfileName', ".".join([str(x) for x in ov]))).values()
except KeyError:
classes = []
for httpclass in classes + [None]:
v = oid2str(ov)
vs = defer.waitForDeferred(self.process_vs(v, httpclass))
yield vs
vs = vs.getResult()
if vs is not None:
if httpclass is not None:
self.lb.virtualservers["%s;%s" % (v, httpclass)] = vs
else:
self.lb.virtualservers[v] = vs
yield self.lb
return
def execute(self, action, actionargs=None, vs=None, rs=None):
"""
Execute an action.
@param action: action to be executed
"""
v, r = self.parse(vs, rs)
if r is None:
yield None
return
if action == "disable" or action == "operdisable":
d = defer.waitForDeferred(
self.proxy.set((self.oids['realServerWeight'], v, r), 0))
yield d
d.getResult()
yield True
return
if action == "enable" or action == "operenable":
try:
weight = int(actionargs[0])
except ValueError, KeyError:
weight = 1
d = defer.waitForDeferred(
self.proxy.set((self.oids['realServerWeight'], v, r), weight))
yield d
d.getResult()
yield True
return
def full(self, _):
if self.method == "clobber":
wfd = defer.waitForDeferred(self.clobber())
yield wfd
wfd.getResult()
return
elif self.method in ["copy", "export"]:
wfd = defer.waitForDeferred(self.copy())
yield wfd
wfd.getResult()
return
wfd = defer.waitForDeferred(self._sourcedirIsUpdatable())
yield wfd
updatable = wfd.getResult()
if not updatable:
d = self._dovccmd(["checkout", self.svnurl, "."])
elif self.method == "clean":
d = self.clean()
elif self.method == "fresh":
d = self.fresh()
wfd = defer.waitForDeferred(d)
yield wfd
wfd.getResult()
def cancelBuildRequest(self):
# first, try to claim the request; if this fails, then it's too late to
# cancel the build anyway
try:
wfd = defer.waitForDeferred(
self.master.db.buildrequests.claimBuildRequests([self.id]))
yield wfd
wfd.getResult()
except buildrequests.AlreadyClaimedError:
log.msg("build request already claimed; cannot cancel")
return
# then complete it with 'FAILURE'; this is the closest we can get to
# cancelling a request without running into trouble with dangling
# references.
wfd = defer.waitForDeferred(
self.master.db.buildrequests.completeBuildRequests([self.id],
FAILURE))
yield wfd
wfd.getResult()
# and let the master know that the enclosing buildset may be complete
wfd = defer.waitForDeferred(
self.master.maybeBuildsetComplete(self.bsid))
yield wfd
wfd.getResult()
def _sourcedirIsUpdatable(self):
# first, perform a stat to ensure that this is really an svn directory
cmd = buildstep.RemoteCommand('stat', {'file': self.workdir + '/.svn',
'logEnviron': self.logEnviron,})
cmd.useLog(self.stdio_log, False)
wfd = defer.waitForDeferred(
self.runCommand(cmd))
yield wfd
wfd.getResult()
if cmd.rc != 0:
yield False
return
# then run 'svn info' to check that the URL matches our repourl
wfd = defer.waitForDeferred(
self._dovccmd(['info'], collectStdout=True))
yield wfd
stdout = wfd.getResult()
# extract the URL, handling whitespace carefully so that \r\n works
# is a line terminator
mo = re.search('^URL:\s*(.*?)\s*$', stdout, re.M)
yield mo and mo.group(1) == self.repourl
return
def test():
print "STARTING TEST"
foo = paisley.CouchDB('localhost')
print "\nCreate database 'testdb':"
d = foo.createDB('testdb1')
d.addCallback(lambda f: log.err(f))
wfd = defer.waitForDeferred(d)
yield wfd
try:
print wfd.getResult()
except Exception as e:
# FIXME: not sure why Error.status is a str compared to http constants
if hasattr(e, 'status') and e.status == str(http.UNAUTHORIZED):
print "\nError: not allowed to create databases"
reactor.stop()
return
else:
raise
print "\nList databases on server:"
d = foo.listDB()
wfd = defer.waitForDeferred(d)
yield wfd
print wfd.getResult()
reactor.stop()
def test_reconfigService_class_name_change(self):
sch1 = self.makeSched(self.ReconfigSched, 'sch1')
self.new_config.schedulers = dict(sch1=sch1)
wfd = defer.waitForDeferred(
self.sm.reconfigService(self.new_config))
yield wfd
wfd.getResult()
self.assertIdentical(sch1.parent, self.sm)
self.assertIdentical(sch1.master, self.master)
self.assertEqual(sch1.reconfig_count, 1)
sch1_new = self.makeSched(self.ReconfigSched2, 'sch1')
self.new_config.schedulers = dict(sch1=sch1_new)
wfd = defer.waitForDeferred(
self.sm.reconfigService(self.new_config))
yield wfd
wfd.getResult()
# sch1 had its class name change, so sch1_new is now the active
# instance
self.assertIdentical(sch1_new.parent, self.sm)
self.assertIdentical(sch1_new.master, self.master)
def test_reconfigService_add_and_change_and_remove_no_reconfig(self):
sch1 = self.makeSched(self.Sched, 'sch1', attr='alpha')
self.new_config.schedulers = dict(sch1=sch1)
wfd = defer.waitForDeferred(
self.sm.reconfigService(self.new_config))
yield wfd
wfd.getResult()
self.assertIdentical(sch1.parent, self.sm)
self.assertIdentical(sch1.master, self.master)
sch1_new = self.makeSched(self.Sched, 'sch1', attr='beta')
sch2 = self.makeSched(self.Sched, 'sch2', attr='alpha')
self.new_config.schedulers = dict(sch1=sch1_new, sch2=sch2)
wfd = defer.waitForDeferred(
self.sm.reconfigService(self.new_config))
yield wfd
wfd.getResult()
# sch1 is not longer active, and sch1_new is
self.assertIdentical(sch1.parent, None)
self.assertIdentical(sch1.master, None)
self.assertIdentical(sch1_new.parent, self.sm)
self.assertIdentical(sch1_new.master, self.master)
self.assertIdentical(sch2.parent, self.sm)
self.assertIdentical(sch2.master, self.master)
def _deferred(deferredResult, caller):
"""only callable by ``newfunc``"""
from twisted.internet import reactor
result = None
attempt = 0
while attempt <= maximum:
try:
wfd = defer.waitForDeferred(deferredResult)
yield wfd
result = wfd.getResult()
break # didn't throw exception then we have the result
except SystemExit:
result = Failure()
break
except KeyboardInterrupt:
result = Failure()
break
except:
attempt += 1
if attempt > maximum:
result = Failure()
break # failure captured
d = defer.Deferred()
reactor.callLater(delay, d.callback, None)
wfd = defer.waitForDeferred(d)
yield wfd
wfd.getResult()
# fortunatly the caller has been set up in advance
caller.write(">>> Retry attempt %d" % attempt)
deferredResult = caller(*caller.args, **caller.kwargs)
yield result
def test_queue_collapsing(self):
# just to check that we're practicing with the right queue size (so
# QUEUE_SIZE_FACTOR is 10)
self.assertEqual(self.lru.max_queue, 30)
for c in 'a' + 'x' * 27 + 'ab':
wfd = defer.waitForDeferred(
self.lru.get(c))
yield wfd
res = wfd.getResult()
self.check_result(res, short('b'), 27, 3)
# at this point, we should have 'x', 'a', and 'b' in the cache, and
# 'axx..xxab' in the queue.
self.assertEqual(len(self.lru.queue), 30)
# This 'get' operation for an existing key should cause compaction
wfd = defer.waitForDeferred(
self.lru.get('b'))
yield wfd
res = wfd.getResult()
self.check_result(res, short('b'), 28, 3)
self.assertEqual(len(self.lru.queue), 3)
# expect a cached short('a')
self.lru.miss_fn = self.long_miss_fn
wfd = defer.waitForDeferred(
self.lru.get('a'))
yield wfd
res = wfd.getResult()
self.check_result(res, short('a'), 29, 3)
def incremental(self, _):
wfd = defer.waitForDeferred(
self._sourcedirIsUpdatable())
yield wfd
updatable = wfd.getResult()
if not updatable:
# blow away the old (un-updatable) directory
wfd = defer.waitForDeferred(
self._rmdir(self.workdir))
yield wfd
wfd.getResult()
# and plan to do a checkout
command = ['checkout', self.repourl, '.']
else:
# otherwise, do an update
command = ['update']
if self.revision:
command.extend(['--revision', str(self.revision)])
wfd = defer.waitForDeferred(
self._dovccmd(command))
yield wfd
wfd.getResult()
def _checkCompletedBuildsets(self, bsid, result):
wfd = defer.waitForDeferred(
self.master.db.buildsets.getSubscribedBuildsets(self.schedulerid))
yield wfd
subs = wfd.getResult()
for (sub_bsid, sub_sssetid, sub_complete, sub_results) in subs:
# skip incomplete builds, handling the case where the 'complete'
# column has not been updated yet
if not sub_complete and sub_bsid != bsid:
continue
# build a dependent build if the status is appropriate
if sub_results in (SUCCESS, WARNINGS):
wfd = defer.waitForDeferred(
self.addBuildsetForSourceStamp(setid=sub_sssetid,
reason='downstream'))
yield wfd
wfd.getResult()
# and regardless of status, remove the subscription
wfd = defer.waitForDeferred(
self.master.db.buildsets.unsubscribeFromBuildset(
self.schedulerid, sub_bsid))
yield wfd
wfd.getResult()
def SubmitTryJobChanges(self, changes):
""" Override of SVNPoller.submit_changes:
http://src.chromium.org/viewvc/chrome/trunk/tools/build/scripts/master/try_job_svn.py?view=markup
We modify it so that the patch file url is added to the build properties.
This allows the slave to download the patch directly rather than receiving
it from the master.
"""
for chdict in changes:
# pylint: disable=E1101
parsed = self.parent.parse_options(text_to_dict(chdict['comments']))
# 'fix' revision.
# LKGR must be known before creating the change object.
wfd = defer.waitForDeferred(self.parent.get_lkgr(parsed))
yield wfd
wfd.getResult()
wfd = defer.waitForDeferred(self.master.addChange(
author=','.join(parsed['email']),
revision=parsed['revision'],
comments='',
properties={'patch_file_url': chdict['repository'] + '/' + \
chdict['files'][0]}))
yield wfd
change = wfd.getResult()
self.parent.addChangeInner(chdict['files'], parsed, change.number)
def runner():
for k in xrange(10):
yield waitForDeferred(self.broker.shutdown())
d = Deferred()
reactor.callLater(0.02, d.callback, None)
yield waitForDeferred(d)
def doMaybeStartBuild(self):
# first, if we're not running, then don't start builds; stopService
# uses this to ensure that any ongoing doMaybeStartBuild invocations
# are complete before it stops.
if not self.running:
return
# Check for available slaves. If there are no available slaves, then
# there is no sense continuing
available_slavebuilders = [
sb for sb in self.slaves if sb.isAvailable()
]
if not available_slavebuilders:
self.updateBigStatus()
return
# now, get the available build requests
wfd = defer.waitForDeferred(
self.master.db.buildrequests.getBuildRequests(
buildername=self.name, claimed=False))
yield wfd
unclaimed_requests = wfd.getResult()
# sort by submitted_at, so the first is the oldest
unclaimed_requests.sort(key=lambda brd: brd['submitted_at'])
# get the mergeRequests function for later
mergeRequests_fn = self._getMergeRequestsFn()
# match them up until we're out of options
while available_slavebuilders and unclaimed_requests:
# first, choose a slave (using nextSlave)
wfd = defer.waitForDeferred(
self._chooseSlave(available_slavebuilders))
yield wfd
slavebuilder = wfd.getResult()
if not slavebuilder:
break
if slavebuilder not in available_slavebuilders:
log.msg(("nextSlave chose a nonexistent slave for builder "
"'%s'; cannot start build") % self.name)
break
# then choose a request (using nextBuild)
wfd = defer.waitForDeferred(self._chooseBuild(unclaimed_requests))
yield wfd
breq = wfd.getResult()
if not breq:
break
if breq not in unclaimed_requests:
log.msg(("nextBuild chose a nonexistent request for builder "
"'%s'; cannot start build") % self.name)
break
# merge the chosen request with any compatible requests in the
# queue
wfd = defer.waitForDeferred(
self._mergeRequests(breq, unclaimed_requests,
mergeRequests_fn))
yield wfd
breqs = wfd.getResult()
# try to claim the build requests
try:
wfd = defer.waitForDeferred(
self.master.db.buildrequests.claimBuildRequests(
[brdict['brid'] for brdict in breqs]))
yield wfd
wfd.getResult()
except buildrequests.AlreadyClaimedError:
# one or more of the build requests was already claimed;
# re-fetch the now-partially-claimed build requests and keep
# trying to match them
self._breakBrdictRefloops(unclaimed_requests)
wfd = defer.waitForDeferred(
self.master.db.buildrequests.getBuildRequests(
buildername=self.name, claimed=False))
yield wfd
unclaimed_requests = wfd.getResult()
# go around the loop again
continue
# claim was successful, so initiate a build for this set of
# requests. Note that if the build fails from here on out (e.g.,
# because a slave has failed), it will be handled outside of this
# loop. TODO: test that!
wfd = defer.waitForDeferred(
self._startBuildFor(slavebuilder, breqs))
yield wfd
wfd.getResult()
# and finally remove the buildrequests and slavebuilder from the
# respective queues
self._breakBrdictRefloops(breqs)
for breq in breqs:
unclaimed_requests.remove(breq)
available_slavebuilders.remove(slavebuilder)
self._breakBrdictRefloops(unclaimed_requests)
self.updateBigStatus()
return
def content(self, req, cxt):
status = self.getStatus(req)
builders = req.args.get("builder", status.getBuilderNames())
branches = [b for b in req.args.get("branch", []) if b]
# get counts of pending builds for each builder
brstatus_ds = []
brcounts = {}
def keep_count(statuses, builderName):
brcounts[builderName] = len(statuses)
for builderName in builders:
builder_status = status.getBuilder(builderName)
d = builder_status.getPendingBuildRequestStatuses()
d.addCallback(keep_count, builderName)
brstatus_ds.append(d)
wfd = defer.waitForDeferred(defer.gatherResults(brstatus_ds))
yield wfd
wfd.getResult()
cxt['branches'] = branches
bs = cxt['builders'] = []
building = 0
online = 0
base_builders_url = path_to_root(req) + "builders/"
for bn in builders:
bld = {
'link': base_builders_url + urllib.quote(bn, safe=''),
'name': bn
}
bs.append(bld)
builder = status.getBuilder(bn)
builds = list(
builder.generateFinishedBuilds(map_branches(branches),
num_builds=1))
if builds:
b = builds[0]
bld['build_url'] = (bld['link'] + "/builds/%d" % b.getNumber())
label = b.getProperty("got_revision")
if not label or len(str(label)) > 20:
label = "#%d" % b.getNumber()
bld['build_label'] = label
bld['build_text'] = " ".join(b.getText())
bld['build_css_class'] = build_get_class(b)
current_box = ICurrentBox(builder).getBox(status, brcounts)
bld['current_box'] = current_box.td()
builder_status = builder.getState()[0]
if builder_status == "building":
building += 1
online += 1
elif builder_status != "offline":
online += 1
cxt['authz'] = self.getAuthz(req)
cxt['num_building'] = building
cxt['num_online'] = online
buildForceContext(cxt, req, self.getBuildmaster(req))
template = req.site.buildbot_service.templates.get_template(
"builders.html")
yield template.render(**cxt)
def content(self, req, cxt):
b = self.builder_status
cxt['name'] = b.getName()
req.setHeader('Cache-Control', 'no-cache')
slaves = b.getSlaves()
connected_slaves = [s for s in slaves if s.isConnected()]
cxt['current'] = [self.builder(x, req) for x in b.getCurrentBuilds()]
cxt['pending'] = []
wfd = defer.waitForDeferred(b.getPendingBuildRequestStatuses())
yield wfd
statuses = wfd.getResult()
for pb in statuses:
changes = []
wfd = defer.waitForDeferred(pb.getSourceStamp())
yield wfd
source = wfd.getResult()
wfd = defer.waitForDeferred(pb.getSubmitTime())
yield wfd
submitTime = wfd.getResult()
wfd = defer.waitForDeferred(pb.getBsid())
yield wfd
bsid = wfd.getResult()
wfd = defer.waitForDeferred(
pb.master.db.buildsets.getBuildsetProperties(bsid))
yield wfd
properties = wfd.getResult()
if source.changes:
for c in source.changes:
changes.append({
'url': path_to_change(req, c),
'who': c.who,
'revision': c.revision,
'repo': c.repository
})
cxt['pending'].append({
'when':
time.strftime("%b %d %H:%M:%S", time.localtime(submitTime)),
'delay':
util.formatInterval(util.now() - submitTime),
'id':
pb.brid,
'changes':
changes,
'num_changes':
len(changes),
'properties':
properties,
})
numbuilds = int(req.args.get('numbuilds', ['5'])[0])
recent = cxt['recent'] = []
for build in b.generateFinishedBuilds(num_builds=int(numbuilds)):
recent.append(self.get_line_values(req, build, False))
sl = cxt['slaves'] = []
connected_slaves = 0
for slave in slaves:
s = {}
sl.append(s)
s['link'] = path_to_slave(req, slave)
s['name'] = slave.getName()
c = s['connected'] = slave.isConnected()
if c:
s['admin'] = unicode(slave.getAdmin() or '', 'utf-8')
connected_slaves += 1
cxt['connected_slaves'] = connected_slaves
cxt['authz'] = self.getAuthz(req)
cxt['builder_url'] = path_to_builder(req, b)
buildForceContext(cxt, req, self.getBuildmaster(req), b.getName())
template = req.site.buildbot_service.templates.get_template(
"builder.html")
yield template.render(**cxt)
def TryJobRietveldSubmitJobs(self, jobs):
""" Override of master.try_job_rietveld.TryJobRietveld.SubmitJobs:
http://src.chromium.org/viewvc/chrome/trunk/tools/build/scripts/master/try_job_rietveld.py?view=markup
We modify it to include "baseurl" as a build property.
"""
log.msg('TryJobRietveld.SubmitJobs: %s' % json.dumps(jobs, indent=2))
for job in jobs:
try:
# Gate the try job on the user that requested the job, not the one that
# authored the CL.
# pylint: disable=W0212
########################## Added by rmistry ##########################
if (job.get('requester')
and not job['requester'].endswith('@google.com')
and not job['requester'].endswith('@chromium.org')
and not job['requester'] in TRYBOTS_REQUESTER_WHITELIST):
# Reject the job only if the requester has an email not ending in
# google.com or chromium.org
raise BadJobfile('TryJobRietveld rejecting job from %s' %
job['requester'])
######################################################################
########################## Added by borenet ##########################
if not (job.get('baseurl')
and config_private.Master.Skia.project_name.lower()
in job['baseurl']):
raise BadJobfile(
'TryJobRietveld rejecting job with unknown baseurl: %s' %
job.get('baseurl'))
######################################################################
if job['email'] != job['requester']:
# Note the fact the try job was requested by someone else in the
# 'reason'.
job['reason'] = job.get('reason') or ''
if job['reason']:
job['reason'] += '; '
job['reason'] += "This CL was triggered by %s" % job[
'requester']
options = {
'bot': {
job['builder']: job['tests']
},
'email': [job['email']],
'project': [self._project],
'try_job_key': job['key'],
}
# Transform some properties as is expected by parse_options().
for key in (
########################## Added by borenet ##########################
'baseurl',
######################################################################
'name',
'user',
'root',
'reason',
'clobber',
'patchset',
'issue',
'requester',
'revision'):
options[key] = [job[key]]
# Now cleanup the job dictionary and submit it.
cleaned_job = self.parse_options(options)
wfd = defer.waitForDeferred(self.get_lkgr(cleaned_job))
yield wfd
wfd.getResult()
wfd = defer.waitForDeferred(
self.master.addChange(
author=','.join(cleaned_job['email']),
# TODO(maruel): Get patchset properties to get the list of files.
# files=[],
revision=cleaned_job['revision'],
comments=''))
yield wfd
changeids = [wfd.getResult().number]
wfd = defer.waitForDeferred(self.SubmitJob(cleaned_job, changeids))
yield wfd
wfd.getResult()
except BadJobfile, e:
# We need to mark it as failed otherwise it'll stay in the pending
# state. Simulate a buildFinished event on the build.
if not job.get('key'):
log.err(
'Got %s for issue %s but not key, not updating Rietveld' %
(e, job.get('issue')))
continue
log.err('Got %s for issue %s, updating Rietveld' %
(e, job.get('issue')))
for service in self.master.services:
if service.__class__.__name__ == 'TryServerHttpStatusPush':
# pylint: disable=W0212,W0612
build = {
'properties': [
('buildername', job.get('builder'), None),
('buildnumber', -1, None),
('issue', job['issue'], None),
('patchset', job['patchset'], None),
('project', self._project, None),
('revision', '', None),
('slavename', '', None),
('try_job_key', job['key'], None),
],
'reason':
job.get('reason', ''),
# Use EXCEPTION until SKIPPED results in a non-green try job
# results on Rietveld.
'results':
EXCEPTION,
}
########################## Added by rmistry #########################
# Do not update Rietveld to mark the try job request as failed.
# See https://code.google.com/p/chromium/issues/detail?id=224014 for
# more context.
# service.push('buildFinished', build=build)
#####################################################################
break
def wfd(self, d):
return defer.waitForDeferred(d)
def storeResource(request,
source=None,
source_uri=None,
data=None,
destination=None,
destination_uri=None,
deletesource=False,
depth="0"):
"""
Function that does common PUT/COPY/MOVE behaviour.
@param request: the L{txweb2.server.Request} for the current HTTP request.
@param source: the L{DAVFile} for the source resource to copy from, or None if source data
is to be read from the request.
@param source_uri: the URI for the source resource.
@param data: a C{str} to copy data from instead of the request stream.
@param destination: the L{DAVFile} for the destination resource to copy into.
@param destination_uri: the URI for the destination resource.
@param deletesource: True if the source resource is to be deleted on successful completion, False otherwise.
@param depth: a C{str} containing the COPY/MOVE Depth header value.
@return: status response.
"""
try:
assert request is not None and destination is not None and destination_uri is not None
assert (source is None) or (source is not None
and source_uri is not None)
assert not deletesource or (deletesource and source is not None)
except AssertionError:
log.error("Invalid arguments to storeResource():")
log.error("request=%s\n" % (request, ))
log.error("source=%s\n" % (source, ))
log.error("source_uri=%s\n" % (source_uri, ))
log.error("data=%s\n" % (data, ))
log.error("destination=%s\n" % (destination, ))
log.error("destination_uri=%s\n" % (destination_uri, ))
log.error("deletesource=%s\n" % (deletesource, ))
log.error("depth=%s\n" % (depth, ))
raise
class RollbackState(object):
"""
This class encapsulates the state needed to rollback the entire PUT/COPY/MOVE
transaction, leaving the server state the same as it was before the request was
processed. The DoRollback method will actually execute the rollback operations.
"""
def __init__(self):
self.active = True
self.source_copy = None
self.destination_copy = None
self.destination_created = False
self.source_deleted = False
def Rollback(self):
"""
Rollback the server state. Do not allow this to raise another exception. If
rollback fails then we are going to be left in an awkward state that will need
to be cleaned up eventually.
"""
if self.active:
self.active = False
log.error("Rollback: rollback")
try:
if self.source_copy and self.source_deleted:
self.source_copy.moveTo(source.fp)
log.error("Rollback: source restored %s to %s" %
(self.source_copy.path, source.fp.path))
self.source_copy = None
self.source_deleted = False
if self.destination_copy:
destination.fp.remove()
log.error(
"Rollback: destination restored %s to %s" %
(self.destination_copy.path, destination.fp.path))
self.destination_copy.moveTo(destination.fp)
self.destination_copy = None
elif self.destination_created:
destination.fp.remove()
log.error("Rollback: destination removed %s" %
(destination.fp.path, ))
self.destination_created = False
except:
log.error(
"Rollback: exception caught and not handled: %s" %
Failure())
def Commit(self):
"""
Commit the resource changes by wiping the rollback state.
"""
if self.active:
log.error("Rollback: commit")
self.active = False
if self.source_copy:
self.source_copy.remove()
log.error("Rollback: removed source backup %s" %
(self.source_copy.path, ))
self.source_copy = None
if self.destination_copy:
self.destination_copy.remove()
log.error("Rollback: removed destination backup %s" %
(self.destination_copy.path, ))
self.destination_copy = None
self.destination_created = False
self.source_deleted = False
rollback = RollbackState()
try:
"""
Handle validation operations here.
"""
"""
Handle rollback setup here.
"""
# Do quota checks on destination and source before we start messing with adding other files
destquota = waitForDeferred(destination.quota(request))
yield destquota
destquota = destquota.getResult()
if destquota is not None and destination.exists():
old_dest_size = waitForDeferred(destination.quotaSize(request))
yield old_dest_size
old_dest_size = old_dest_size.getResult()
else:
old_dest_size = 0
if source is not None:
sourcequota = waitForDeferred(source.quota(request))
yield sourcequota
sourcequota = sourcequota.getResult()
if sourcequota is not None and source.exists():
old_source_size = waitForDeferred(source.quotaSize(request))
yield old_source_size
old_source_size = old_source_size.getResult()
else:
old_source_size = 0
else:
sourcequota = None
old_source_size = 0
# We may need to restore the original resource data if the PUT/COPY/MOVE fails,
# so rename the original file in case we need to rollback.
overwrite = destination.exists()
if overwrite:
rollback.destination_copy = FilePath(destination.fp.path)
rollback.destination_copy.path += ".rollback"
destination.fp.copyTo(rollback.destination_copy)
else:
rollback.destination_created = True
if deletesource:
rollback.source_copy = FilePath(source.fp.path)
rollback.source_copy.path += ".rollback"
source.fp.copyTo(rollback.source_copy)
"""
Handle actual store operations here.
"""
# Do put or copy based on whether source exists
if source is not None:
response = maybeDeferred(copy, source.fp, destination.fp,
destination_uri, depth)
else:
datastream = request.stream
if data is not None:
datastream = MemoryStream(data)
md5 = MD5Stream(datastream)
response = maybeDeferred(put, md5, destination.fp)
response = waitForDeferred(response)
yield response
response = response.getResult()
# Update the MD5 value on the resource
if source is not None:
# Copy MD5 value from source to destination
if source.hasDeadProperty(TwistedGETContentMD5):
md5 = source.readDeadProperty(TwistedGETContentMD5)
destination.writeDeadProperty(md5)
else:
# Finish MD5 calc and write dead property
md5.close()
md5 = md5.getMD5()
destination.writeDeadProperty(TwistedGETContentMD5.fromString(md5))
# Update the content-type value on the resource if it is not been copied or moved
if source is None:
content_type = request.headers.getHeader("content-type")
if content_type is not None:
destination.writeDeadProperty(
davxml.GETContentType.fromString(
generateContentType(content_type)))
response = IResponse(response)
# Do quota check on destination
if destquota is not None:
# Get size of new/old resources
new_dest_size = waitForDeferred(destination.quotaSize(request))
yield new_dest_size
new_dest_size = new_dest_size.getResult()
diff_size = new_dest_size - old_dest_size
if diff_size >= destquota[0]:
log.error("Over quota: available %d, need %d" %
(destquota[0], diff_size))
raise HTTPError(
ErrorResponse(responsecode.INSUFFICIENT_STORAGE_SPACE,
(dav_namespace, "quota-not-exceeded")))
d = waitForDeferred(destination.quotaSizeAdjust(
request, diff_size))
yield d
d.getResult()
if deletesource:
# Delete the source resource
if sourcequota is not None:
delete_size = 0 - old_source_size
d = waitForDeferred(
source.quotaSizeAdjust(request, delete_size))
yield d
d.getResult()
delete(source_uri, source.fp, depth)
rollback.source_deleted = True
# Can now commit changes and forget the rollback details
rollback.Commit()
yield response
return
except:
# Roll back changes to original server state. Note this may do nothing
# if the rollback has already ocurred or changes already committed.
rollback.Rollback()
raise
def BlockForEverythingBeingSent():
d = self.serverPushCb()
if d:
x = defer.waitForDeferred(d)
yield x
x.getResult()
def shutdown(self):
"""Shutdown the slave"""
if not self.slave:
log.msg("no remote; slave is already shut down")
return
# First, try the "new" way - calling our own remote's shutdown
# method. The method was only added in 0.8.3, so ignore NoSuchMethod
# failures.
def new_way():
d = self.slave.callRemote('shutdown')
d.addCallback(lambda _: True) # successful shutdown request
def check_nsm(f):
f.trap(pb.NoSuchMethod)
return False # fall through to the old way
d.addErrback(check_nsm)
def check_connlost(f):
f.trap(pb.PBConnectionLost)
return True # the slave is gone, so call it finished
d.addErrback(check_connlost)
return d
wfd = defer.waitForDeferred(new_way())
yield wfd
if wfd.getResult():
return # done!
# Now, the old way. Look for a builder with a remote reference to the
# client side slave. If we can find one, then call "shutdown" on the
# remote builder, which will cause the slave buildbot process to exit.
def old_way():
d = None
for b in self.slavebuilders.values():
if b.remote:
d = b.remote.callRemote("shutdown")
break
if d:
log.msg("Shutting down (old) slave: %s" % self.slavename)
# The remote shutdown call will not complete successfully since the
# buildbot process exits almost immediately after getting the
# shutdown request.
# Here we look at the reason why the remote call failed, and if
# it's because the connection was lost, that means the slave
# shutdown as expected.
def _errback(why):
if why.check(pb.PBConnectionLost):
log.msg("Lost connection to %s" % self.slavename)
else:
log.err("Unexpected error when trying to shutdown %s" %
self.slavename)
d.addErrback(_errback)
return d
log.err("Couldn't find remote builder to shut down slave")
return defer.succeed(None)
def _genBuggy(self):
yield waitForDeferred(getThing())
1/0
def _startBuildFor(self, slavebuilder, buildrequests):
"""Start a build on the given slave.
@param build: the L{base.Build} to start
@param sb: the L{SlaveBuilder} which will host this build
@return: (via Deferred) boolean indicating that the build was
succesfully started.
"""
# as of the Python versions supported now, try/finally can't be used
# with a generator expression. So instead, we push cleanup functions
# into a list so that, at any point, we can abort this operation.
cleanups = []
def run_cleanups():
while cleanups:
fn = cleanups.pop()
fn()
# the last cleanup we want to perform is to update the big
# status based on any other cleanup
cleanups.append(lambda : self.updateBigStatus())
build = self.buildFactory.newBuild(buildrequests)
build.setBuilder(self)
log.msg("starting build %s using slave %s" % (build, slavebuilder))
# set up locks
build.setLocks(self.locks)
cleanups.append(lambda : slavebuilder.slave.releaseLocks())
if len(self.env) > 0:
build.setSlaveEnvironment(self.env)
# append the build to self.building
self.building.append(build)
cleanups.append(lambda : self.building.remove(build))
# update the big status accordingly
self.updateBigStatus()
try:
wfd = defer.waitForDeferred(
slavebuilder.prepare(self.builder_status, build))
yield wfd
ready = wfd.getResult()
except:
log.err(failure.Failure(), 'while preparing slavebuilder:')
ready = False
# If prepare returns True then it is ready and we start a build
# If it returns false then we don't start a new build.
if not ready:
log.msg("slave %s can't build %s after all; re-queueing the "
"request" % (build, slavebuilder))
run_cleanups()
yield False
return
# ping the slave to make sure they're still there. If they've
# fallen off the map (due to a NAT timeout or something), this
# will fail in a couple of minutes, depending upon the TCP
# timeout.
#
# TODO: This can unnecessarily suspend the starting of a build, in
# situations where the slave is live but is pushing lots of data to
# us in a build.
log.msg("starting build %s.. pinging the slave %s"
% (build, slavebuilder))
try:
wfd = defer.waitForDeferred(
slavebuilder.ping())
yield wfd
ping_success = wfd.getResult()
except:
log.err(failure.Failure(), 'while pinging slave before build:')
ping_success = False
if not ping_success:
log.msg("slave ping failed; re-queueing the request")
run_cleanups()
yield False
return
# The buildslave is ready to go. slavebuilder.buildStarted() sets its
# state to BUILDING (so we won't try to use it for any other builds).
# This gets set back to IDLE by the Build itself when it finishes.
slavebuilder.buildStarted()
cleanups.append(lambda : slavebuilder.buildFinished())
# tell the remote that it's starting a build, too
try:
wfd = defer.waitForDeferred(
slavebuilder.remote.callRemote("startBuild"))
yield wfd
wfd.getResult()
except:
log.err(failure.Failure(), 'while calling remote startBuild:')
run_cleanups()
yield False
return
# create the BuildStatus object that goes with the Build
bs = self.builder_status.newBuild()
# record the build in the db - one row per buildrequest
try:
bids = []
for req in build.requests:
wfd = defer.waitForDeferred(
self.master.db.builds.addBuild(req.id, bs.number))
yield wfd
bids.append(wfd.getResult())
except:
log.err(failure.Failure(), 'while adding rows to build table:')
run_cleanups()
yield False
return
# let status know
self.master.status.build_started(req.id, self.name, bs)
# start the build. This will first set up the steps, then tell the
# BuildStatus that it has started, which will announce it to the world
# (through our BuilderStatus object, which is its parent). Finally it
# will start the actual build process. This is done with a fresh
# Deferred since _startBuildFor should not wait until the build is
# finished.
d = build.startBuild(bs, self.expectations, slavebuilder)
d.addCallback(self.buildFinished, slavebuilder, bids)
# this shouldn't happen. if it does, the slave will be wedged
d.addErrback(log.err)
# make sure the builder's status is represented correctly
self.updateBigStatus()
yield True
error = "Request XML body is required."
log.error("Error: {err}", err=error)
raise HTTPError(StatusResponse(responsecode.BAD_REQUEST, error))
#
# Parse request
#
acl = doc.root_element
if not isinstance(acl, davxml.ACL):
error = ("Request XML body must be an acl element."
% (davxml.PropertyUpdate.sname(),))
log.error("Error: {err}", err=error)
raise HTTPError(StatusResponse(responsecode.BAD_REQUEST, error))
#
# Do ACL merger
#
result = waitForDeferred(self.mergeAccessControlList(acl, request))
yield result
result = result.getResult()
#
# Return response
#
if result is None:
yield responsecode.OK
else:
yield ErrorResponse(responsecode.FORBIDDEN, result)
http_ACL = deferredGenerator(http_ACL)
def test_gotChange_treeStableTimer_sequence(self):
sched = self.makeScheduler(self.Subclass,
treeStableTimer=9,
branch='master')
self.master.db.insertTestData([
fakedb.Change(changeid=1, branch='master', when_timestamp=1110),
fakedb.ChangeFile(changeid=1, file='readme.txt'),
fakedb.Change(changeid=2, branch='master', when_timestamp=2220),
fakedb.ChangeFile(changeid=2, file='readme.txt'),
fakedb.Change(changeid=3, branch='master', when_timestamp=3330),
fakedb.ChangeFile(changeid=3, file='readme.txt'),
fakedb.Change(changeid=4, branch='master', when_timestamp=4440),
fakedb.ChangeFile(changeid=4, file='readme.txt'),
])
sched.startService()
self.clock.advance(2220)
# this important change arrives at 2220, so the stable timer will last
# until 2229
wfd = defer.waitForDeferred(
sched.gotChange(
self.makeFakeChange(branch='master', number=1, when=2220),
True))
yield wfd
wfd.getResult()
self.assertEqual(self.events, [])
self.db.schedulers.assertClassifications(self.SCHEDULERID, {1: True})
# but another (unimportant) change arrives before then
self.clock.advance(6) # to 2226
self.assertEqual(self.events, [])
wfd = defer.waitForDeferred(
sched.gotChange(
self.makeFakeChange(branch='master', number=2, when=2226),
False))
yield wfd
wfd.getResult()
self.assertEqual(self.events, [])
self.db.schedulers.assertClassifications(self.SCHEDULERID, {
1: True,
2: False
})
self.clock.advance(3) # to 2229
self.assertEqual(self.events, [])
self.clock.advance(3) # to 2232
self.assertEqual(self.events, [])
# another important change arrives at 2232
wfd = defer.waitForDeferred(
sched.gotChange(
self.makeFakeChange(branch='master', number=3, when=2232),
True))
yield wfd
wfd.getResult()
self.assertEqual(self.events, [])
self.db.schedulers.assertClassifications(self.SCHEDULERID, {
1: True,
2: False,
3: True
})
self.clock.advance(3) # to 2235
self.assertEqual(self.events, [])
# finally, time to start the build!
self.clock.advance(6) # to 2241
self.assertEqual(self.events, ['B[1,2,3]@2241'])
self.db.schedulers.assertClassifications(self.SCHEDULERID, {})
wfd = defer.waitForDeferred(sched.stopService())
yield wfd
wfd.getResult()
def maybeStartBuild(self):
# This method is called by the botmaster whenever this builder should
# check for and potentially start new builds. Do not call this method
# directly - use master.botmaster.maybeStartBuildsForBuilder, or one
# of the other similar methods if more appropriate
# first, if we're not running, then don't start builds; stopService
# uses this to ensure that any ongoing maybeStartBuild invocations
# are complete before it stops.
if not self.running:
return
# Check for available slaves. If there are no available slaves, then
# there is no sense continuing
available_slavebuilders = [ sb for sb in self.slaves
if sb.isAvailable() ]
if not available_slavebuilders:
self.updateBigStatus()
return
# now, get the available build requests
wfd = defer.waitForDeferred(
self.master.db.buildrequests.getBuildRequests(
buildername=self.name, claimed=False))
yield wfd
unclaimed_requests = wfd.getResult()
if not unclaimed_requests:
self.updateBigStatus()
return
# sort by submitted_at, so the first is the oldest
unclaimed_requests.sort(key=lambda brd : brd['submitted_at'])
# get the mergeRequests function for later
mergeRequests_fn = self._getMergeRequestsFn()
# match them up until we're out of options
while available_slavebuilders and unclaimed_requests:
# first, choose a slave (using nextSlave)
wfd = defer.waitForDeferred(
self._chooseSlave(available_slavebuilders))
yield wfd
slavebuilder = wfd.getResult()
if not slavebuilder:
break
if slavebuilder not in available_slavebuilders:
log.msg(("nextSlave chose a nonexistent slave for builder "
"'%s'; cannot start build") % self.name)
break
# then choose a request (using nextBuild)
wfd = defer.waitForDeferred(
self._chooseBuild(unclaimed_requests))
yield wfd
brdict = wfd.getResult()
if not brdict:
break
if brdict not in unclaimed_requests:
log.msg(("nextBuild chose a nonexistent request for builder "
"'%s'; cannot start build") % self.name)
break
# merge the chosen request with any compatible requests in the
# queue
wfd = defer.waitForDeferred(
self._mergeRequests(brdict, unclaimed_requests,
mergeRequests_fn))
yield wfd
brdicts = wfd.getResult()
# try to claim the build requests
brids = [ brdict['brid'] for brdict in brdicts ]
try:
wfd = defer.waitForDeferred(
self.master.db.buildrequests.claimBuildRequests(brids))
yield wfd
wfd.getResult()
except buildrequests.AlreadyClaimedError:
# one or more of the build requests was already claimed;
# re-fetch the now-partially-claimed build requests and keep
# trying to match them
self._breakBrdictRefloops(unclaimed_requests)
wfd = defer.waitForDeferred(
self.master.db.buildrequests.getBuildRequests(
buildername=self.name, claimed=False))
yield wfd
unclaimed_requests = wfd.getResult()
# go around the loop again
continue
# claim was successful, so initiate a build for this set of
# requests. Note that if the build fails from here on out (e.g.,
# because a slave has failed), it will be handled outside of this
# loop. TODO: test that!
# _startBuildFor expects BuildRequest objects, so cook some up
wfd = defer.waitForDeferred(
defer.gatherResults([ self._brdictToBuildRequest(brdict)
for brdict in brdicts ]))
yield wfd
breqs = wfd.getResult()
wfd = defer.waitForDeferred(
self._startBuildFor(slavebuilder, breqs))
yield wfd
build_started = wfd.getResult()
if not build_started:
# build was not started, so unclaim the build requests
wfd = defer.waitForDeferred(
self.master.db.buildrequests.unclaimBuildRequests(brids))
yield wfd
wfd.getResult()
# and try starting builds again. If we still have a working slave,
# then this may re-claim the same buildrequests
self.botmaster.maybeStartBuildsForBuilder(self.name)
# finally, remove the buildrequests and slavebuilder from the
# respective queues
self._breakBrdictRefloops(brdicts)
for brdict in brdicts:
unclaimed_requests.remove(brdict)
available_slavebuilders.remove(slavebuilder)
self._breakBrdictRefloops(unclaimed_requests)
self.updateBigStatus()
return
def _poll(self):
args = []
if self.p4port:
args.extend(['-p', self.p4port])
if self.p4user:
args.extend(['-u', self.p4user])
if self.p4passwd:
args.extend(['-P', self.p4passwd])
args.extend(['changes'])
if self.last_change is not None:
args.extend(['%s...@%d,now' % (self.p4base, self.last_change + 1)])
else:
args.extend(['-m', '1', '%s...' % (self.p4base, )])
wfd = defer.waitForDeferred(self._get_process_output(args))
yield wfd
result = wfd.getResult()
last_change = self.last_change
changelists = []
for line in result.split('\n'):
line = line.strip()
if not line: continue
m = self.changes_line_re.match(line)
if not m:
raise P4PollerError("Unexpected 'p4 changes' output: %r" %
result)
num = int(m.group('num'))
if last_change is None:
# first time through, the poller just gets a "baseline" for where to
# start on the next poll
log.msg('P4Poller: starting at change %d' % num)
self.last_change = num
return
changelists.append(num)
changelists.reverse() # oldest first
# Retrieve each sequentially.
for num in changelists:
args = []
if self.p4port:
args.extend(['-p', self.p4port])
if self.p4user:
args.extend(['-u', self.p4user])
if self.p4passwd:
args.extend(['-P', self.p4passwd])
args.extend(['describe', '-s', str(num)])
wfd = defer.waitForDeferred(self._get_process_output(args))
yield wfd
result = wfd.getResult()
# decode the result from its designated encoding
result = result.decode(self.encoding)
lines = result.split('\n')
# SF#1555985: Wade Brainerd reports a stray ^M at the end of the date
# field. The rstrip() is intended to remove that.
lines[0] = lines[0].rstrip()
m = self.describe_header_re.match(lines[0])
if not m:
raise P4PollerError("Unexpected 'p4 describe -s' result: %r" %
result)
who = m.group('who')
when = time.mktime(time.strptime(m.group('when'), self.datefmt))
comments = ''
while not lines[0].startswith('Affected files'):
comments += lines.pop(0) + '\n'
lines.pop(0) # affected files
branch_files = {} # dict for branch mapped to file(s)
while lines:
line = lines.pop(0).strip()
if not line: continue
m = self.file_re.match(line)
if not m:
raise P4PollerError("Invalid file line: %r" % line)
path = m.group('path')
if path.startswith(self.p4base):
branch, file = self.split_file(path[len(self.p4base):])
if (branch == None and file == None): continue
if branch_files.has_key(branch):
branch_files[branch].append(file)
else:
branch_files[branch] = [file]
for branch in branch_files:
d = self.master.addChange(
author=who,
files=branch_files[branch],
comments=comments,
revision=str(num),
when_timestamp=util.epoch2datetime(when),
branch=branch,
project=self.project)
wfd = defer.waitForDeferred(d)
yield wfd
wfd.getResult()
self.last_change = num
def submit_changes(self, changes):
for chdict in changes:
wfd = defer.waitForDeferred(self.master.addChange(**chdict))
yield wfd
wfd.getResult()
def stopInstance(self, label):
"""Stops an application instance by label
@param label: (string)
@fires Event('instance-stopped')
return defer.Deferred()
"""
result = {}
template = '[%(application)s,%(label)s] %(description)s'
context = {'code': 254}
thisInst = None
try:
thisInst = self.model.getInstance(label)
thisInst.shouldBeRunning = False
if not thisInst.running:
context.update(self.model.statusInstance(label))
raise DroneCommandFailed(context)
pid = thisInst.process.pid
self.log("Trying to shutdown %d gracefully" % (pid, ))
def failed(result):
"""attempting to be consistant"""
self.log("Failed to shutdown process gracefully")
return result
def success(result):
"""attempting to be consistant"""
self.log("process %d gracefully shutdown" % (pid, ))
return result
d = self._start_stop_common(label, 'stopInstance')
d.addCallback(success)
d.addErrback(failed)
d.addErrback(self._killInstance, thisInst)
wfd = defer.waitForDeferred(d)
yield wfd
#refresh the instance as it can change
thisInst = self.model.getInstance(label)
result = wfd.getResult()
if isinstance(result, dict):
context.update(result)
elif isinstance(result, DroneCommandFailed):
context.update(result.resultContext)
if not thisInst.running:
context['code'] = 0
Event('instance-stopped').fire(instance=thisInst)
raise AssertionError('ignore me')
raise DroneCommandFailed(context)
except AssertionError:
#update the instance model
wfd = defer.waitForDeferred(self.statusInstance(label))
yield wfd
result = wfd.getResult()
result['code'] = context['code']
except:
failure = Failure()
if failure.check(DroneCommandFailed):
context = failure.value.resultContext
template = '%(description)s'
else:
temp = "%s: %s" % (getException(failure),
failure.getErrorMessage())
context = {'error': failure, 'code': 253, 'description': temp}
result = self.resultContext(template, thisInst, **context)
try:
thisInst = self.model.getInstance(label)
thisInst.shouldBeRunning = False
except:
pass
yield result
def report_DAV__expand_property(self, request, expand_property):
"""
Generate an expand-property REPORT. (RFC 3253, section 3.8)
"""
# FIXME: Handle depth header
if not isinstance(expand_property, davxml.ExpandProperty):
raise ValueError(
"%s expected as root element, not %s." %
(davxml.ExpandProperty.sname(), expand_property.sname()))
#
# Expand DAV:allprop
#
properties = {}
for property in expand_property.children:
namespace = property.getAttribute("namespace")
name = property.getAttribute("name")
if not namespace: namespace = dav_namespace
if (namespace, name) == (dav_namespace, "allprop"):
all_properties = waitForDeferred(self.listAllProp(request))
yield all_properties
all_properties = all_properties.getResult()
for all_property in all_properties:
properties[all_property.qname()] = property
else:
properties[(namespace, name)] = property
#
# Look up the requested properties
#
properties_by_status = {
responsecode.OK: [],
responsecode.NOT_FOUND: [],
}
for property in properties:
my_properties = waitForDeferred(self.listProperties(request))
yield my_properties
my_properties = my_properties.getResult()
if property in my_properties:
try:
value = waitForDeferred(self.readProperty(property, request))
yield value
value = value.getResult()
if isinstance(value, davxml.HRef):
raise NotImplementedError()
else:
raise NotImplementedError()
except:
f = Failure()
log.err("Error reading property %r for resource %s: %s" %
(property, self, f.value))
status = statusForFailure(
f, "getting property: %s" % (property, ))
if status not in properties_by_status:
properties_by_status[status] = []
raise NotImplementedError()
#properties_by_status[status].append(
# ____propertyName(property)
#)
else:
log.err("Can't find property %r for resource %s" %
(property, self))
properties_by_status[responsecode.NOT_FOUND].append(property)
raise NotImplementedError()
def _genError():
yield waitForDeferred(getThing())
1 / 0
def writeMetric(metric_path,
value,
timestamp,
host,
port,
username,
password,
vhost,
exchange,
spec=None,
channel_number=1,
ssl=False):
if not spec:
spec = txamqp.spec.load(
os.path.normpath(
os.path.join(os.path.dirname(__file__), 'amqp0-8.xml')))
delegate = TwistedDelegate()
connector = ClientCreator(reactor,
AMQClient,
delegate=delegate,
vhost=vhost,
spec=spec)
if ssl:
from twisted.internet.ssl import ClientContextFactory
wfd = waitForDeferred(
connector.connectSSL(host, port, ClientContextFactory()))
yield wfd
conn = wfd.getResult()
else:
wfd = waitForDeferred(connector.connectTCP(host, port))
yield wfd
conn = wfd.getResult()
wfd = waitForDeferred(conn.authenticate(username, password))
yield wfd
wfd = waitForDeferred(conn.channel(channel_number))
yield wfd
channel = wfd.getResult()
wfd = waitForDeferred(channel.channel_open())
yield wfd
wfd = waitForDeferred(
channel.exchange_declare(exchange=exchange,
type="topic",
durable=True,
auto_delete=False))
yield wfd
message = Content("%f %d" % (value, timestamp))
message["delivery mode"] = 2
channel.basic_publish(exchange=exchange,
content=message,
routing_key=metric_path)
wfd = waitForDeferred(channel.channel_close())
yield wfd
uidList = []
# Consumer for listUID - adds to the working set and processes
# a batch if appropriate.
def consumeUIDLine(ent):
uidWorkingSet.append(ent)
if len(uidWorkingSet) >= N:
processBatch()
def processBatch():
L = self.shouldRetrieve(uidWorkingSet)
L.sort()
uidList.extend(L)
del uidWorkingSet[:]
d = defer.waitForDeferred(self.listUID(consumeUIDLine))
self.setStatus(u"Retrieving message list...")
yield d
try:
d.getResult()
except (error.ConnectionDone, error.ConnectionLost):
self.setStatus(u"Connection lost", False)
return
except:
f = failure.Failure()
log.err(f, "Failure retrieving UIDL")
self.setStatus(unicode(f.getErrorMessage()), False)
self.transport.loseConnection()
return
# Clean up any stragglers.
def startInstance(self, label):
"""Starts an application instance by label
@param label: (string)
@fires Event('instance-started')
return defer.Deferred()
"""
template = '[%(application)s,%(label)s] %(description)s'
context = {'description': 'Failed to Start', 'code': 254}
result = {}
thisInst = None
try:
if self.model.getInstance(label).running:
context.update(self.model.statusInstance(label))
raise DroneCommandFailed(context)
d = self._start_stop_common(label, 'startInstance')
wfd = defer.waitForDeferred(d)
yield wfd
result = wfd.getResult()
d = self.statusInstance(label)
wfd = defer.waitForDeferred(d)
yield wfd
result.update(wfd.getResult())
#refresh the instance as it can change
thisInst = self.model.getInstance(label)
if isinstance(result, dict):
context.update(result)
elif isinstance(result, DroneCommandFailed):
context.update(result.resultContext)
if thisInst.running:
Event('instance-started').fire(instance=thisInst)
context['code'] = 0
raise AssertionError('ignore')
raise DroneCommandFailed(context)
except AssertionError:
#update the instance model
wfd = defer.waitForDeferred(self.statusInstance(label))
yield wfd
result = wfd.getResult()
except:
thisInst = self.model.getInstance(label)
failure = Failure()
if failure.check(DroneCommandFailed):
template = '%(description)s'
context = failure.value.resultContext
else:
#log the error, allowing for debugging
self.debugReport()
#be nice and return something to the end user
temp = "%s: %s" % (getException(failure),
failure.getErrorMessage())
context = {'error': failure, 'code': 253, 'description': temp}
result = self.resultContext(template, thisInst, **context)
try:
thisInst = self.model.getInstance(label)
thisInst.shouldBeRunning = True
except:
pass
yield result
returnMinimal = any([key == "return" and value == "minimal" for key, value, _ignore_args in prefer])
noRoot = any([key == "depth-noroot" and value is None for key, value, _ignore_args in prefer])
if not returnMinimal:
returnMinimal = request.headers.getHeader("brief", False)
xml_responses = []
# FIXME: take advantage of the new generative properties of findChildren
my_url = normalizeURL(request_uri)
if self.isCollection() and not my_url.endswith("/"):
my_url += "/"
# Do some optimisation of access control calculation by determining any inherited ACLs outside of
# the child resource loop and supply those to the checkPrivileges on each child.
filtered_aces = waitForDeferred(self.inheritedACEsforChildren(request))
yield filtered_aces
filtered_aces = filtered_aces.getResult()
if depth in ("1", "infinity") and noRoot:
resources = []
else:
resources = [(self, my_url)]
d = self.findChildren(depth, request, lambda x, y: resources.append((x, y)), (davxml.Read(),), inherited_aces=filtered_aces)
x = waitForDeferred(d)
yield x
x.getResult()
for resource, uri in resources:
def _process_changes(self, unused_output):
# get the change list
revListArgs = [
'log',
'%s..origin/%s' % (self.branch, self.branch), r'--format=%H'
]
self.changeCount = 0
d = utils.getProcessOutput(self.gitbin,
revListArgs,
path=self.workdir,
env=dict(PATH=os.environ['PATH']),
errortoo=False)
wfd = defer.waitForDeferred(d)
yield wfd
results = wfd.getResult()
# process oldest change first
revList = results.split()
if not revList:
return
revList.reverse()
self.changeCount = len(revList)
log.msg('gitpoller: processing %d changes: %s in "%s"' %
(self.changeCount, revList, self.workdir))
for rev in revList:
dl = defer.DeferredList([
self._get_commit_timestamp(rev),
self._get_commit_name(rev),
self._get_commit_files(rev),
self._get_commit_comments(rev),
],
consumeErrors=True)
wfd = defer.waitForDeferred(dl)
yield wfd
results = wfd.getResult()
# check for failures
failures = [r[1] for r in results if not r[0]]
if failures:
# just fail on the first error; they're probably all related!
raise failures[0]
revlink = ''
if self.revlinktmpl and rev:
revlink = self.revlinktmpl % urllib.quote_plus(rev)
timestamp, name, files, comments = [r[1] for r in results]
d = self.master.addChange(author=name,
revision=rev,
files=files,
comments=comments,
when_timestamp=epoch2datetime(timestamp),
branch=self.branch,
category=self.category,
project=self.project,
repository=self.repourl,
revlink=revlink)
wfd = defer.waitForDeferred(d)
yield wfd
results = wfd.getResult()
def addBuildsetForChangesMultiRepo(self,
reason='',
external_idstring=None,
changeids=[],
builderNames=None,
properties=None):
assert changeids is not []
chDicts = {}
def getChange(changeid=None):
d = self.master.db.changes.getChange(changeid)
def chdict2change(chdict):
if not chdict:
return None
return changes.Change.fromChdict(self.master, chdict)
d.addCallback(chdict2change)
def groupChange(change):
if change.repository not in chDicts:
chDicts[change.repository] = []
chDicts[change.repository].append(change)
def get_changeids_from_repo(repository):
changeids = []
for change in chDicts[repository]:
changeids.append(change.number)
return changeids
def create_sourcestamp(changeids, change=None, setid=None):
def add_sourcestamp(setid, changeids=None):
return self.master.db.sourcestamps.addSourceStamp(
branch=change.branch,
revision=change.revision,
repository=change.repository,
project=change.project,
changeids=changeids,
setid=setid)
d.addCallback(add_sourcestamp, setid=setid, changeids=changeids)
return d
def create_sourcestamp_without_changes(setid, repository):
return self.master.db.sourcestamps.addSourceStamp(
branch=self.default_branch,
revision=None,
repository=repository,
project=self.default_project,
changeids=changeids,
sourcestampsetid=setid)
d = defer.Deferred()
if self.repositories is None:
# attributes for this sourcestamp will be based on the most recent
# change, so fetch the change with the highest id (= old single
# sourcestamp functionality)
d.addCallBack(getChange, changeid=max(changeids))
d.addCallBack(groupChange)
else:
for changeid in changeids:
d.addCallBack(getChange, changeid=changeid)
d.addCallBack(groupChange)
# Define setid for this set of changed repositories
wfd = defer.waitForDeferred(
self.master.db.sourcestampsets.addSourceStampSet)
yield wfd
setid = wfd.getResult()
#process all unchanged repositories
if self.repositories is not None:
for repo in self.repositories:
if repo not in chDicts:
# repository was not changed
# call create_sourcestamp
d.addCallback(create_sourcestamp_without_changes, setid,
repo)
# process all changed
for repo in chDicts:
d.addCallback(get_changeids_from_repo, repository=repo)
d.addCallback(create_sourcestamp,
setid=setid,
change=chDicts[repo][-1])
# add one buildset, this buildset is connected to the sourcestamps by the setid
d.addCallback(self.addBuildsetForSourceStamp,
setid=setid,
reason=reason,
external_idstring=external_idstring,
builderNames=builderNames,
properties=properties)
yield d
if self.exists():
log.error("Attempt to create collection where file exists: %s"
% (self,))
raise HTTPError(responsecode.NOT_ALLOWED)
if not parent.isCollection():
log.error("Attempt to create collection with non-collection parent: %s"
% (self,))
raise HTTPError(StatusResponse(
responsecode.CONFLICT,
"Parent resource is not a collection."
))
#
# Read request body
#
x = waitForDeferred(noDataFromStream(request.stream))
yield x
try:
x.getResult()
except ValueError, e:
log.error("Error while handling MKCOL body: %s" % (e,))
raise HTTPError(responsecode.UNSUPPORTED_MEDIA_TYPE)
response = waitForDeferred(mkcollection(self.fp))
yield response
yield response.getResult()
http_MKCOL = deferredGenerator(http_MKCOL)
def perspective_commandline(self, op, bb_username, bb_password, ids, info):
"""
This performs the requested operations from the `buildbot user`
call by calling the proper buildbot.db.users methods based on
the operation. It yields a deferred instance with the results
from the database methods.
@param op: operation to perform (add, remove, update, get)
@type op: string
@param bb_username: username portion of auth credentials
@type bb_username: string
@param bb_password: hashed password portion of auth credentials
@type bb_password: hashed string
@param ids: user identifiers used to find existing users
@type ids: list of strings or None
@param info: type/value pairs for each user that will be added
or updated in the database
@type info: list of dictionaries or None
@returns: results from db.users methods via deferred
"""
log.msg("perspective_commandline called")
results = []
if ids:
for user in ids:
# get identifier, guaranteed to be in user from checks
# done in C{scripts.runner}
d = self.master.db.users.identifierToUid(identifier=user)
wfd = defer.waitForDeferred(d)
yield wfd
uid = wfd.getResult()
result = None
if op == 'remove':
if uid:
d = self.master.db.users.removeUser(uid)
wfd = defer.waitForDeferred(d)
yield wfd
wfd.getResult()
result = user
else:
log.msg("Unable to find uid for identifier %s" % user)
elif op == 'get':
if uid:
d = self.master.db.users.getUser(uid)
wfd = defer.waitForDeferred(d)
yield wfd
result = wfd.getResult()
else:
log.msg("Unable to find uid for identifier %s" % user)
results.append(result)
else:
for user in info:
# get identifier, guaranteed to be in user from checks
# done in C{scripts.runner}
ident = user.pop('identifier')
d = self.master.db.users.identifierToUid(identifier=ident)
wfd = defer.waitForDeferred(d)
yield wfd
uid = wfd.getResult()
# if only an identifier was in user, we're updating only
# the bb_username and bb_password.
if not user:
if uid:
d = self.master.db.users.updateUser(
uid=uid,
identifier=ident,
bb_username=bb_username,
bb_password=bb_password)
wfd = defer.waitForDeferred(d)
yield wfd
results.append(ident)
result = wfd.getResult()
else:
log.msg("Unable to find uid for identifier %s" % user)
else:
# when adding, we update the user after the first attr
once_through = False
for attr in user:
if op == 'update' or once_through:
if uid:
d = self.master.db.users.updateUser(
uid=uid,
identifier=ident,
bb_username=bb_username,
bb_password=bb_password,
attr_type=attr,
attr_data=user[attr])
else:
log.msg(
"Unable to find uid for identifier %s" %
user)
elif op == 'add':
d = self.master.db.users.findUserByAttr(
identifier=ident,
attr_type=attr,
attr_data=user[attr])
once_through = True
wfd = defer.waitForDeferred(d)
yield wfd
results.append(ident)
result = wfd.getResult()
# result is None from updateUser calls
if result:
results.append(result)
uid = result
results = self.formatResults(op, results)
yield results
def addBuildsetForSourceStamp(self,
ssid=None,
setid=None,
reason='',
external_idstring=None,
properties=None,
builderNames=None):
"""
Add a buildset for the given, already-existing sourcestamp.
This method will add any properties provided to the scheduler
constructor to the buildset, and will call the master's
L{BuildMaster.addBuildset} method with the appropriate parameters, and
return the same result.
@param reason: reason for this buildset
@type reason: unicode string
@param external_idstring: external identifier for this buildset, or None
@param properties: a properties object containing initial properties for
the buildset
@type properties: L{buildbot.process.properties.Properties}
@param builderNames: builders to name in the buildset (defaults to
C{self.builderNames})
@param setid: idenitification of a set of sourcestamps
@returns: (buildset ID, buildrequest IDs) via Deferred
"""
assert (ssid is None and setid is not None) \
or (ssid is not None and setid is None), "pass a single sourcestamp OR set not both"
# combine properties
if properties:
properties.updateFromProperties(self.properties)
else:
properties = self.properties
# apply the default builderNames
if not builderNames:
builderNames = self.builderNames
# translate properties object into a dict as required by the
# addBuildset method
properties_dict = properties.asDict()
if setid == None:
if ssid != None:
wfd = defer.waitForDeferred(
self.master.db.sourcestamps.getSourceStamp(ssid))
yield wfd
ssdict = wfd.getResult()
setid = ssdict['sourcestampsetid']
else:
# no sourcestamp and no sets
yield None
wfd = defer.waitForDeferred(
self.master.addBuildset(sourcestampsetid=setid,
reason=reason,
properties=properties_dict,
builderNames=builderNames,
external_idstring=external_idstring))
yield wfd
yield wfd.getResult()
def connectionMade(self):
AMQClient.connectionMade(self)
log.listener("New AMQP connection made")
self.setup()
wfd = waitForDeferred(self.receive_loop())
yield wfd
def addBuildsetForChanges(self,
reason='',
external_idstring=None,
changeids=[],
builderNames=None,
properties=None):
"""
Add a buildset for the combination of the given changesets, creating
a sourcestamp based on those changes. The sourcestamp for the buildset
will reference all of the indicated changes.
This method will add any properties provided to the scheduler
constructor to the buildset, and will call the master's addBuildset
method with the appropriate parameters.
@param reason: reason for this buildset
@type reason: unicode string
@param external_idstring: external identifier for this buildset, or None
@param changeids: nonempty list of changes to include in this buildset
@param builderNames: builders to name in the buildset (defaults to
C{self.builderNames})
@param properties: a properties object containing initial properties for
the buildset
@type properties: L{buildbot.process.properties.Properties}
@returns: (buildset ID, buildrequest IDs) via Deferred
"""
assert changeids is not []
# attributes for this sourcestamp will be based on the most recent
# change, so fetch the change with the highest id
wfd = defer.waitForDeferred(
self.master.db.changes.getChange(max(changeids)))
yield wfd
chdict = wfd.getResult()
change = None
if chdict:
wfd = defer.waitForDeferred(
changes.Change.fromChdict(self.master, chdict))
yield wfd
change = wfd.getResult()
# Define setid for this set of changed repositories
wfd = defer.waitForDeferred(
self.master.db.sourcestampsets.addSourceStampSet())
yield wfd
setid = wfd.getResult()
wfd = defer.waitForDeferred(
self.master.db.sourcestamps.addSourceStamp(
branch=change.branch,
revision=change.revision,
repository=change.repository,
project=change.project,
changeids=changeids,
sourcestampsetid=setid))
yield wfd
wfd.getResult()
wfd = defer.waitForDeferred(
self.addBuildsetForSourceStamp(setid=setid,
reason=reason,
external_idstring=external_idstring,
builderNames=builderNames,
properties=properties))
yield wfd
yield wfd.getResult()
