def test_reclaimAllBuilds(self): wfd = defer.waitForDeferred( self.makeBuilder()) yield wfd wfd.getResult() claims = [] def fakeClaimBRs(*args): claims.append(args) return defer.succeed(None) self.bldr.master.db.buildrequests.claimBuildRequests = fakeClaimBRs self.bldr.master.db.buildrequests.reclaimBuildRequests = fakeClaimBRs def mkbld(brids): bld = mock.Mock(name='Build') bld.requests = [] for brid in brids: br = mock.Mock(name='BuildRequest %d' % brid) br.id = brid bld.requests.append(br) return bld old = mkbld([15]) # keep a reference to the "old" build self.bldr.old_building[old] = None self.bldr.building.append(mkbld([10,11,12])) wfd = defer.waitForDeferred( self.bldr.reclaimAllBuilds()) yield wfd wfd.getResult() self.assertEqual(claims, [ (set([10,11,12,15]),) ])
def http_PROPFIND(self, request): """ Respond to a PROPFIND request. (RFC 2518, section 8.1) """ if not self.exists(): log.err("File not found: %s" % (self,)) raise HTTPError(responsecode.NOT_FOUND) # # Check authentication and access controls # x = waitForDeferred(self.authorize(request, (davxml.Read(),))) yield x x.getResult() # # Read request body # try: doc = waitForDeferred(davXMLFromStream(request.stream)) yield doc doc = doc.getResult() except ValueError, e: log.err("Error while handling PROPFIND body: %s" % (e,)) raise HTTPError(StatusResponse(responsecode.BAD_REQUEST, str(e)))
def produceAllMetrics(host, port, protocol, timeout=5.0, delay=0.0): """Produce ALL metrics of ALL metricID's to the endpoint with some delay in between. @param host (string) @param port (int) @param protocol (class) - twisted.internet.protocol.Protocol @param timeout (int|float) - timeout parameter for the protocol @param delay (int|float) - delay between sends @callback (int) - how many metrics were sent @errback N/A @return defer.Deferred() """ result = 0 for obj in TimeSeriesData.objects: while obj.pending: try: d = obj.produce_all(host, port, protocol, timeout=timeout) wfd = defer.waitForDeferred(d) yield wfd result += wfd.getResult() if delay: d = defer.Deferred() config.reactor.callLater(delay, d.callback, None) wfd = defer.waitForDeferred(d) yield wfd wfd.getResult() except: pass #swallow errors yield result
def test_stopService_flushes(self): wfd = defer.waitForDeferred( self.makeBuilder()) yield wfd wfd.getResult() # just check that stopService calls this and waits # for the deferred to fire events = [] long_d = defer.Deferred() long_d.addCallback(lambda _ : events.append('long_d')) self.bldr.maybeStartBuild = lambda : long_d stop_d = self.bldr.stopService() stop_d.addCallback(lambda _ : events.append('stop_d')) # nothing should have happened yet self.assertEqual(events, []) # finish the maybeStartBuild invocation.. long_d.callback(None) wfd = defer.waitForDeferred(stop_d) yield wfd wfd.getResult() # and then check that things happened in the right order self.assertEqual(events, [ 'long_d', 'stop_d' ])
def produce(self): """Produce entries.""" while not self._paused: wfd = defer.waitForDeferred(self._write_pending()) yield wfd wfd.getResult() if self.finished: self.finishProducing() return first = self._last_fetching while (self._currently_fetching <= self._max_currently_fetching and self._last_fetching <= self._end and self._currently_stored <= FLAGS.entries_buffer): last = min(self._last_fetching + self._batch_size - 1, self._end, self._last_fetching + self._max_currently_fetching - self._currently_fetching + 1) self._batches.put(self._create_fetch_deferred(first, last)) self._currently_fetching += last - first + 1 first = last + 1 self._last_fetching = first wfd = defer.waitForDeferred(self._batches.get()) # Pause here until the body of the response is available. yield wfd # The producer may have been paused while waiting for the response, # or errored out upon receiving it: do not write the entries out # until after the next self._paused check. self._pending = wfd.getResult()
def scanExistingClassifiedChanges(self): # call gotChange for each classified change. This is called at startup # and is intended to re-start the treeStableTimer for any changes that # had not yet been built when the scheduler was stopped. # NOTE: this may double-call gotChange for changes that arrive just as # the scheduler starts up. In practice, this doesn't hurt anything. wfd = defer.waitForDeferred( self.master.db.schedulers.getChangeClassifications( self.schedulerid)) yield wfd classifications = wfd.getResult() # call gotChange for each change, after first fetching it from the db for changeid, important in classifications.iteritems(): wfd = defer.waitForDeferred( self.master.db.changes.getChange(changeid)) yield wfd chdict = wfd.getResult() if not chdict: continue wfd = defer.waitForDeferred( changes.Change.fromChdict(self.master, chdict)) yield wfd change = wfd.getResult() wfd = defer.waitForDeferred( self.gotChange(change, important)) yield wfd wfd.getResult()
def stableTimerFired(self, timer_name): # if the service has already been stoppd then just bail out if not self._stable_timers[timer_name]: return # delete this now-fired timer del self._stable_timers[timer_name] wfd = defer.waitForDeferred( self.getChangeClassificationsForTimer(self.schedulerid, timer_name)) yield wfd classifications = wfd.getResult() # just in case: databases do weird things sometimes! if not classifications: # pragma: no cover return changeids = sorted(classifications.keys()) wfd = defer.waitForDeferred( self.addBuildsetForChanges(reason='scheduler', changeids=changeids)) yield wfd wfd.getResult() max_changeid = changeids[-1] # (changeids are sorted) wfd = defer.waitForDeferred( self.master.db.schedulers.flushChangeClassifications( self.schedulerid, less_than=max_changeid+1)) yield wfd wfd.getResult()
def http_MKCOL(self, request): """ Respond to a MKCOL request. (RFC 2518, section 8.3) """ parent = waitForDeferred(request.locateResource(parentForURL(request.uri))) yield parent parent = parent.getResult() x = waitForDeferred(parent.authorize(request, (davxml.Bind(),))) yield x x.getResult() if self.exists(): log.err("Attempt to create collection where file exists: %s" % (self,)) raise HTTPError(responsecode.NOT_ALLOWED) if not parent.isCollection(): log.err("Attempt to create collection with non-collection parent: %s" % (self,)) raise HTTPError(StatusResponse( responsecode.CONFLICT, "Parent resource is not a collection." )) # # Read request body # x = waitForDeferred(noDataFromStream(request.stream)) yield x try: x.getResult() except ValueError, e: log.err("Error while handling MKCOL body: %s" % (e,)) raise HTTPError(responsecode.UNSUPPORTED_MEDIA_TYPE)
def deleteResource(request, resource, resource_uri, depth="0"): """ Handle a resource delete with proper quota etc updates """ if not resource.exists(): log.error("File not found: %s" % (resource,)) raise HTTPError(responsecode.NOT_FOUND) # Do quota checks before we start deleting things myquota = waitForDeferred(resource.quota(request)) yield myquota myquota = myquota.getResult() if myquota is not None: old_size = waitForDeferred(resource.quotaSize(request)) yield old_size old_size = old_size.getResult() else: old_size = 0 # Do delete x = waitForDeferred(delete(resource_uri, resource.fp, depth)) yield x result = x.getResult() # Adjust quota if myquota is not None: d = waitForDeferred(resource.quotaSizeAdjust(request, -old_size)) yield d d.getResult() yield result
def test_reconfig_bad_config(self): reactor = self.make_reactor() self.master.reconfigService = mock.Mock( side_effect=lambda n : defer.succeed(None)) wfd = defer.waitForDeferred( self.master.startService(_reactor=reactor)) yield wfd wfd.getResult() # reset, since startService called reconfigService self.master.reconfigService.reset_mock() # reconfig, with a failure self.patch_loadConfig_fail() wfd = defer.waitForDeferred( self.master.reconfig()) yield wfd wfd.getResult() self.master.stopService() self.assertLogged("reconfig aborted without") self.failIf(self.master.reconfigService.called)
def test_reconfigService_default_changed(self): config = mock.Mock() config.slavePortnum = '9876' self.attachChangeSource(pb.PBChangeSource()) self.startChangeSource() wfd = defer.waitForDeferred( self.changesource.reconfigService(config)) yield wfd wfd.getResult() self.assertRegistered('9876', 'change', 'changepw') config.slavePortnum = '1234' wfd = defer.waitForDeferred( self.changesource.reconfigService(config)) yield wfd wfd.getResult() self.assertUnregistered('9876', 'change', 'changepw') self.assertRegistered('1234', 'change', 'changepw') wfd = defer.waitForDeferred( self.stopChangeSource()) yield wfd wfd.getResult() self.assertUnregistered('1234', 'change', 'changepw')
def test_brdictToBuildRequest(self): self.makeBuilder() # set up all of the data required for a BuildRequest object wfd = defer.waitForDeferred( self.db.insertTestData([ fakedb.SourceStamp(id=234), fakedb.Buildset(id=30, sourcestampid=234, reason='foo', submitted_at=1300305712, results=-1), fakedb.BuildRequest(id=19, buildsetid=30, buildername='bldr', priority=13, submitted_at=1300305712, results=-1), ])) yield wfd wfd.getResult() wfd = defer.waitForDeferred( self.db.buildrequests.getBuildRequest(19)) yield wfd brdict = wfd.getResult() wfd = defer.waitForDeferred( self.bldr._brdictToBuildRequest(brdict)) yield wfd br = wfd.getResult() # just check that the BuildRequest looks reasonable - # test_process_buildrequest checks the whole thing self.assertEqual(br.reason, 'foo') # and check that the cross-pointers are correct self.assertIdentical(br.brdict, brdict) self.assertIdentical(brdict['brobj'], br) self.bldr._breakBrdictRefloops([brdict])
def full(self, _): if self.method == 'clobber': wfd = defer.waitForDeferred(self.clobber()) yield wfd wfd.getResult() return elif self.method in ['copy', 'export']: wfd = defer.waitForDeferred(self.copy()) yield wfd wfd.getResult() return wfd = defer.waitForDeferred(self._sourcedirIsUpdatable()) yield wfd updatable = wfd.getResult() if not updatable: d = self._dovccmd(['checkout', self.svnurl, '.']) elif self.method == 'clean': d = self.clean() elif self.method == 'fresh': d = self.fresh() wfd = defer.waitForDeferred(d) yield wfd wfd.getResult()
def _mergeRequests(self, breq, unclaimed_requests, mergeRequests_fn): """Use C{mergeRequests_fn} to merge C{breq} against C{unclaimed_requests}, where both are build request dictionaries""" # short circuit if there is no merging to do if not mergeRequests_fn or len(unclaimed_requests) == 1: yield [ breq ] return # we'll need BuildRequest objects, so get those first wfd = defer.waitForDeferred( defer.gatherResults( [ self._brdictToBuildRequest(brdict) for brdict in unclaimed_requests ])) yield wfd unclaimed_request_objects = wfd.getResult() breq_object = unclaimed_request_objects.pop( unclaimed_requests.index(breq)) # gather the mergeable requests merged_request_objects = [breq_object] for other_breq_object in unclaimed_request_objects: wfd = defer.waitForDeferred( defer.maybeDeferred(lambda : mergeRequests_fn(self, breq_object, other_breq_object))) yield wfd if wfd.getResult(): merged_request_objects.append(other_breq_object) # convert them back to brdicts and return merged_requests = [ br.brdict for br in merged_request_objects ] yield merged_requests
def start(self): args = self.args # args['dir'] is relative to Builder directory, and is required. assert args["dir"] is not None dirnames = args["dir"] self.timeout = args.get("timeout", 120) self.maxTime = args.get("maxTime", None) self.rc = 0 if type(dirnames) is list: assert len(dirnames) != 0 for dirname in dirnames: wfd = defer.waitForDeferred(self.removeSingleDir(dirname)) yield wfd res = wfd.getResult() # Even if single removal of single file/dir consider it as # failure of whole command, but continue removing other files # Send 'rc' to master to handle failure cases if res != 0: self.rc = res else: wfd = defer.waitForDeferred(self.removeSingleDir(dirnames)) yield wfd self.rc = wfd.getResult() self.sendStatus({"rc": self.rc})
def full(self): if self.method == 'clobber': wfd = defer.waitForDeferred(self.clobber()) yield wfd wfd.getResult() return elif self.method == 'copy': wfd = defer.waitForDeferred(self.copy()) yield wfd wfd.getResult() return wfd = defer.waitForDeferred(self._sourcedirIsUpdatable()) yield wfd updatable = wfd.getResult() if not updatable: log.msg("No git repo present, making full clone") d = self._doFull() elif self.method == 'clean': d = self.clean() elif self.method == 'fresh': d = self.fresh() else: raise ValueError("Unknown method, check your configuration") wfd = defer.waitForDeferred(d) yield wfd wfd.getResult()
def asDict_async(self): result = {} wfd = defer.waitForDeferred( self.getSourceStamp()) yield wfd ss = wfd.getResult() result['source'] = ss.asDict() result['builderName'] = self.getBuilderName() wfd = defer.waitForDeferred( self.getSubmitTime()) yield wfd submittedAt = wfd.getResult() result['submittedAt'] = submittedAt wfd = defer.waitForDeferred( self.getBuilds()) yield wfd builds = wfd.getResult() result['builds'] = [ build.asDict() for build in builds ] yield result
def process_all(self): """ Process data when no virtual server and no real server are provided. """ # Retrieve all data for oid in self.oids: w = defer.waitForDeferred(self.proxy.walk(self.oids[oid])) yield w w.getResult() # For each virtual server, build it for ov in self.cache('ltmVirtualServAddrType'): # Grab HTTP class try: classes = self.cache(('ltmVsHttpClassProfileName', ".".join([str(x) for x in ov]))).values() except KeyError: classes = [] for httpclass in classes + [None]: v = oid2str(ov) vs = defer.waitForDeferred(self.process_vs(v, httpclass)) yield vs vs = vs.getResult() if vs is not None: if httpclass is not None: self.lb.virtualservers["%s;%s" % (v, httpclass)] = vs else: self.lb.virtualservers[v] = vs yield self.lb return
def execute(self, action, actionargs=None, vs=None, rs=None): """ Execute an action. @param action: action to be executed """ v, r = self.parse(vs, rs) if r is None: yield None return if action == "disable" or action == "operdisable": d = defer.waitForDeferred( self.proxy.set((self.oids['realServerWeight'], v, r), 0)) yield d d.getResult() yield True return if action == "enable" or action == "operenable": try: weight = int(actionargs[0]) except ValueError, KeyError: weight = 1 d = defer.waitForDeferred( self.proxy.set((self.oids['realServerWeight'], v, r), weight)) yield d d.getResult() yield True return
def full(self, _): if self.method == "clobber": wfd = defer.waitForDeferred(self.clobber()) yield wfd wfd.getResult() return elif self.method in ["copy", "export"]: wfd = defer.waitForDeferred(self.copy()) yield wfd wfd.getResult() return wfd = defer.waitForDeferred(self._sourcedirIsUpdatable()) yield wfd updatable = wfd.getResult() if not updatable: d = self._dovccmd(["checkout", self.svnurl, "."]) elif self.method == "clean": d = self.clean() elif self.method == "fresh": d = self.fresh() wfd = defer.waitForDeferred(d) yield wfd wfd.getResult()
def cancelBuildRequest(self): # first, try to claim the request; if this fails, then it's too late to # cancel the build anyway try: wfd = defer.waitForDeferred( self.master.db.buildrequests.claimBuildRequests([self.id])) yield wfd wfd.getResult() except buildrequests.AlreadyClaimedError: log.msg("build request already claimed; cannot cancel") return # then complete it with 'FAILURE'; this is the closest we can get to # cancelling a request without running into trouble with dangling # references. wfd = defer.waitForDeferred( self.master.db.buildrequests.completeBuildRequests([self.id], FAILURE)) yield wfd wfd.getResult() # and let the master know that the enclosing buildset may be complete wfd = defer.waitForDeferred( self.master.maybeBuildsetComplete(self.bsid)) yield wfd wfd.getResult()
def _sourcedirIsUpdatable(self): # first, perform a stat to ensure that this is really an svn directory cmd = buildstep.RemoteCommand('stat', {'file': self.workdir + '/.svn', 'logEnviron': self.logEnviron,}) cmd.useLog(self.stdio_log, False) wfd = defer.waitForDeferred( self.runCommand(cmd)) yield wfd wfd.getResult() if cmd.rc != 0: yield False return # then run 'svn info' to check that the URL matches our repourl wfd = defer.waitForDeferred( self._dovccmd(['info'], collectStdout=True)) yield wfd stdout = wfd.getResult() # extract the URL, handling whitespace carefully so that \r\n works # is a line terminator mo = re.search('^URL:\s*(.*?)\s*$', stdout, re.M) yield mo and mo.group(1) == self.repourl return
def test(): print "STARTING TEST" foo = paisley.CouchDB('localhost') print "\nCreate database 'testdb':" d = foo.createDB('testdb1') d.addCallback(lambda f: log.err(f)) wfd = defer.waitForDeferred(d) yield wfd try: print wfd.getResult() except Exception as e: # FIXME: not sure why Error.status is a str compared to http constants if hasattr(e, 'status') and e.status == str(http.UNAUTHORIZED): print "\nError: not allowed to create databases" reactor.stop() return else: raise print "\nList databases on server:" d = foo.listDB() wfd = defer.waitForDeferred(d) yield wfd print wfd.getResult() reactor.stop()
def test_reconfigService_class_name_change(self): sch1 = self.makeSched(self.ReconfigSched, 'sch1') self.new_config.schedulers = dict(sch1=sch1) wfd = defer.waitForDeferred( self.sm.reconfigService(self.new_config)) yield wfd wfd.getResult() self.assertIdentical(sch1.parent, self.sm) self.assertIdentical(sch1.master, self.master) self.assertEqual(sch1.reconfig_count, 1) sch1_new = self.makeSched(self.ReconfigSched2, 'sch1') self.new_config.schedulers = dict(sch1=sch1_new) wfd = defer.waitForDeferred( self.sm.reconfigService(self.new_config)) yield wfd wfd.getResult() # sch1 had its class name change, so sch1_new is now the active # instance self.assertIdentical(sch1_new.parent, self.sm) self.assertIdentical(sch1_new.master, self.master)
def test_reconfigService_add_and_change_and_remove_no_reconfig(self): sch1 = self.makeSched(self.Sched, 'sch1', attr='alpha') self.new_config.schedulers = dict(sch1=sch1) wfd = defer.waitForDeferred( self.sm.reconfigService(self.new_config)) yield wfd wfd.getResult() self.assertIdentical(sch1.parent, self.sm) self.assertIdentical(sch1.master, self.master) sch1_new = self.makeSched(self.Sched, 'sch1', attr='beta') sch2 = self.makeSched(self.Sched, 'sch2', attr='alpha') self.new_config.schedulers = dict(sch1=sch1_new, sch2=sch2) wfd = defer.waitForDeferred( self.sm.reconfigService(self.new_config)) yield wfd wfd.getResult() # sch1 is not longer active, and sch1_new is self.assertIdentical(sch1.parent, None) self.assertIdentical(sch1.master, None) self.assertIdentical(sch1_new.parent, self.sm) self.assertIdentical(sch1_new.master, self.master) self.assertIdentical(sch2.parent, self.sm) self.assertIdentical(sch2.master, self.master)
def _deferred(deferredResult, caller): """only callable by ``newfunc``""" from twisted.internet import reactor result = None attempt = 0 while attempt <= maximum: try: wfd = defer.waitForDeferred(deferredResult) yield wfd result = wfd.getResult() break # didn't throw exception then we have the result except SystemExit: result = Failure() break except KeyboardInterrupt: result = Failure() break except: attempt += 1 if attempt > maximum: result = Failure() break # failure captured d = defer.Deferred() reactor.callLater(delay, d.callback, None) wfd = defer.waitForDeferred(d) yield wfd wfd.getResult() # fortunatly the caller has been set up in advance caller.write(">>> Retry attempt %d" % attempt) deferredResult = caller(*caller.args, **caller.kwargs) yield result
def test_queue_collapsing(self): # just to check that we're practicing with the right queue size (so # QUEUE_SIZE_FACTOR is 10) self.assertEqual(self.lru.max_queue, 30) for c in 'a' + 'x' * 27 + 'ab': wfd = defer.waitForDeferred( self.lru.get(c)) yield wfd res = wfd.getResult() self.check_result(res, short('b'), 27, 3) # at this point, we should have 'x', 'a', and 'b' in the cache, and # 'axx..xxab' in the queue. self.assertEqual(len(self.lru.queue), 30) # This 'get' operation for an existing key should cause compaction wfd = defer.waitForDeferred( self.lru.get('b')) yield wfd res = wfd.getResult() self.check_result(res, short('b'), 28, 3) self.assertEqual(len(self.lru.queue), 3) # expect a cached short('a') self.lru.miss_fn = self.long_miss_fn wfd = defer.waitForDeferred( self.lru.get('a')) yield wfd res = wfd.getResult() self.check_result(res, short('a'), 29, 3)
def incremental(self, _): wfd = defer.waitForDeferred( self._sourcedirIsUpdatable()) yield wfd updatable = wfd.getResult() if not updatable: # blow away the old (un-updatable) directory wfd = defer.waitForDeferred( self._rmdir(self.workdir)) yield wfd wfd.getResult() # and plan to do a checkout command = ['checkout', self.repourl, '.'] else: # otherwise, do an update command = ['update'] if self.revision: command.extend(['--revision', str(self.revision)]) wfd = defer.waitForDeferred( self._dovccmd(command)) yield wfd wfd.getResult()
def _checkCompletedBuildsets(self, bsid, result): wfd = defer.waitForDeferred( self.master.db.buildsets.getSubscribedBuildsets(self.schedulerid)) yield wfd subs = wfd.getResult() for (sub_bsid, sub_sssetid, sub_complete, sub_results) in subs: # skip incomplete builds, handling the case where the 'complete' # column has not been updated yet if not sub_complete and sub_bsid != bsid: continue # build a dependent build if the status is appropriate if sub_results in (SUCCESS, WARNINGS): wfd = defer.waitForDeferred( self.addBuildsetForSourceStamp(setid=sub_sssetid, reason='downstream')) yield wfd wfd.getResult() # and regardless of status, remove the subscription wfd = defer.waitForDeferred( self.master.db.buildsets.unsubscribeFromBuildset( self.schedulerid, sub_bsid)) yield wfd wfd.getResult()
def SubmitTryJobChanges(self, changes): """ Override of SVNPoller.submit_changes: http://src.chromium.org/viewvc/chrome/trunk/tools/build/scripts/master/try_job_svn.py?view=markup We modify it so that the patch file url is added to the build properties. This allows the slave to download the patch directly rather than receiving it from the master. """ for chdict in changes: # pylint: disable=E1101 parsed = self.parent.parse_options(text_to_dict(chdict['comments'])) # 'fix' revision. # LKGR must be known before creating the change object. wfd = defer.waitForDeferred(self.parent.get_lkgr(parsed)) yield wfd wfd.getResult() wfd = defer.waitForDeferred(self.master.addChange( author=','.join(parsed['email']), revision=parsed['revision'], comments='', properties={'patch_file_url': chdict['repository'] + '/' + \ chdict['files'][0]})) yield wfd change = wfd.getResult() self.parent.addChangeInner(chdict['files'], parsed, change.number)
def runner(): for k in xrange(10): yield waitForDeferred(self.broker.shutdown()) d = Deferred() reactor.callLater(0.02, d.callback, None) yield waitForDeferred(d)
def doMaybeStartBuild(self): # first, if we're not running, then don't start builds; stopService # uses this to ensure that any ongoing doMaybeStartBuild invocations # are complete before it stops. if not self.running: return # Check for available slaves. If there are no available slaves, then # there is no sense continuing available_slavebuilders = [ sb for sb in self.slaves if sb.isAvailable() ] if not available_slavebuilders: self.updateBigStatus() return # now, get the available build requests wfd = defer.waitForDeferred( self.master.db.buildrequests.getBuildRequests( buildername=self.name, claimed=False)) yield wfd unclaimed_requests = wfd.getResult() # sort by submitted_at, so the first is the oldest unclaimed_requests.sort(key=lambda brd: brd['submitted_at']) # get the mergeRequests function for later mergeRequests_fn = self._getMergeRequestsFn() # match them up until we're out of options while available_slavebuilders and unclaimed_requests: # first, choose a slave (using nextSlave) wfd = defer.waitForDeferred( self._chooseSlave(available_slavebuilders)) yield wfd slavebuilder = wfd.getResult() if not slavebuilder: break if slavebuilder not in available_slavebuilders: log.msg(("nextSlave chose a nonexistent slave for builder " "'%s'; cannot start build") % self.name) break # then choose a request (using nextBuild) wfd = defer.waitForDeferred(self._chooseBuild(unclaimed_requests)) yield wfd breq = wfd.getResult() if not breq: break if breq not in unclaimed_requests: log.msg(("nextBuild chose a nonexistent request for builder " "'%s'; cannot start build") % self.name) break # merge the chosen request with any compatible requests in the # queue wfd = defer.waitForDeferred( self._mergeRequests(breq, unclaimed_requests, mergeRequests_fn)) yield wfd breqs = wfd.getResult() # try to claim the build requests try: wfd = defer.waitForDeferred( self.master.db.buildrequests.claimBuildRequests( [brdict['brid'] for brdict in breqs])) yield wfd wfd.getResult() except buildrequests.AlreadyClaimedError: # one or more of the build requests was already claimed; # re-fetch the now-partially-claimed build requests and keep # trying to match them self._breakBrdictRefloops(unclaimed_requests) wfd = defer.waitForDeferred( self.master.db.buildrequests.getBuildRequests( buildername=self.name, claimed=False)) yield wfd unclaimed_requests = wfd.getResult() # go around the loop again continue # claim was successful, so initiate a build for this set of # requests. Note that if the build fails from here on out (e.g., # because a slave has failed), it will be handled outside of this # loop. TODO: test that! wfd = defer.waitForDeferred( self._startBuildFor(slavebuilder, breqs)) yield wfd wfd.getResult() # and finally remove the buildrequests and slavebuilder from the # respective queues self._breakBrdictRefloops(breqs) for breq in breqs: unclaimed_requests.remove(breq) available_slavebuilders.remove(slavebuilder) self._breakBrdictRefloops(unclaimed_requests) self.updateBigStatus() return
def content(self, req, cxt): status = self.getStatus(req) builders = req.args.get("builder", status.getBuilderNames()) branches = [b for b in req.args.get("branch", []) if b] # get counts of pending builds for each builder brstatus_ds = [] brcounts = {} def keep_count(statuses, builderName): brcounts[builderName] = len(statuses) for builderName in builders: builder_status = status.getBuilder(builderName) d = builder_status.getPendingBuildRequestStatuses() d.addCallback(keep_count, builderName) brstatus_ds.append(d) wfd = defer.waitForDeferred(defer.gatherResults(brstatus_ds)) yield wfd wfd.getResult() cxt['branches'] = branches bs = cxt['builders'] = [] building = 0 online = 0 base_builders_url = path_to_root(req) + "builders/" for bn in builders: bld = { 'link': base_builders_url + urllib.quote(bn, safe=''), 'name': bn } bs.append(bld) builder = status.getBuilder(bn) builds = list( builder.generateFinishedBuilds(map_branches(branches), num_builds=1)) if builds: b = builds[0] bld['build_url'] = (bld['link'] + "/builds/%d" % b.getNumber()) label = b.getProperty("got_revision") if not label or len(str(label)) > 20: label = "#%d" % b.getNumber() bld['build_label'] = label bld['build_text'] = " ".join(b.getText()) bld['build_css_class'] = build_get_class(b) current_box = ICurrentBox(builder).getBox(status, brcounts) bld['current_box'] = current_box.td() builder_status = builder.getState()[0] if builder_status == "building": building += 1 online += 1 elif builder_status != "offline": online += 1 cxt['authz'] = self.getAuthz(req) cxt['num_building'] = building cxt['num_online'] = online buildForceContext(cxt, req, self.getBuildmaster(req)) template = req.site.buildbot_service.templates.get_template( "builders.html") yield template.render(**cxt)
def content(self, req, cxt): b = self.builder_status cxt['name'] = b.getName() req.setHeader('Cache-Control', 'no-cache') slaves = b.getSlaves() connected_slaves = [s for s in slaves if s.isConnected()] cxt['current'] = [self.builder(x, req) for x in b.getCurrentBuilds()] cxt['pending'] = [] wfd = defer.waitForDeferred(b.getPendingBuildRequestStatuses()) yield wfd statuses = wfd.getResult() for pb in statuses: changes = [] wfd = defer.waitForDeferred(pb.getSourceStamp()) yield wfd source = wfd.getResult() wfd = defer.waitForDeferred(pb.getSubmitTime()) yield wfd submitTime = wfd.getResult() wfd = defer.waitForDeferred(pb.getBsid()) yield wfd bsid = wfd.getResult() wfd = defer.waitForDeferred( pb.master.db.buildsets.getBuildsetProperties(bsid)) yield wfd properties = wfd.getResult() if source.changes: for c in source.changes: changes.append({ 'url': path_to_change(req, c), 'who': c.who, 'revision': c.revision, 'repo': c.repository }) cxt['pending'].append({ 'when': time.strftime("%b %d %H:%M:%S", time.localtime(submitTime)), 'delay': util.formatInterval(util.now() - submitTime), 'id': pb.brid, 'changes': changes, 'num_changes': len(changes), 'properties': properties, }) numbuilds = int(req.args.get('numbuilds', ['5'])[0]) recent = cxt['recent'] = [] for build in b.generateFinishedBuilds(num_builds=int(numbuilds)): recent.append(self.get_line_values(req, build, False)) sl = cxt['slaves'] = [] connected_slaves = 0 for slave in slaves: s = {} sl.append(s) s['link'] = path_to_slave(req, slave) s['name'] = slave.getName() c = s['connected'] = slave.isConnected() if c: s['admin'] = unicode(slave.getAdmin() or '', 'utf-8') connected_slaves += 1 cxt['connected_slaves'] = connected_slaves cxt['authz'] = self.getAuthz(req) cxt['builder_url'] = path_to_builder(req, b) buildForceContext(cxt, req, self.getBuildmaster(req), b.getName()) template = req.site.buildbot_service.templates.get_template( "builder.html") yield template.render(**cxt)
def TryJobRietveldSubmitJobs(self, jobs): """ Override of master.try_job_rietveld.TryJobRietveld.SubmitJobs: http://src.chromium.org/viewvc/chrome/trunk/tools/build/scripts/master/try_job_rietveld.py?view=markup We modify it to include "baseurl" as a build property. """ log.msg('TryJobRietveld.SubmitJobs: %s' % json.dumps(jobs, indent=2)) for job in jobs: try: # Gate the try job on the user that requested the job, not the one that # authored the CL. # pylint: disable=W0212 ########################## Added by rmistry ########################## if (job.get('requester') and not job['requester'].endswith('@google.com') and not job['requester'].endswith('@chromium.org') and not job['requester'] in TRYBOTS_REQUESTER_WHITELIST): # Reject the job only if the requester has an email not ending in # google.com or chromium.org raise BadJobfile('TryJobRietveld rejecting job from %s' % job['requester']) ###################################################################### ########################## Added by borenet ########################## if not (job.get('baseurl') and config_private.Master.Skia.project_name.lower() in job['baseurl']): raise BadJobfile( 'TryJobRietveld rejecting job with unknown baseurl: %s' % job.get('baseurl')) ###################################################################### if job['email'] != job['requester']: # Note the fact the try job was requested by someone else in the # 'reason'. job['reason'] = job.get('reason') or '' if job['reason']: job['reason'] += '; ' job['reason'] += "This CL was triggered by %s" % job[ 'requester'] options = { 'bot': { job['builder']: job['tests'] }, 'email': [job['email']], 'project': [self._project], 'try_job_key': job['key'], } # Transform some properties as is expected by parse_options(). for key in ( ########################## Added by borenet ########################## 'baseurl', ###################################################################### 'name', 'user', 'root', 'reason', 'clobber', 'patchset', 'issue', 'requester', 'revision'): options[key] = [job[key]] # Now cleanup the job dictionary and submit it. cleaned_job = self.parse_options(options) wfd = defer.waitForDeferred(self.get_lkgr(cleaned_job)) yield wfd wfd.getResult() wfd = defer.waitForDeferred( self.master.addChange( author=','.join(cleaned_job['email']), # TODO(maruel): Get patchset properties to get the list of files. # files=[], revision=cleaned_job['revision'], comments='')) yield wfd changeids = [wfd.getResult().number] wfd = defer.waitForDeferred(self.SubmitJob(cleaned_job, changeids)) yield wfd wfd.getResult() except BadJobfile, e: # We need to mark it as failed otherwise it'll stay in the pending # state. Simulate a buildFinished event on the build. if not job.get('key'): log.err( 'Got %s for issue %s but not key, not updating Rietveld' % (e, job.get('issue'))) continue log.err('Got %s for issue %s, updating Rietveld' % (e, job.get('issue'))) for service in self.master.services: if service.__class__.__name__ == 'TryServerHttpStatusPush': # pylint: disable=W0212,W0612 build = { 'properties': [ ('buildername', job.get('builder'), None), ('buildnumber', -1, None), ('issue', job['issue'], None), ('patchset', job['patchset'], None), ('project', self._project, None), ('revision', '', None), ('slavename', '', None), ('try_job_key', job['key'], None), ], 'reason': job.get('reason', ''), # Use EXCEPTION until SKIPPED results in a non-green try job # results on Rietveld. 'results': EXCEPTION, } ########################## Added by rmistry ######################### # Do not update Rietveld to mark the try job request as failed. # See https://code.google.com/p/chromium/issues/detail?id=224014 for # more context. # service.push('buildFinished', build=build) ##################################################################### break
def wfd(self, d): return defer.waitForDeferred(d)
def storeResource(request, source=None, source_uri=None, data=None, destination=None, destination_uri=None, deletesource=False, depth="0"): """ Function that does common PUT/COPY/MOVE behaviour. @param request: the L{txweb2.server.Request} for the current HTTP request. @param source: the L{DAVFile} for the source resource to copy from, or None if source data is to be read from the request. @param source_uri: the URI for the source resource. @param data: a C{str} to copy data from instead of the request stream. @param destination: the L{DAVFile} for the destination resource to copy into. @param destination_uri: the URI for the destination resource. @param deletesource: True if the source resource is to be deleted on successful completion, False otherwise. @param depth: a C{str} containing the COPY/MOVE Depth header value. @return: status response. """ try: assert request is not None and destination is not None and destination_uri is not None assert (source is None) or (source is not None and source_uri is not None) assert not deletesource or (deletesource and source is not None) except AssertionError: log.error("Invalid arguments to storeResource():") log.error("request=%s\n" % (request, )) log.error("source=%s\n" % (source, )) log.error("source_uri=%s\n" % (source_uri, )) log.error("data=%s\n" % (data, )) log.error("destination=%s\n" % (destination, )) log.error("destination_uri=%s\n" % (destination_uri, )) log.error("deletesource=%s\n" % (deletesource, )) log.error("depth=%s\n" % (depth, )) raise class RollbackState(object): """ This class encapsulates the state needed to rollback the entire PUT/COPY/MOVE transaction, leaving the server state the same as it was before the request was processed. The DoRollback method will actually execute the rollback operations. """ def __init__(self): self.active = True self.source_copy = None self.destination_copy = None self.destination_created = False self.source_deleted = False def Rollback(self): """ Rollback the server state. Do not allow this to raise another exception. If rollback fails then we are going to be left in an awkward state that will need to be cleaned up eventually. """ if self.active: self.active = False log.error("Rollback: rollback") try: if self.source_copy and self.source_deleted: self.source_copy.moveTo(source.fp) log.error("Rollback: source restored %s to %s" % (self.source_copy.path, source.fp.path)) self.source_copy = None self.source_deleted = False if self.destination_copy: destination.fp.remove() log.error( "Rollback: destination restored %s to %s" % (self.destination_copy.path, destination.fp.path)) self.destination_copy.moveTo(destination.fp) self.destination_copy = None elif self.destination_created: destination.fp.remove() log.error("Rollback: destination removed %s" % (destination.fp.path, )) self.destination_created = False except: log.error( "Rollback: exception caught and not handled: %s" % Failure()) def Commit(self): """ Commit the resource changes by wiping the rollback state. """ if self.active: log.error("Rollback: commit") self.active = False if self.source_copy: self.source_copy.remove() log.error("Rollback: removed source backup %s" % (self.source_copy.path, )) self.source_copy = None if self.destination_copy: self.destination_copy.remove() log.error("Rollback: removed destination backup %s" % (self.destination_copy.path, )) self.destination_copy = None self.destination_created = False self.source_deleted = False rollback = RollbackState() try: """ Handle validation operations here. """ """ Handle rollback setup here. """ # Do quota checks on destination and source before we start messing with adding other files destquota = waitForDeferred(destination.quota(request)) yield destquota destquota = destquota.getResult() if destquota is not None and destination.exists(): old_dest_size = waitForDeferred(destination.quotaSize(request)) yield old_dest_size old_dest_size = old_dest_size.getResult() else: old_dest_size = 0 if source is not None: sourcequota = waitForDeferred(source.quota(request)) yield sourcequota sourcequota = sourcequota.getResult() if sourcequota is not None and source.exists(): old_source_size = waitForDeferred(source.quotaSize(request)) yield old_source_size old_source_size = old_source_size.getResult() else: old_source_size = 0 else: sourcequota = None old_source_size = 0 # We may need to restore the original resource data if the PUT/COPY/MOVE fails, # so rename the original file in case we need to rollback. overwrite = destination.exists() if overwrite: rollback.destination_copy = FilePath(destination.fp.path) rollback.destination_copy.path += ".rollback" destination.fp.copyTo(rollback.destination_copy) else: rollback.destination_created = True if deletesource: rollback.source_copy = FilePath(source.fp.path) rollback.source_copy.path += ".rollback" source.fp.copyTo(rollback.source_copy) """ Handle actual store operations here. """ # Do put or copy based on whether source exists if source is not None: response = maybeDeferred(copy, source.fp, destination.fp, destination_uri, depth) else: datastream = request.stream if data is not None: datastream = MemoryStream(data) md5 = MD5Stream(datastream) response = maybeDeferred(put, md5, destination.fp) response = waitForDeferred(response) yield response response = response.getResult() # Update the MD5 value on the resource if source is not None: # Copy MD5 value from source to destination if source.hasDeadProperty(TwistedGETContentMD5): md5 = source.readDeadProperty(TwistedGETContentMD5) destination.writeDeadProperty(md5) else: # Finish MD5 calc and write dead property md5.close() md5 = md5.getMD5() destination.writeDeadProperty(TwistedGETContentMD5.fromString(md5)) # Update the content-type value on the resource if it is not been copied or moved if source is None: content_type = request.headers.getHeader("content-type") if content_type is not None: destination.writeDeadProperty( davxml.GETContentType.fromString( generateContentType(content_type))) response = IResponse(response) # Do quota check on destination if destquota is not None: # Get size of new/old resources new_dest_size = waitForDeferred(destination.quotaSize(request)) yield new_dest_size new_dest_size = new_dest_size.getResult() diff_size = new_dest_size - old_dest_size if diff_size >= destquota[0]: log.error("Over quota: available %d, need %d" % (destquota[0], diff_size)) raise HTTPError( ErrorResponse(responsecode.INSUFFICIENT_STORAGE_SPACE, (dav_namespace, "quota-not-exceeded"))) d = waitForDeferred(destination.quotaSizeAdjust( request, diff_size)) yield d d.getResult() if deletesource: # Delete the source resource if sourcequota is not None: delete_size = 0 - old_source_size d = waitForDeferred( source.quotaSizeAdjust(request, delete_size)) yield d d.getResult() delete(source_uri, source.fp, depth) rollback.source_deleted = True # Can now commit changes and forget the rollback details rollback.Commit() yield response return except: # Roll back changes to original server state. Note this may do nothing # if the rollback has already ocurred or changes already committed. rollback.Rollback() raise
def BlockForEverythingBeingSent(): d = self.serverPushCb() if d: x = defer.waitForDeferred(d) yield x x.getResult()
def shutdown(self): """Shutdown the slave""" if not self.slave: log.msg("no remote; slave is already shut down") return # First, try the "new" way - calling our own remote's shutdown # method. The method was only added in 0.8.3, so ignore NoSuchMethod # failures. def new_way(): d = self.slave.callRemote('shutdown') d.addCallback(lambda _: True) # successful shutdown request def check_nsm(f): f.trap(pb.NoSuchMethod) return False # fall through to the old way d.addErrback(check_nsm) def check_connlost(f): f.trap(pb.PBConnectionLost) return True # the slave is gone, so call it finished d.addErrback(check_connlost) return d wfd = defer.waitForDeferred(new_way()) yield wfd if wfd.getResult(): return # done! # Now, the old way. Look for a builder with a remote reference to the # client side slave. If we can find one, then call "shutdown" on the # remote builder, which will cause the slave buildbot process to exit. def old_way(): d = None for b in self.slavebuilders.values(): if b.remote: d = b.remote.callRemote("shutdown") break if d: log.msg("Shutting down (old) slave: %s" % self.slavename) # The remote shutdown call will not complete successfully since the # buildbot process exits almost immediately after getting the # shutdown request. # Here we look at the reason why the remote call failed, and if # it's because the connection was lost, that means the slave # shutdown as expected. def _errback(why): if why.check(pb.PBConnectionLost): log.msg("Lost connection to %s" % self.slavename) else: log.err("Unexpected error when trying to shutdown %s" % self.slavename) d.addErrback(_errback) return d log.err("Couldn't find remote builder to shut down slave") return defer.succeed(None)
def _genBuggy(self): yield waitForDeferred(getThing()) 1/0
def _startBuildFor(self, slavebuilder, buildrequests): """Start a build on the given slave. @param build: the L{base.Build} to start @param sb: the L{SlaveBuilder} which will host this build @return: (via Deferred) boolean indicating that the build was succesfully started. """ # as of the Python versions supported now, try/finally can't be used # with a generator expression. So instead, we push cleanup functions # into a list so that, at any point, we can abort this operation. cleanups = [] def run_cleanups(): while cleanups: fn = cleanups.pop() fn() # the last cleanup we want to perform is to update the big # status based on any other cleanup cleanups.append(lambda : self.updateBigStatus()) build = self.buildFactory.newBuild(buildrequests) build.setBuilder(self) log.msg("starting build %s using slave %s" % (build, slavebuilder)) # set up locks build.setLocks(self.locks) cleanups.append(lambda : slavebuilder.slave.releaseLocks()) if len(self.env) > 0: build.setSlaveEnvironment(self.env) # append the build to self.building self.building.append(build) cleanups.append(lambda : self.building.remove(build)) # update the big status accordingly self.updateBigStatus() try: wfd = defer.waitForDeferred( slavebuilder.prepare(self.builder_status, build)) yield wfd ready = wfd.getResult() except: log.err(failure.Failure(), 'while preparing slavebuilder:') ready = False # If prepare returns True then it is ready and we start a build # If it returns false then we don't start a new build. if not ready: log.msg("slave %s can't build %s after all; re-queueing the " "request" % (build, slavebuilder)) run_cleanups() yield False return # ping the slave to make sure they're still there. If they've # fallen off the map (due to a NAT timeout or something), this # will fail in a couple of minutes, depending upon the TCP # timeout. # # TODO: This can unnecessarily suspend the starting of a build, in # situations where the slave is live but is pushing lots of data to # us in a build. log.msg("starting build %s.. pinging the slave %s" % (build, slavebuilder)) try: wfd = defer.waitForDeferred( slavebuilder.ping()) yield wfd ping_success = wfd.getResult() except: log.err(failure.Failure(), 'while pinging slave before build:') ping_success = False if not ping_success: log.msg("slave ping failed; re-queueing the request") run_cleanups() yield False return # The buildslave is ready to go. slavebuilder.buildStarted() sets its # state to BUILDING (so we won't try to use it for any other builds). # This gets set back to IDLE by the Build itself when it finishes. slavebuilder.buildStarted() cleanups.append(lambda : slavebuilder.buildFinished()) # tell the remote that it's starting a build, too try: wfd = defer.waitForDeferred( slavebuilder.remote.callRemote("startBuild")) yield wfd wfd.getResult() except: log.err(failure.Failure(), 'while calling remote startBuild:') run_cleanups() yield False return # create the BuildStatus object that goes with the Build bs = self.builder_status.newBuild() # record the build in the db - one row per buildrequest try: bids = [] for req in build.requests: wfd = defer.waitForDeferred( self.master.db.builds.addBuild(req.id, bs.number)) yield wfd bids.append(wfd.getResult()) except: log.err(failure.Failure(), 'while adding rows to build table:') run_cleanups() yield False return # let status know self.master.status.build_started(req.id, self.name, bs) # start the build. This will first set up the steps, then tell the # BuildStatus that it has started, which will announce it to the world # (through our BuilderStatus object, which is its parent). Finally it # will start the actual build process. This is done with a fresh # Deferred since _startBuildFor should not wait until the build is # finished. d = build.startBuild(bs, self.expectations, slavebuilder) d.addCallback(self.buildFinished, slavebuilder, bids) # this shouldn't happen. if it does, the slave will be wedged d.addErrback(log.err) # make sure the builder's status is represented correctly self.updateBigStatus() yield True
error = "Request XML body is required." log.error("Error: {err}", err=error) raise HTTPError(StatusResponse(responsecode.BAD_REQUEST, error)) # # Parse request # acl = doc.root_element if not isinstance(acl, davxml.ACL): error = ("Request XML body must be an acl element." % (davxml.PropertyUpdate.sname(),)) log.error("Error: {err}", err=error) raise HTTPError(StatusResponse(responsecode.BAD_REQUEST, error)) # # Do ACL merger # result = waitForDeferred(self.mergeAccessControlList(acl, request)) yield result result = result.getResult() # # Return response # if result is None: yield responsecode.OK else: yield ErrorResponse(responsecode.FORBIDDEN, result) http_ACL = deferredGenerator(http_ACL)
def test_gotChange_treeStableTimer_sequence(self): sched = self.makeScheduler(self.Subclass, treeStableTimer=9, branch='master') self.master.db.insertTestData([ fakedb.Change(changeid=1, branch='master', when_timestamp=1110), fakedb.ChangeFile(changeid=1, file='readme.txt'), fakedb.Change(changeid=2, branch='master', when_timestamp=2220), fakedb.ChangeFile(changeid=2, file='readme.txt'), fakedb.Change(changeid=3, branch='master', when_timestamp=3330), fakedb.ChangeFile(changeid=3, file='readme.txt'), fakedb.Change(changeid=4, branch='master', when_timestamp=4440), fakedb.ChangeFile(changeid=4, file='readme.txt'), ]) sched.startService() self.clock.advance(2220) # this important change arrives at 2220, so the stable timer will last # until 2229 wfd = defer.waitForDeferred( sched.gotChange( self.makeFakeChange(branch='master', number=1, when=2220), True)) yield wfd wfd.getResult() self.assertEqual(self.events, []) self.db.schedulers.assertClassifications(self.SCHEDULERID, {1: True}) # but another (unimportant) change arrives before then self.clock.advance(6) # to 2226 self.assertEqual(self.events, []) wfd = defer.waitForDeferred( sched.gotChange( self.makeFakeChange(branch='master', number=2, when=2226), False)) yield wfd wfd.getResult() self.assertEqual(self.events, []) self.db.schedulers.assertClassifications(self.SCHEDULERID, { 1: True, 2: False }) self.clock.advance(3) # to 2229 self.assertEqual(self.events, []) self.clock.advance(3) # to 2232 self.assertEqual(self.events, []) # another important change arrives at 2232 wfd = defer.waitForDeferred( sched.gotChange( self.makeFakeChange(branch='master', number=3, when=2232), True)) yield wfd wfd.getResult() self.assertEqual(self.events, []) self.db.schedulers.assertClassifications(self.SCHEDULERID, { 1: True, 2: False, 3: True }) self.clock.advance(3) # to 2235 self.assertEqual(self.events, []) # finally, time to start the build! self.clock.advance(6) # to 2241 self.assertEqual(self.events, ['B[1,2,3]@2241']) self.db.schedulers.assertClassifications(self.SCHEDULERID, {}) wfd = defer.waitForDeferred(sched.stopService()) yield wfd wfd.getResult()
def maybeStartBuild(self): # This method is called by the botmaster whenever this builder should # check for and potentially start new builds. Do not call this method # directly - use master.botmaster.maybeStartBuildsForBuilder, or one # of the other similar methods if more appropriate # first, if we're not running, then don't start builds; stopService # uses this to ensure that any ongoing maybeStartBuild invocations # are complete before it stops. if not self.running: return # Check for available slaves. If there are no available slaves, then # there is no sense continuing available_slavebuilders = [ sb for sb in self.slaves if sb.isAvailable() ] if not available_slavebuilders: self.updateBigStatus() return # now, get the available build requests wfd = defer.waitForDeferred( self.master.db.buildrequests.getBuildRequests( buildername=self.name, claimed=False)) yield wfd unclaimed_requests = wfd.getResult() if not unclaimed_requests: self.updateBigStatus() return # sort by submitted_at, so the first is the oldest unclaimed_requests.sort(key=lambda brd : brd['submitted_at']) # get the mergeRequests function for later mergeRequests_fn = self._getMergeRequestsFn() # match them up until we're out of options while available_slavebuilders and unclaimed_requests: # first, choose a slave (using nextSlave) wfd = defer.waitForDeferred( self._chooseSlave(available_slavebuilders)) yield wfd slavebuilder = wfd.getResult() if not slavebuilder: break if slavebuilder not in available_slavebuilders: log.msg(("nextSlave chose a nonexistent slave for builder " "'%s'; cannot start build") % self.name) break # then choose a request (using nextBuild) wfd = defer.waitForDeferred( self._chooseBuild(unclaimed_requests)) yield wfd brdict = wfd.getResult() if not brdict: break if brdict not in unclaimed_requests: log.msg(("nextBuild chose a nonexistent request for builder " "'%s'; cannot start build") % self.name) break # merge the chosen request with any compatible requests in the # queue wfd = defer.waitForDeferred( self._mergeRequests(brdict, unclaimed_requests, mergeRequests_fn)) yield wfd brdicts = wfd.getResult() # try to claim the build requests brids = [ brdict['brid'] for brdict in brdicts ] try: wfd = defer.waitForDeferred( self.master.db.buildrequests.claimBuildRequests(brids)) yield wfd wfd.getResult() except buildrequests.AlreadyClaimedError: # one or more of the build requests was already claimed; # re-fetch the now-partially-claimed build requests and keep # trying to match them self._breakBrdictRefloops(unclaimed_requests) wfd = defer.waitForDeferred( self.master.db.buildrequests.getBuildRequests( buildername=self.name, claimed=False)) yield wfd unclaimed_requests = wfd.getResult() # go around the loop again continue # claim was successful, so initiate a build for this set of # requests. Note that if the build fails from here on out (e.g., # because a slave has failed), it will be handled outside of this # loop. TODO: test that! # _startBuildFor expects BuildRequest objects, so cook some up wfd = defer.waitForDeferred( defer.gatherResults([ self._brdictToBuildRequest(brdict) for brdict in brdicts ])) yield wfd breqs = wfd.getResult() wfd = defer.waitForDeferred( self._startBuildFor(slavebuilder, breqs)) yield wfd build_started = wfd.getResult() if not build_started: # build was not started, so unclaim the build requests wfd = defer.waitForDeferred( self.master.db.buildrequests.unclaimBuildRequests(brids)) yield wfd wfd.getResult() # and try starting builds again. If we still have a working slave, # then this may re-claim the same buildrequests self.botmaster.maybeStartBuildsForBuilder(self.name) # finally, remove the buildrequests and slavebuilder from the # respective queues self._breakBrdictRefloops(brdicts) for brdict in brdicts: unclaimed_requests.remove(brdict) available_slavebuilders.remove(slavebuilder) self._breakBrdictRefloops(unclaimed_requests) self.updateBigStatus() return
def _poll(self): args = [] if self.p4port: args.extend(['-p', self.p4port]) if self.p4user: args.extend(['-u', self.p4user]) if self.p4passwd: args.extend(['-P', self.p4passwd]) args.extend(['changes']) if self.last_change is not None: args.extend(['%s...@%d,now' % (self.p4base, self.last_change + 1)]) else: args.extend(['-m', '1', '%s...' % (self.p4base, )]) wfd = defer.waitForDeferred(self._get_process_output(args)) yield wfd result = wfd.getResult() last_change = self.last_change changelists = [] for line in result.split('\n'): line = line.strip() if not line: continue m = self.changes_line_re.match(line) if not m: raise P4PollerError("Unexpected 'p4 changes' output: %r" % result) num = int(m.group('num')) if last_change is None: # first time through, the poller just gets a "baseline" for where to # start on the next poll log.msg('P4Poller: starting at change %d' % num) self.last_change = num return changelists.append(num) changelists.reverse() # oldest first # Retrieve each sequentially. for num in changelists: args = [] if self.p4port: args.extend(['-p', self.p4port]) if self.p4user: args.extend(['-u', self.p4user]) if self.p4passwd: args.extend(['-P', self.p4passwd]) args.extend(['describe', '-s', str(num)]) wfd = defer.waitForDeferred(self._get_process_output(args)) yield wfd result = wfd.getResult() # decode the result from its designated encoding result = result.decode(self.encoding) lines = result.split('\n') # SF#1555985: Wade Brainerd reports a stray ^M at the end of the date # field. The rstrip() is intended to remove that. lines[0] = lines[0].rstrip() m = self.describe_header_re.match(lines[0]) if not m: raise P4PollerError("Unexpected 'p4 describe -s' result: %r" % result) who = m.group('who') when = time.mktime(time.strptime(m.group('when'), self.datefmt)) comments = '' while not lines[0].startswith('Affected files'): comments += lines.pop(0) + '\n' lines.pop(0) # affected files branch_files = {} # dict for branch mapped to file(s) while lines: line = lines.pop(0).strip() if not line: continue m = self.file_re.match(line) if not m: raise P4PollerError("Invalid file line: %r" % line) path = m.group('path') if path.startswith(self.p4base): branch, file = self.split_file(path[len(self.p4base):]) if (branch == None and file == None): continue if branch_files.has_key(branch): branch_files[branch].append(file) else: branch_files[branch] = [file] for branch in branch_files: d = self.master.addChange( author=who, files=branch_files[branch], comments=comments, revision=str(num), when_timestamp=util.epoch2datetime(when), branch=branch, project=self.project) wfd = defer.waitForDeferred(d) yield wfd wfd.getResult() self.last_change = num
def submit_changes(self, changes): for chdict in changes: wfd = defer.waitForDeferred(self.master.addChange(**chdict)) yield wfd wfd.getResult()
def stopInstance(self, label): """Stops an application instance by label @param label: (string) @fires Event('instance-stopped') return defer.Deferred() """ result = {} template = '[%(application)s,%(label)s] %(description)s' context = {'code': 254} thisInst = None try: thisInst = self.model.getInstance(label) thisInst.shouldBeRunning = False if not thisInst.running: context.update(self.model.statusInstance(label)) raise DroneCommandFailed(context) pid = thisInst.process.pid self.log("Trying to shutdown %d gracefully" % (pid, )) def failed(result): """attempting to be consistant""" self.log("Failed to shutdown process gracefully") return result def success(result): """attempting to be consistant""" self.log("process %d gracefully shutdown" % (pid, )) return result d = self._start_stop_common(label, 'stopInstance') d.addCallback(success) d.addErrback(failed) d.addErrback(self._killInstance, thisInst) wfd = defer.waitForDeferred(d) yield wfd #refresh the instance as it can change thisInst = self.model.getInstance(label) result = wfd.getResult() if isinstance(result, dict): context.update(result) elif isinstance(result, DroneCommandFailed): context.update(result.resultContext) if not thisInst.running: context['code'] = 0 Event('instance-stopped').fire(instance=thisInst) raise AssertionError('ignore me') raise DroneCommandFailed(context) except AssertionError: #update the instance model wfd = defer.waitForDeferred(self.statusInstance(label)) yield wfd result = wfd.getResult() result['code'] = context['code'] except: failure = Failure() if failure.check(DroneCommandFailed): context = failure.value.resultContext template = '%(description)s' else: temp = "%s: %s" % (getException(failure), failure.getErrorMessage()) context = {'error': failure, 'code': 253, 'description': temp} result = self.resultContext(template, thisInst, **context) try: thisInst = self.model.getInstance(label) thisInst.shouldBeRunning = False except: pass yield result
def report_DAV__expand_property(self, request, expand_property): """ Generate an expand-property REPORT. (RFC 3253, section 3.8) """ # FIXME: Handle depth header if not isinstance(expand_property, davxml.ExpandProperty): raise ValueError( "%s expected as root element, not %s." % (davxml.ExpandProperty.sname(), expand_property.sname())) # # Expand DAV:allprop # properties = {} for property in expand_property.children: namespace = property.getAttribute("namespace") name = property.getAttribute("name") if not namespace: namespace = dav_namespace if (namespace, name) == (dav_namespace, "allprop"): all_properties = waitForDeferred(self.listAllProp(request)) yield all_properties all_properties = all_properties.getResult() for all_property in all_properties: properties[all_property.qname()] = property else: properties[(namespace, name)] = property # # Look up the requested properties # properties_by_status = { responsecode.OK: [], responsecode.NOT_FOUND: [], } for property in properties: my_properties = waitForDeferred(self.listProperties(request)) yield my_properties my_properties = my_properties.getResult() if property in my_properties: try: value = waitForDeferred(self.readProperty(property, request)) yield value value = value.getResult() if isinstance(value, davxml.HRef): raise NotImplementedError() else: raise NotImplementedError() except: f = Failure() log.err("Error reading property %r for resource %s: %s" % (property, self, f.value)) status = statusForFailure( f, "getting property: %s" % (property, )) if status not in properties_by_status: properties_by_status[status] = [] raise NotImplementedError() #properties_by_status[status].append( # ____propertyName(property) #) else: log.err("Can't find property %r for resource %s" % (property, self)) properties_by_status[responsecode.NOT_FOUND].append(property) raise NotImplementedError()
def _genError(): yield waitForDeferred(getThing()) 1 / 0
def writeMetric(metric_path, value, timestamp, host, port, username, password, vhost, exchange, spec=None, channel_number=1, ssl=False): if not spec: spec = txamqp.spec.load( os.path.normpath( os.path.join(os.path.dirname(__file__), 'amqp0-8.xml'))) delegate = TwistedDelegate() connector = ClientCreator(reactor, AMQClient, delegate=delegate, vhost=vhost, spec=spec) if ssl: from twisted.internet.ssl import ClientContextFactory wfd = waitForDeferred( connector.connectSSL(host, port, ClientContextFactory())) yield wfd conn = wfd.getResult() else: wfd = waitForDeferred(connector.connectTCP(host, port)) yield wfd conn = wfd.getResult() wfd = waitForDeferred(conn.authenticate(username, password)) yield wfd wfd = waitForDeferred(conn.channel(channel_number)) yield wfd channel = wfd.getResult() wfd = waitForDeferred(channel.channel_open()) yield wfd wfd = waitForDeferred( channel.exchange_declare(exchange=exchange, type="topic", durable=True, auto_delete=False)) yield wfd message = Content("%f %d" % (value, timestamp)) message["delivery mode"] = 2 channel.basic_publish(exchange=exchange, content=message, routing_key=metric_path) wfd = waitForDeferred(channel.channel_close()) yield wfd
uidList = [] # Consumer for listUID - adds to the working set and processes # a batch if appropriate. def consumeUIDLine(ent): uidWorkingSet.append(ent) if len(uidWorkingSet) >= N: processBatch() def processBatch(): L = self.shouldRetrieve(uidWorkingSet) L.sort() uidList.extend(L) del uidWorkingSet[:] d = defer.waitForDeferred(self.listUID(consumeUIDLine)) self.setStatus(u"Retrieving message list...") yield d try: d.getResult() except (error.ConnectionDone, error.ConnectionLost): self.setStatus(u"Connection lost", False) return except: f = failure.Failure() log.err(f, "Failure retrieving UIDL") self.setStatus(unicode(f.getErrorMessage()), False) self.transport.loseConnection() return # Clean up any stragglers.
def startInstance(self, label): """Starts an application instance by label @param label: (string) @fires Event('instance-started') return defer.Deferred() """ template = '[%(application)s,%(label)s] %(description)s' context = {'description': 'Failed to Start', 'code': 254} result = {} thisInst = None try: if self.model.getInstance(label).running: context.update(self.model.statusInstance(label)) raise DroneCommandFailed(context) d = self._start_stop_common(label, 'startInstance') wfd = defer.waitForDeferred(d) yield wfd result = wfd.getResult() d = self.statusInstance(label) wfd = defer.waitForDeferred(d) yield wfd result.update(wfd.getResult()) #refresh the instance as it can change thisInst = self.model.getInstance(label) if isinstance(result, dict): context.update(result) elif isinstance(result, DroneCommandFailed): context.update(result.resultContext) if thisInst.running: Event('instance-started').fire(instance=thisInst) context['code'] = 0 raise AssertionError('ignore') raise DroneCommandFailed(context) except AssertionError: #update the instance model wfd = defer.waitForDeferred(self.statusInstance(label)) yield wfd result = wfd.getResult() except: thisInst = self.model.getInstance(label) failure = Failure() if failure.check(DroneCommandFailed): template = '%(description)s' context = failure.value.resultContext else: #log the error, allowing for debugging self.debugReport() #be nice and return something to the end user temp = "%s: %s" % (getException(failure), failure.getErrorMessage()) context = {'error': failure, 'code': 253, 'description': temp} result = self.resultContext(template, thisInst, **context) try: thisInst = self.model.getInstance(label) thisInst.shouldBeRunning = True except: pass yield result
returnMinimal = any([key == "return" and value == "minimal" for key, value, _ignore_args in prefer]) noRoot = any([key == "depth-noroot" and value is None for key, value, _ignore_args in prefer]) if not returnMinimal: returnMinimal = request.headers.getHeader("brief", False) xml_responses = [] # FIXME: take advantage of the new generative properties of findChildren my_url = normalizeURL(request_uri) if self.isCollection() and not my_url.endswith("/"): my_url += "/" # Do some optimisation of access control calculation by determining any inherited ACLs outside of # the child resource loop and supply those to the checkPrivileges on each child. filtered_aces = waitForDeferred(self.inheritedACEsforChildren(request)) yield filtered_aces filtered_aces = filtered_aces.getResult() if depth in ("1", "infinity") and noRoot: resources = [] else: resources = [(self, my_url)] d = self.findChildren(depth, request, lambda x, y: resources.append((x, y)), (davxml.Read(),), inherited_aces=filtered_aces) x = waitForDeferred(d) yield x x.getResult() for resource, uri in resources:
def _process_changes(self, unused_output): # get the change list revListArgs = [ 'log', '%s..origin/%s' % (self.branch, self.branch), r'--format=%H' ] self.changeCount = 0 d = utils.getProcessOutput(self.gitbin, revListArgs, path=self.workdir, env=dict(PATH=os.environ['PATH']), errortoo=False) wfd = defer.waitForDeferred(d) yield wfd results = wfd.getResult() # process oldest change first revList = results.split() if not revList: return revList.reverse() self.changeCount = len(revList) log.msg('gitpoller: processing %d changes: %s in "%s"' % (self.changeCount, revList, self.workdir)) for rev in revList: dl = defer.DeferredList([ self._get_commit_timestamp(rev), self._get_commit_name(rev), self._get_commit_files(rev), self._get_commit_comments(rev), ], consumeErrors=True) wfd = defer.waitForDeferred(dl) yield wfd results = wfd.getResult() # check for failures failures = [r[1] for r in results if not r[0]] if failures: # just fail on the first error; they're probably all related! raise failures[0] revlink = '' if self.revlinktmpl and rev: revlink = self.revlinktmpl % urllib.quote_plus(rev) timestamp, name, files, comments = [r[1] for r in results] d = self.master.addChange(author=name, revision=rev, files=files, comments=comments, when_timestamp=epoch2datetime(timestamp), branch=self.branch, category=self.category, project=self.project, repository=self.repourl, revlink=revlink) wfd = defer.waitForDeferred(d) yield wfd results = wfd.getResult()
def addBuildsetForChangesMultiRepo(self, reason='', external_idstring=None, changeids=[], builderNames=None, properties=None): assert changeids is not [] chDicts = {} def getChange(changeid=None): d = self.master.db.changes.getChange(changeid) def chdict2change(chdict): if not chdict: return None return changes.Change.fromChdict(self.master, chdict) d.addCallback(chdict2change) def groupChange(change): if change.repository not in chDicts: chDicts[change.repository] = [] chDicts[change.repository].append(change) def get_changeids_from_repo(repository): changeids = [] for change in chDicts[repository]: changeids.append(change.number) return changeids def create_sourcestamp(changeids, change=None, setid=None): def add_sourcestamp(setid, changeids=None): return self.master.db.sourcestamps.addSourceStamp( branch=change.branch, revision=change.revision, repository=change.repository, project=change.project, changeids=changeids, setid=setid) d.addCallback(add_sourcestamp, setid=setid, changeids=changeids) return d def create_sourcestamp_without_changes(setid, repository): return self.master.db.sourcestamps.addSourceStamp( branch=self.default_branch, revision=None, repository=repository, project=self.default_project, changeids=changeids, sourcestampsetid=setid) d = defer.Deferred() if self.repositories is None: # attributes for this sourcestamp will be based on the most recent # change, so fetch the change with the highest id (= old single # sourcestamp functionality) d.addCallBack(getChange, changeid=max(changeids)) d.addCallBack(groupChange) else: for changeid in changeids: d.addCallBack(getChange, changeid=changeid) d.addCallBack(groupChange) # Define setid for this set of changed repositories wfd = defer.waitForDeferred( self.master.db.sourcestampsets.addSourceStampSet) yield wfd setid = wfd.getResult() #process all unchanged repositories if self.repositories is not None: for repo in self.repositories: if repo not in chDicts: # repository was not changed # call create_sourcestamp d.addCallback(create_sourcestamp_without_changes, setid, repo) # process all changed for repo in chDicts: d.addCallback(get_changeids_from_repo, repository=repo) d.addCallback(create_sourcestamp, setid=setid, change=chDicts[repo][-1]) # add one buildset, this buildset is connected to the sourcestamps by the setid d.addCallback(self.addBuildsetForSourceStamp, setid=setid, reason=reason, external_idstring=external_idstring, builderNames=builderNames, properties=properties) yield d
if self.exists(): log.error("Attempt to create collection where file exists: %s" % (self,)) raise HTTPError(responsecode.NOT_ALLOWED) if not parent.isCollection(): log.error("Attempt to create collection with non-collection parent: %s" % (self,)) raise HTTPError(StatusResponse( responsecode.CONFLICT, "Parent resource is not a collection." )) # # Read request body # x = waitForDeferred(noDataFromStream(request.stream)) yield x try: x.getResult() except ValueError, e: log.error("Error while handling MKCOL body: %s" % (e,)) raise HTTPError(responsecode.UNSUPPORTED_MEDIA_TYPE) response = waitForDeferred(mkcollection(self.fp)) yield response yield response.getResult() http_MKCOL = deferredGenerator(http_MKCOL)
def perspective_commandline(self, op, bb_username, bb_password, ids, info): """ This performs the requested operations from the `buildbot user` call by calling the proper buildbot.db.users methods based on the operation. It yields a deferred instance with the results from the database methods. @param op: operation to perform (add, remove, update, get) @type op: string @param bb_username: username portion of auth credentials @type bb_username: string @param bb_password: hashed password portion of auth credentials @type bb_password: hashed string @param ids: user identifiers used to find existing users @type ids: list of strings or None @param info: type/value pairs for each user that will be added or updated in the database @type info: list of dictionaries or None @returns: results from db.users methods via deferred """ log.msg("perspective_commandline called") results = [] if ids: for user in ids: # get identifier, guaranteed to be in user from checks # done in C{scripts.runner} d = self.master.db.users.identifierToUid(identifier=user) wfd = defer.waitForDeferred(d) yield wfd uid = wfd.getResult() result = None if op == 'remove': if uid: d = self.master.db.users.removeUser(uid) wfd = defer.waitForDeferred(d) yield wfd wfd.getResult() result = user else: log.msg("Unable to find uid for identifier %s" % user) elif op == 'get': if uid: d = self.master.db.users.getUser(uid) wfd = defer.waitForDeferred(d) yield wfd result = wfd.getResult() else: log.msg("Unable to find uid for identifier %s" % user) results.append(result) else: for user in info: # get identifier, guaranteed to be in user from checks # done in C{scripts.runner} ident = user.pop('identifier') d = self.master.db.users.identifierToUid(identifier=ident) wfd = defer.waitForDeferred(d) yield wfd uid = wfd.getResult() # if only an identifier was in user, we're updating only # the bb_username and bb_password. if not user: if uid: d = self.master.db.users.updateUser( uid=uid, identifier=ident, bb_username=bb_username, bb_password=bb_password) wfd = defer.waitForDeferred(d) yield wfd results.append(ident) result = wfd.getResult() else: log.msg("Unable to find uid for identifier %s" % user) else: # when adding, we update the user after the first attr once_through = False for attr in user: if op == 'update' or once_through: if uid: d = self.master.db.users.updateUser( uid=uid, identifier=ident, bb_username=bb_username, bb_password=bb_password, attr_type=attr, attr_data=user[attr]) else: log.msg( "Unable to find uid for identifier %s" % user) elif op == 'add': d = self.master.db.users.findUserByAttr( identifier=ident, attr_type=attr, attr_data=user[attr]) once_through = True wfd = defer.waitForDeferred(d) yield wfd results.append(ident) result = wfd.getResult() # result is None from updateUser calls if result: results.append(result) uid = result results = self.formatResults(op, results) yield results
def addBuildsetForSourceStamp(self, ssid=None, setid=None, reason='', external_idstring=None, properties=None, builderNames=None): """ Add a buildset for the given, already-existing sourcestamp. This method will add any properties provided to the scheduler constructor to the buildset, and will call the master's L{BuildMaster.addBuildset} method with the appropriate parameters, and return the same result. @param reason: reason for this buildset @type reason: unicode string @param external_idstring: external identifier for this buildset, or None @param properties: a properties object containing initial properties for the buildset @type properties: L{buildbot.process.properties.Properties} @param builderNames: builders to name in the buildset (defaults to C{self.builderNames}) @param setid: idenitification of a set of sourcestamps @returns: (buildset ID, buildrequest IDs) via Deferred """ assert (ssid is None and setid is not None) \ or (ssid is not None and setid is None), "pass a single sourcestamp OR set not both" # combine properties if properties: properties.updateFromProperties(self.properties) else: properties = self.properties # apply the default builderNames if not builderNames: builderNames = self.builderNames # translate properties object into a dict as required by the # addBuildset method properties_dict = properties.asDict() if setid == None: if ssid != None: wfd = defer.waitForDeferred( self.master.db.sourcestamps.getSourceStamp(ssid)) yield wfd ssdict = wfd.getResult() setid = ssdict['sourcestampsetid'] else: # no sourcestamp and no sets yield None wfd = defer.waitForDeferred( self.master.addBuildset(sourcestampsetid=setid, reason=reason, properties=properties_dict, builderNames=builderNames, external_idstring=external_idstring)) yield wfd yield wfd.getResult()
def connectionMade(self): AMQClient.connectionMade(self) log.listener("New AMQP connection made") self.setup() wfd = waitForDeferred(self.receive_loop()) yield wfd
def addBuildsetForChanges(self, reason='', external_idstring=None, changeids=[], builderNames=None, properties=None): """ Add a buildset for the combination of the given changesets, creating a sourcestamp based on those changes. The sourcestamp for the buildset will reference all of the indicated changes. This method will add any properties provided to the scheduler constructor to the buildset, and will call the master's addBuildset method with the appropriate parameters. @param reason: reason for this buildset @type reason: unicode string @param external_idstring: external identifier for this buildset, or None @param changeids: nonempty list of changes to include in this buildset @param builderNames: builders to name in the buildset (defaults to C{self.builderNames}) @param properties: a properties object containing initial properties for the buildset @type properties: L{buildbot.process.properties.Properties} @returns: (buildset ID, buildrequest IDs) via Deferred """ assert changeids is not [] # attributes for this sourcestamp will be based on the most recent # change, so fetch the change with the highest id wfd = defer.waitForDeferred( self.master.db.changes.getChange(max(changeids))) yield wfd chdict = wfd.getResult() change = None if chdict: wfd = defer.waitForDeferred( changes.Change.fromChdict(self.master, chdict)) yield wfd change = wfd.getResult() # Define setid for this set of changed repositories wfd = defer.waitForDeferred( self.master.db.sourcestampsets.addSourceStampSet()) yield wfd setid = wfd.getResult() wfd = defer.waitForDeferred( self.master.db.sourcestamps.addSourceStamp( branch=change.branch, revision=change.revision, repository=change.repository, project=change.project, changeids=changeids, sourcestampsetid=setid)) yield wfd wfd.getResult() wfd = defer.waitForDeferred( self.addBuildsetForSourceStamp(setid=setid, reason=reason, external_idstring=external_idstring, builderNames=builderNames, properties=properties)) yield wfd yield wfd.getResult()