def _callFinished(self, res, delivery):
reqID = delivery.reqID
if reqID == 0:
return
methodSchema = delivery.methodSchema
assert self.activeLocalCalls[reqID]
methodName = None
if methodSchema:
methodName = methodSchema.name
try:
methodSchema.checkResults(res, False) # may raise Violation
except Violation as v:
v.prependLocation("in return value of %s.%s" %
(delivery.obj, methodSchema.name))
raise
answer = call.AnswerSlicer(reqID, res, methodName)
# once the answer has started transmitting, any exceptions must be
# logged and dropped, and not turned into an Error to be sent.
try:
self.send(answer)
# TODO: .send should return a Deferred that fires when the last
# byte has been queued, and we should delete the local note then
except:
f = failure.Failure()
log.msg("Broker._callfinished unable to send",
facility="foolscap",
level=log.UNUSUAL,
failure=f)
del self.activeLocalCalls[reqID]
class Broker(banana.Banana, referenceable.Referenceable):
"""I manage a connection to a remote Broker.
@ivar tub: the L{Tub} which contains us
@ivar yourReferenceByCLID: maps your CLID to a RemoteReferenceData
#@ivar yourReferenceByName: maps a per-Tub name to a RemoteReferenceData
@ivar yourReferenceByURL: maps a global URL to a RemoteReferenceData
"""
implements(RIBroker, IBroker)
slicerClass = PBRootSlicer
unslicerClass = PBRootUnslicer
unsafeTracebacks = True
requireSchema = False
disconnected = False
factory = None
tub = None
remote_broker = None
startingTLS = False
startedTLS = False
use_remote_broker = True
def __init__(self,
remote_tubref,
params={},
keepaliveTimeout=None,
disconnectTimeout=None,
connectionInfo=None):
banana.Banana.__init__(self, params)
self._expose_remote_exception_types = True
self.remote_tubref = remote_tubref
self.keepaliveTimeout = keepaliveTimeout
self.disconnectTimeout = disconnectTimeout
self._banana_decision_version = params.get("banana-decision-version")
vocab_table_index = params.get('initial-vocab-table-index')
if vocab_table_index:
table = vocab.INITIAL_VOCAB_TABLES[vocab_table_index]
self.populateVocabTable(table)
self.initBroker()
self.current_slave_IR = params.get('current-slave-IR')
self.current_seqnum = params.get('current-seqnum')
self.creation_timestamp = time.time()
self._connectionInfo = connectionInfo
def initBroker(self):
# tracking Referenceables
# sending side uses these
self.nextCLID = count(1).next # 0 is for the broker
self.myReferenceByPUID = {} # maps ref.processUniqueID to a tracker
self.myReferenceByCLID = {} # maps CLID to a tracker
# receiving side uses these
self.yourReferenceByCLID = {}
self.yourReferenceByURL = {}
# tracking Gifts
self.nextGiftID = count(1).next
self.myGifts = {} # maps (broker,clid) to (rref, giftID, count)
self.myGiftsByGiftID = {} # maps giftID to (broker,clid)
# remote calls
# sending side uses these
self.nextReqID = count(1).next # 0 means "we don't want a response"
self.waitingForAnswers = {} # we wait for the other side to answer
self.disconnectWatchers = []
# receiving side uses these
self.inboundDeliveryQueue = []
self._waiting_for_call_to_be_ready = False
self.activeLocalCalls = {} # the other side wants an answer from us
def setTub(self, tub):
assert ipb.ITub.providedBy(tub)
self.tub = tub
self.unsafeTracebacks = tub.unsafeTracebacks
self._expose_remote_exception_types = tub._expose_remote_exception_types
if tub.debugBanana:
self.debugSend = True
self.debugReceive = True
def connectionMade(self):
banana.Banana.connectionMade(self)
self.rootSlicer.broker = self
self.rootUnslicer.broker = self
if self.use_remote_broker:
self._create_remote_broker()
def _create_remote_broker(self):
# create the remote_broker object. We don't use the usual
# reference-counting mechanism here, because this is a synthetic
# object that lives forever.
tracker = referenceable.RemoteReferenceTracker(self, 0, None,
"RIBroker")
self.remote_broker = referenceable.RemoteReference(tracker)
# connectionTimedOut is called in response to the Banana layer detecting
# the lack of connection activity
def connectionTimedOut(self):
err = error.ConnectionLost("banana timeout: connection dropped")
why = failure.Failure(err)
self.shutdown(why)
def shutdown(self, why, fireDisconnectWatchers=True):
"""Stop using this connection. If fireDisconnectWatchers is False,
all disconnect watchers are removed before shutdown, so they will not
be called (this is appropriate when the Broker is shutting down
because the whole Tub is being shut down). We terminate the
connection quickly, rather than waiting for the transmit queue to
drain.
"""
assert isinstance(why, failure.Failure)
if not fireDisconnectWatchers:
self.disconnectWatchers = []
self.finish(why)
# loseConnection eventually provokes connectionLost()
self.transport.loseConnection()
def connectionLost(self, why):
tubid = "?"
if self.remote_tubref:
tubid = self.remote_tubref.getShortTubID()
log.msg("connection to %s lost" % tubid,
facility="foolscap.connection")
banana.Banana.connectionLost(self, why)
self.finish(why)
def finish(self, why):
if self.disconnected:
return
assert isinstance(why, failure.Failure), why
self.disconnected = True
self.remote_broker = None
self.abandonAllRequests(why)
# TODO: why reset all the tables to something useable? There may be
# outstanding RemoteReferences that point to us, but I don't see why
# that requires all these empty dictionaries.
self.myReferenceByPUID = {}
self.myReferenceByCLID = {}
self.yourReferenceByCLID = {}
self.yourReferenceByURL = {}
self.myGifts = {}
self.myGiftsByGiftID = {}
for (cb, args, kwargs) in self.disconnectWatchers:
eventually(cb, *args, **kwargs)
self.disconnectWatchers = []
if self.tub:
# TODO: remove the conditional. It is only here to accomodate
# some tests: test_pb.TestCall.testDisconnect[123]
self.tub.brokerDetached(self, why)
def notifyOnDisconnect(self, callback, *args, **kwargs):
marker = (callback, args, kwargs)
if self.disconnected:
eventually(callback, *args, **kwargs)
else:
self.disconnectWatchers.append(marker)
return marker
def dontNotifyOnDisconnect(self, marker):
if self.disconnected:
return
# be tolerant of attempts to unregister a callback that has already
# fired. I think it is hard to write safe code without this
# tolerance.
# TODO: on the other hand, I'm not sure this is the best policy,
# since you lose the feedback that tells you about
# unregistering-the-wrong-thing bugs. We need to look at the way that
# register/unregister gets used and see if there is a way to retain
# the typechecking that results from insisting that you can only
# remove something that was stil in the list.
if marker in self.disconnectWatchers:
self.disconnectWatchers.remove(marker)
def getConnectionInfo(self):
return self._connectionInfo
# methods to send my Referenceables to the other side
def getTrackerForMyReference(self, puid, obj):
tracker = self.myReferenceByPUID.get(puid)
if not tracker:
# need to add one
clid = self.nextCLID()
tracker = referenceable.ReferenceableTracker(
self.tub, obj, puid, clid)
self.myReferenceByPUID[puid] = tracker
self.myReferenceByCLID[clid] = tracker
return tracker
def getTrackerForMyCall(self, puid, obj):
# just like getTrackerForMyReference, but with a negative clid
tracker = self.myReferenceByPUID.get(puid)
if not tracker:
# need to add one
clid = self.nextCLID()
clid = -clid
tracker = referenceable.ReferenceableTracker(
self.tub, obj, puid, clid)
self.myReferenceByPUID[puid] = tracker
self.myReferenceByCLID[clid] = tracker
return tracker
# methods to handle inbound 'my-reference' sequences
def getTrackerForYourReference(self, clid, interfaceName=None, url=None):
"""The far end holds a Referenceable and has just sent us a reference
to it (expressed as a small integer). If this is a new reference,
they will give us an interface name too, and possibly a global URL
for it. Obtain a RemoteReference object (creating it if necessary) to
give to the local recipient.
The sender remembers that we hold a reference to their object. When
our RemoteReference goes away, we send a decref message to them, so
they can possibly free their object. """
assert type(interfaceName) is str or interfaceName is None
if url is not None:
assert type(url) is str
tracker = self.yourReferenceByCLID.get(clid)
if not tracker:
# TODO: translate interfaceNames to RemoteInterfaces
if clid >= 0:
trackerclass = referenceable.RemoteReferenceTracker
else:
trackerclass = referenceable.RemoteMethodReferenceTracker
tracker = trackerclass(self, clid, url, interfaceName)
self.yourReferenceByCLID[clid] = tracker
if url:
self.yourReferenceByURL[url] = tracker
return tracker
def freeYourReference(self, tracker, count):
# this is called when the RemoteReference is deleted
if not self.remote_broker: # tests do not set this up
self.freeYourReferenceTracker(None, tracker)
return
try:
rb = self.remote_broker
# TODO: do we want callRemoteOnly here? is there a way we can
# avoid wanting to know when the decref has completed? Only if we
# send the interface list and URL on every occurrence of the
# my-reference sequence. Either A) we use callRemote("decref")
# and wait until the ack to free the tracker, or B) we use
# callRemoteOnly("decref") and free the tracker right away. In
# case B, the far end has no way to know that we've just freed
# the tracker and will therefore forget about everything they
# told us (including the interface list), so they cannot
# accurately do anything special on the "first" send of this
# reference. Which means that if we do B, we must either send
# that extra information on every my-reference sequence, or do
# without it, or make it optional, or retrieve it separately, or
# something.
# rb.callRemoteOnly("decref", clid=tracker.clid, count=count)
# self.freeYourReferenceTracker('bogus', tracker)
# return
d = rb.callRemote("decref", clid=tracker.clid, count=count)
# if the connection was lost before we can get an ack, we're
# tearing this down anyway
def _ignore_loss(f):
f.trap(DeadReferenceError, *LOST_CONNECTION_ERRORS)
return None
d.addErrback(_ignore_loss)
# once the ack comes back, or if we know we'll never get one,
# release the tracker
d.addCallback(self.freeYourReferenceTracker, tracker)
except:
f = failure.Failure()
log.msg("failure during freeRemoteReference",
facility="foolscap",
level=log.UNUSUAL,
failure=f)
def freeYourReferenceTracker(self, res, tracker):
if tracker.received_count != 0:
return
if self.yourReferenceByCLID.has_key(tracker.clid):
del self.yourReferenceByCLID[tracker.clid]
if tracker.url and self.yourReferenceByURL.has_key(tracker.url):
del self.yourReferenceByURL[tracker.url]
# methods to handle inbound 'your-reference' sequences
def getMyReferenceByCLID(self, clid):
"""clid is the connection-local ID of the Referenceable the other
end is trying to invoke or point to. If it is a number, they want an
implicitly-created per-connection object that we sent to them at
some point in the past. If it is a string, they want an object that
was registered with our Factory.
"""
assert isinstance(clid, (int, long))
if clid == 0:
return self
return self.myReferenceByCLID[clid].obj
# obj = IReferenceable(obj)
# assert isinstance(obj, pb.Referenceable)
# obj needs .getMethodSchema, which needs .getArgConstraint
def remote_decref(self, clid, count):
# invoked when the other side sends us a decref message
assert isinstance(clid, (int, long))
assert clid != 0
tracker = self.myReferenceByCLID.get(clid, None)
if not tracker:
return # already gone, probably because we're shutting down
done = tracker.decref(count)
if done:
del self.myReferenceByPUID[tracker.puid]
del self.myReferenceByCLID[clid]
# methods to send RemoteReference 'gifts' to third-parties
def makeGift(self, rref):
# return the giftid
broker, clid = rref.tracker.broker, rref.tracker.clid
i = (broker, clid)
old = self.myGifts.get(i)
if old:
rref, giftID, count = old
self.myGifts[i] = (rref, giftID, count + 1)
else:
giftID = self.nextGiftID()
self.myGiftsByGiftID[giftID] = i
self.myGifts[i] = (rref, giftID, 1)
return giftID
def remote_decgift(self, giftID, count):
broker, clid = self.myGiftsByGiftID[giftID]
rref, giftID, gift_count = self.myGifts[(broker, clid)]
gift_count -= count
if gift_count == 0:
del self.myGiftsByGiftID[giftID]
del self.myGifts[(broker, clid)]
else:
self.myGifts[(broker, clid)] = (rref, giftID, gift_count)
# methods to deal with URLs
def getYourReferenceByName(self, name):
d = self.remote_broker.callRemote("getReferenceByName", name=name)
return d
def remote_getReferenceByName(self, name):
return self.tub.getReferenceForName(name)
# remote-method-invocation methods, calling side, invoked by
# RemoteReference.callRemote and CallSlicer
def newRequestID(self):
if self.disconnected:
raise DeadReferenceError("Calling Stale Broker")
return self.nextReqID()
def addRequest(self, req):
req.broker = self
self.waitingForAnswers[req.reqID] = req
def removeRequest(self, req):
del self.waitingForAnswers[req.reqID]
def getRequest(self, reqID):
# invoked by AnswerUnslicer and ErrorUnslicer
try:
return self.waitingForAnswers[reqID]
except KeyError:
raise Violation("non-existent reqID '%d'" % reqID)
def abandonAllRequests(self, why):
for req in self.waitingForAnswers.values():
if why.check(*LOST_CONNECTION_ERRORS):
# map all connection-lost errors to DeadReferenceError, so
# application code only needs to check for one exception type
tubid = None
# since we're creating a new exception object for each call,
# let's add more information to it
if self.remote_tubref:
tubid = self.remote_tubref.getShortTubID()
e = DeadReferenceError("Connection was lost", tubid, req)
why = failure.Failure(e)
eventually(req.fail, why)
# target-side, invoked by CallUnslicer
def getRemoteInterfaceByName(self, riname):
# this lives in the broker because it ought to be per-connection
return remoteinterface.RemoteInterfaceRegistry[riname]
def getSchemaForMethod(self, rifaces, methodname):
# this lives in the Broker so it can override the resolution order,
# not that overlapping RemoteInterfaces should be allowed to happen
# all that often
for ri in rifaces:
m = ri.get(methodname)
if m:
return m
return None
def scheduleCall(self, delivery, ready_deferred):
self.inboundDeliveryQueue.append((delivery, ready_deferred))
eventually(self.doNextCall)
def doNextCall(self):
if self.disconnected:
return
if self._waiting_for_call_to_be_ready:
return
if not self.inboundDeliveryQueue:
return
delivery, ready_deferred = self.inboundDeliveryQueue.pop(0)
self._waiting_for_call_to_be_ready = True
if not ready_deferred:
ready_deferred = defer.succeed(None)
d = ready_deferred
def _ready(res):
self._waiting_for_call_to_be_ready = False
eventually(self.doNextCall)
return res
d.addBoth(_ready)
# at this point, the Deferred chain for this one delivery runs
# independently of any other, and methods which take a long time to
# complete will not hold up other methods. We must call _doCall and
# let the remote_ method get control before we process any other
# message, but the eventually() above insures we'll have a chance to
# do that before we give up control.
d.addCallback(lambda res: self._doCall(delivery))
d.addCallback(self._callFinished, delivery)
d.addErrback(self.callFailed, delivery.reqID, delivery)
d.addErrback(log.err)
return None
def _doCall(self, delivery):
# our ordering rules require that the order in which each
# remote_foo() method gets control is exactly the same as the order
# in which the original caller invoked callRemote(). To insure this,
# _startCall() is not allowed to insert additional delays before it
# runs doRemoteCall() on the target object.
obj = delivery.obj
args = delivery.allargs.args
kwargs = delivery.allargs.kwargs
for i in args + kwargs.values():
assert not isinstance(i, defer.Deferred)
if delivery.methodSchema:
# we asked about each argument on the way in, but ask again so
# they can look for missing arguments. TODO: see if we can remove
# the redundant per-argument checks.
delivery.methodSchema.checkAllArgs(args, kwargs, True)
# interesting case: if the method completes successfully, but
# our schema prohibits us from sending the result (perhaps the
# method returned an int but the schema insists upon a string).
# TODO: move the return-value schema check into
# Referenceable.doRemoteCall, so the exception's traceback will be
# attached to the object that caused it
if delivery.methodname is None:
assert callable(obj)
return obj(*args, **kwargs)
else:
obj = ipb.IRemotelyCallable(obj)
return obj.doRemoteCall(delivery.methodname, args, kwargs)
def _callFinished(self, res, delivery):
reqID = delivery.reqID
if reqID == 0:
return
methodSchema = delivery.methodSchema
assert self.activeLocalCalls[reqID]
methodName = None
if methodSchema:
methodName = methodSchema.name
try:
methodSchema.checkResults(res, False) # may raise Violation
except Violation, v:
v.prependLocation("in return value of %s.%s" %
(delivery.obj, methodSchema.name))
raise
answer = call.AnswerSlicer(reqID, res, methodName)
# once the answer has started transmitting, any exceptions must be
# logged and dropped, and not turned into an Error to be sent.
try:
self.send(answer)
# TODO: .send should return a Deferred that fires when the last
# byte has been queued, and we should delete the local note then
except:
f = failure.Failure()
log.msg("Broker._callfinished unable to send",
facility="foolscap",
level=log.UNUSUAL,
failure=f)
del self.activeLocalCalls[reqID]