def checkObject(self, obj): if type(obj) != types.TupleType: raise Violation("not a tuple") if len(obj) != len(self.constraints): raise Violation("wrong size tuple") for i in range(len(self.constraints)): self.constraints[i].checkObject(obj[i])
def checkObject(self, obj): if type(obj) != types.ListType: raise Violation("not a list") if len(obj) > self.maxLength: raise Violation("list too long") for o in obj: self.constraint.checkObject(o)
def checkObject(self, obj): if not isinstance(obj, (types.IntType, types.LongType)): raise Violation("not a number") if self.maxBytes == -1: if obj >= 2**31 or obj < -2**31: raise Violation("number too large") elif self.maxBytes != None: if abs(obj) >= 2**(8*self.maxBytes): raise Violation("number too large")
def checkToken(self, typebyte, size): """Check the token type. Raise an exception if it is not accepted right now, or if the body-length limit is exceeded.""" limit = self.taster.get(typebyte, "not in list") if limit == "not in list": if self.strictTaster: raise BananaError("invalid token type") else: raise Violation("%s token rejected by %s" % \ (tokenNames[typebyte], self.name)) if limit and size > limit: raise Violation("token too large: %d>%d" % (size, limit))
def openerCheckToken(self, typebyte, size, opentype): if typebyte == tokens.STRING: if size > self.maxIndexLength: why = "STRING token is too long, %d>%d" % \ (size, self.maxIndexLength) raise Violation(why) elif typebyte == tokens.VOCAB: return else: # TODO: hack for testing raise Violation("index token 0x%02x not STRING or VOCAB" % \ ord(typebyte)) raise BananaError("index token 0x%02x not STRING or VOCAB" % \ ord(typebyte))
def checkToken(self, typebyte, size): if self.maxLength != None and len(self.list) >= self.maxLength: # list is full, no more tokens accepted # this is hit if the max+1 item is a primitive type raise Violation("the list is full") if self.itemConstraint: self.itemConstraint.checkToken(typebyte, size)
def checkArgs(self, argdict): # this is called on the inbound side. Each argument has already been # checked individually, so all we have to do is verify global things # like all required arguments have been provided. for argname in self.required: if not argdict.has_key(argname): raise Violation("missing required argument '%s'" % argname)
def checkToken(self, typebyte, size): if self.maxKeys != None: if len(self.d) >= self.maxKeys: raise Violation("the dict is full") if self.gettingKey: if self.keyConstraint: self.keyConstraint.checkToken(typebyte, size) else: if self.valueConstraint: self.valueConstraint.checkToken(typebyte, size)
def receiveChild(self, obj, ready_deferred=None): assert not isinstance(obj, Deferred) assert ready_deferred is None assert type(obj) == int if self.constraint: if self.constraint.value != None: if bool(obj) != self.constraint.value: raise Violation("This boolean can only be %s" % \ self.constraint.value) self.value = bool(obj)
def checkObject(self, obj): ok = False for c in self.alternatives: try: c.checkObject(obj) ok = True except Violation: pass if not ok: raise Violation("does not satisfy any of %s" \ % (self.alternatives,))
def doOpen(self, opentype): where = len(self.list) if self.constraints != None: if where >= len(self.constraints): raise Violation("the tuple is full") self.constraints[where].checkOpentype(opentype) unslicer = self.open(opentype) if unslicer: if self.constraints != None: unslicer.setConstraint(self.constraints[where]) return unslicer
def getArgConstraint(self, argname): c = self.argConstraints.get(argname) if c: if isinstance(c, Optional): c = c.constraint return (True, c) # what do we do with unknown arguments? if self.ignoreUnknown: return (False, None) if self.acceptUnknown: return (True, None) raise Violation("unknown argument '%s'" % argname)
def open(self, opentype): # called (by delegation) by the top Unslicer on the stack, regardless # of what kind of unslicer it is. This is only used for "internal" # objects: non-top-level nodes assert len(self.protocol.receiveStack) > 1 for reg in self.openRegistry: opener = reg.get(opentype) if opener is not None: child = opener() return child else: raise Violation("unknown OPEN type %s" % (opentype, ))
def getAttrConstraint(self, attrname): c = self.keys.get(attrname) if c: if isinstance(c, Optional): c = c.constraint return (True, c) # unknown attribute if self.ignoreUnknown: return (False, None) if self.acceptUnknown: return (True, None) raise Violation("unknown attribute '%s'" % attrname)
def doOpen(self, opentype): # decide whether the given object type is acceptable here. Raise a # Violation exception if not, otherwise give it to our opener (which # will normally be the RootUnslicer). Apply a constraint to the new # unslicer. if self.maxLength != None and len(self.list) >= self.maxLength: # this is hit if the max+1 item is a non-primitive type raise Violation("the list is full") if self.itemConstraint: self.itemConstraint.checkOpentype(opentype) unslicer = self.open(opentype) if unslicer: if self.itemConstraint: unslicer.setConstraint(self.itemConstraint) return unslicer
def doOpen(self, opentype): if self.maxKeys != None: if len(self.d) >= self.maxKeys: raise Violation("the dict is full") if self.gettingKey: if self.keyConstraint: self.keyConstraint.checkOpentype(opentype) else: if self.valueConstraint: self.valueConstraint.checkOpentype(opentype) unslicer = self.open(opentype) if unslicer: if self.gettingKey: if self.keyConstraint: unslicer.setConstraint(self.keyConstraint) else: if self.valueConstraint: unslicer.setConstraint(self.valueConstraint) return unslicer
def doOpen(self, opentype): # this is only called for top-level objects assert len(self.protocol.receiveStack) == 1 if self.constraint: self.constraint.checkOpentype(opentype) if opentype == ("vocab", ): # only legal at top-level return VocabUnslicer() for reg in self.topRegistry: opener = reg.get(opentype) if opener is not None: child = opener() break else: raise Violation("unknown top-level OPEN type %s" % (opentype, )) if self.constraint: child.setConstraint(self.constraint) return child
def slicerForObject(self, obj): # could use a table here if you think it'd be faster than an # adapter lookup if self.debug: print "slicerForObject(%s)" % type(obj) # do the adapter lookup first, so that registered adapters override # UnsafeSlicerTable's InstanceSlicer slicer = tokens.ISlicer(obj, None) if slicer: if self.debug: print "got ISlicer", slicer return slicer slicerFactory = self.slicerTable.get(type(obj)) if slicerFactory: if self.debug: print " got slicerFactory", slicerFactory return slicerFactory(obj) if issubclass(type(obj), types.InstanceType): name = str(obj.__class__) else: name = str(type(obj)) if self.debug: print "cannot serialize %s (%s)" % (obj, name) raise Violation("cannot serialize %s (%s)" % (obj, name))
def receiveChild(self, obj, ready_deferred=None): assert ready_deferred is None if self.debug: print "%s[%d].receiveChild(%s)" % (self, self.count, obj) # obj could be a primitive type, a Deferred, or a complex type like # those returned from an InstanceUnslicer. However, the individual # object has already been through the schema validation process. The # only remaining question is whether the larger schema will accept # it. if self.maxLength != None and len(self.list) >= self.maxLength: # this is redundant # (if it were a non-primitive one, it would be caught in doOpen) # (if it were a primitive one, it would be caught in checkToken) raise Violation("the list is full") if isinstance(obj, Deferred): if self.debug: print " adding my update[%d] to %s" % (len(self.list), obj) obj.addCallback(self.update, len(self.list)) obj.addErrback(self.printErr) self.list.append("placeholder") else: self.list.append(obj)
def checkObject(self, obj, inbound): if type(obj) != type({}): raise Violation, "'%s' (%s) is not a Dictionary" % (obj, type(obj)) allkeys = self.keys.keys() for k in obj.keys(): try: constraint = self.keys[k] allkeys.remove(k) except KeyError: if not self.ignoreUnknown: raise Violation, "key '%s' not in schema" % k else: # hmm. kind of a soft violation. allow it for now. pass else: constraint.checkObject(obj[k], inbound) for k in allkeys[:]: if isinstance(self.keys[k], Optional): allkeys.remove(k) if allkeys: raise Violation("object is missing required keys: %s" % \ ",".join(allkeys))
def doOpen(self, opentype): raise Violation("'%s' does not accept sub-objects" % self)
def checkObject(self, obj): if type(obj) != types.BooleanType: raise Violation("not a bool") if self.value != None: if obj != self.value: raise Violation("not %s" % self.value)
def checkObject(self, obj): # TODO: maybe try to get an adapter instead? if not self.interface.providedBy(obj): raise Violation("does not provide interface %s" % self.interface)
def produce(self, dummy=None): # optimize: cache 'next' because we get many more tokens than stack # pushes/pops while self.slicerStack and not self.paused: if self.debugSend: print "produce.loop" try: slicer, next, openID = self.slicerStack[-1] obj = next() if self.debugSend: print " produce.obj=%s" % (obj, ) if isinstance(obj, defer.Deferred): for s, n, o in self.slicerStack: if not s.streamable: raise Violation("parent not streamable") obj.addCallback(self.produce) obj.addErrback(self._slice_error, s) # this is the primary exit point break elif type(obj) in (int, long, float, str): # sendToken raises a BananaError for weird tokens self.sendToken(obj) else: # newSlicerFor raises a Violation for unsendable types # pushSlicer calls .slice, which can raise Violation try: slicer = self.newSlicerFor(obj) self.pushSlicer(slicer, obj) except Violation, v: # pushSlicer is arranged such that the pushing of # the Slicer and the sending of the OPEN happen # together: either both occur or neither occur. In # addition, there is nothing past the OPEN/push # which can cause an exception. # Therefore, if an exception was raised, we know # that the OPEN has not been sent (so we don't have # to send an ABORT), and that the new Unslicer has # not been pushed (so we don't have to pop one from # the stack) f = BananaFailure() if self.debugSend: print " violation in newSlicerFor:", f self.handleSendViolation(f, doPop=False, sendAbort=False) except StopIteration: if self.debugSend: print "StopIteration" self.popSlicer() except Violation, v: # Violations that occur because of Constraints are caught # before the Slicer is pushed. A Violation that is caught # here was raised inside .next(), or .streamable wasn't # obeyed. The Slicer should now be abandoned. if self.debugSend: print " violation in .next:", v f = BananaFailure() self.handleSendViolation(f, doPop=True, sendAbort=True) except:
def checkObject(self, obj, inbound): if not isinstance(obj, self.klass): raise Violation("is not an instance of %s" % self.klass)
def receiveChild(self, token, ready_deferred=None): assert not isinstance(token, defer.Deferred) if self.debug: log.msg("%s.receiveChild [s%d]: %s" % (self, self.stage, repr(token))) if self.stage == 0: # reqID # we don't yet know which reqID to send any failure to assert ready_deferred is None self.reqID = token self.stage = 1 if self.reqID != 0: assert self.reqID not in self.broker.activeLocalCalls self.broker.activeLocalCalls[self.reqID] = self return if self.stage == 1: # objID # this might raise an exception if objID is invalid assert ready_deferred is None self.objID = token try: self.obj = self.broker.getMyReferenceByCLID(token) except KeyError: raise Violation("unknown CLID %d" % (token, )) #iface = self.broker.getRemoteInterfaceByName(token) if self.objID < 0: self.interface = None else: self.interface = self.obj.getInterface() self.stage = 2 return if self.stage == 2: # methodname # validate the methodname, get the schema. This may raise an # exception for unknown methods # must find the schema, using the interfaces # TODO: getSchema should probably be in an adapter instead of in # a pb.Referenceable base class. Old-style (unconstrained) # flavors.Referenceable should be adapted to something which # always returns None # TODO: make this faster. A likely optimization is to take a # tuple of components.getInterfaces(obj) and use it as a cache # key. It would be even faster to use obj.__class__, but that # would probably violate the expectation that instances can # define their own __implements__ (independently from their # class). If this expectation were to go away, a quick # obj.__class__ -> RemoteReferenceSchema cache could be built. assert ready_deferred is None self.stage = 3 if self.objID < 0: # the target is a bound method, ignore the methodname self.methodSchema = getattr(self.obj, "methodSchema", None) self.methodname = None # TODO: give it something useful if self.broker.requireSchema and not self.methodSchema: why = "This broker does not accept unconstrained " + \ "method calls" raise Violation(why) return self.methodname = token if self.interface: # they are calling an interface+method pair ms = self.interface.get(self.methodname) if not ms: why = "method '%s' not defined in %s" % \ (self.methodname, self.interface.__remote_name__) raise Violation(why) self.methodSchema = ms return if self.stage == 3: # arguments assert isinstance(token, ArgumentUnslicer) self.allargs = token # queue the message. It will not be executed until all the # arguments are ready. The .args list and .kwargs dict may change # before then. if ready_deferred: self._ready_deferreds.append(ready_deferred) self.stage = 4 return
def requireBroker(self, protocol): broker = IBroker(protocol, None) if not broker: msg = "This object can only be serialized by a broker" raise Violation(msg) return broker
def checkToken(self, typebyte, size): if self.constraints == None: return if len(self.list) >= len(self.constraints): raise Violation("the tuple is full") self.constraints[len(self.list)].checkToken(typebyte, size)
def receiveChild(self, token, ready_deferred=None): if self.stage < 3: assert not isinstance(token, defer.Deferred) assert ready_deferred is None #print "CallUnslicer.receiveChild [s%d]" % self.stage, repr(token) # TODO: if possible, return an error to the other side if self.stage == 0: # reqID # we don't yet know which reqID to send any failure to self.reqID = token self.stage += 1 assert not self.broker.activeLocalCalls.get(self.reqID) self.broker.activeLocalCalls[self.reqID] = self return if self.stage == 1: # objID # this might raise an exception if objID is invalid self.objID = token self.obj = self.broker.getMyReferenceByCLID(token) #iface = self.broker.getRemoteInterfaceByName(token) if self.objID < 0: self.interface = None else: self.interface = self.obj.getInterface() self.stage = 2 return if self.stage == 2: # methodname # validate the methodname, get the schema. This may raise an # exception for unknown methods if self.objID < 0: # the target is a bound method self.methodSchema = getattr(self.obj, "methodSchema", None) self.methodname = None # TODO: give it something useful if self.broker.requireSchema and not self.methodSchema: why = "This broker does not accept unconstrained " + \ "method calls" raise Violation(why) self.stage = 3 return methodname = token # must find the schema, using the interfaces # TODO: getSchema should probably be in an adapter instead of in # a pb.Referenceable base class. Old-style (unconstrained) # flavors.Referenceable should be adapted to something which # always returns None # TODO: make this faster. A likely optimization is to take a # tuple of components.getInterfaces(obj) and use it as a cache # key. It would be even faster to use obj.__class__, but that # would probably violate the expectation that instances can # define their own __implements__ (independently from their # class). If this expectation were to go away, a quick # obj.__class__ -> RemoteReferenceSchema cache could be built. ms = None if self.interface: # they are calling an interface+method pair ms = self.interface.get(methodname) if not ms: why = "method '%s' not defined in %s" % \ (methodname, self.interface.__remote_name__) raise Violation(why) self.methodSchema = ms self.methodname = methodname if self.broker.requireSchema and not self.methodSchema: why = "This broker does not accept unconstrained method calls" raise Violation(why) self.stage = 3 return if self.stage == 3: # argname/value pairs if self.argname == None: assert not isinstance(token, defer.Deferred) assert ready_deferred is None argname = token if self.args.has_key(argname): raise BananaError("duplicate argument '%s'" % argname) ms = self.methodSchema if ms: # if the argname is invalid, this may raise Violation accept, self.argConstraint = ms.getArgConstraint(argname) assert accept # TODO: discard if not self.argname = argname else: argvalue = token if isinstance(argvalue, defer.Deferred): self.num_unreferenceable_children += 1 argvalue.addCallback(self.update, self.argname) argvalue.addErrback(self.explode) self.args[self.argname] = argvalue self.argname = None if ready_deferred: self.num_unready_children += 1 ready_deferred.addCallback(self.ready)
def handleData(self, chunk): # buffer, assemble into tokens # call self.receiveToken(token) with each if self.skipBytes: if len(chunk) <= self.skipBytes: # skip the whole chunk self.skipBytes -= len(chunk) return # skip part of the chunk, and stop skipping chunk = chunk[self.skipBytes:] self.skipBytes = 0 self.buffer.append(chunk) # Loop through the available input data, extracting one token per # pass. while len(self.buffer): first65 = self.buffer.popleft(65) pos = 0 for ch in first65: if ch >= HIGH_BIT_SET: break pos = pos + 1 if pos > 64: # drop the connection. We log more of the buffer, but not # all of it, to make it harder for someone to spam our # logs. s = first65 + self.buffer.popleft(200) raise BananaError("token prefix is limited to 64 bytes: " "but got %r" % s) else: # we've run out of buffer without seeing the high bit, which # means we're still waiting for header to finish self.buffer.appendleft(first65) return assert pos <= 64 # At this point, the header and type byte have been received. # The body may or may not be complete. typebyte = first65[pos] if pos: header = b1282int(first65[:pos]) else: header = 0 # rejected is set as soon as a violation is detected. It # indicates that this single token will be rejected. rejected = False if self.discardCount: rejected = True wasInOpen = self.inOpen if typebyte == OPEN: self.inboundObjectCount = self.objectCounter self.objectCounter += 1 if self.inOpen: raise BananaError("OPEN token followed by OPEN") self.inOpen = True # the inOpen flag is set as soon as the OPEN token is # witnessed (even it it gets rejected later), because it # means that there is a new sequence starting that must be # handled somehow (either discarded or given to a new # Unslicer). # The inOpen flag is cleared when the Index Phase ends. There # are two possibilities: 1) a new Unslicer is pushed, and # tokens are delivered to it normally. 2) a Violation was # raised, and the tokens must be discarded # (self.discardCount++). *any* rejection-caused True->False # transition of self.inOpen must be accompanied by exactly # one increment of self.discardCount # determine if this token will be accepted, and if so, how large # it is allowed to be (for STRING and LONGINT/LONGNEG) if ((not rejected) and (typebyte not in (PING, PONG, ABORT, CLOSE, ERROR))): # PING, PONG, ABORT, CLOSE, and ERROR are always legal. All # others (including OPEN) can be rejected by the schema: for # example, a list of integers would reject STRING, VOCAB, and # OPEN because none of those will produce integers. If the # unslicer's .checkToken rejects the tokentype, its # .receiveChild will immediately get an Failure try: # the purpose here is to limit the memory consumed by # the body of a STRING, OPEN, LONGINT, or LONGNEG token # (i.e., the size of a primitive type). If the sender # wants to feed us more data than we want to accept, the # checkToken() method should raise a Violation. This # will never be called with ABORT or CLOSE types. top = self.receiveStack[-1] if wasInOpen: top.openerCheckToken(typebyte, header, self.opentype) else: top.checkToken(typebyte, header) except Violation: rejected = True f = BananaFailure() if wasInOpen: methname = "openerCheckToken" else: methname = "checkToken" self.handleViolation(f, methname, inOpen=self.inOpen) self.inOpen = False if typebyte == ERROR and header > SIZE_LIMIT: # someone is trying to spam us with an ERROR token. Drop # them with extreme prejudice. raise BananaError("oversized ERROR token") self.buffer.appendleft(first65[pos + 1:]) # determine what kind of token it is. Each clause finishes in # one of four ways: # # raise BananaError: the protocol was violated so badly there is # nothing to do for it but hang up abruptly # # return: if the token is not yet complete (need more data) # # continue: if the token is complete but no object (for # handleToken) was produced, e.g. OPEN, CLOSE, ABORT # # obj=foo: the token is complete and an object was produced # # note that if rejected==True, the object is dropped instead of # being passed up to the current Unslicer if typebyte == OPEN: self.inboundOpenCount = header if rejected: if self.debugReceive: print "DROP (OPEN)" if self.inOpen: # we are discarding everything at the old level, so # discard everything in the new level too self.discardCount += 1 if self.debugReceive: print "++discardCount (OPEN), now %d" \ % self.discardCount self.inOpen = False else: # the checkToken handleViolation has already started # discarding this new sequence, we don't have to pass else: self.inOpen = True self.opentype = [] continue elif typebyte == CLOSE: count = header if self.discardCount: self.discardCount -= 1 if self.debugReceive: print "--discardCount (CLOSE), now %d" \ % self.discardCount else: self.handleClose(count) continue elif typebyte == ABORT: count = header # TODO: this isn't really a Violation, but we need something # to describe it. It does behave identically to what happens # when receiveChild raises a Violation. The .handleViolation # will pop the now-useless Unslicer and start discarding # tokens just as if the Unslicer had made the decision. if rejected: if self.debugReceive: print "DROP (ABORT)" # I'm ignoring you, LALALALALA. # # In particular, do not deliver a second Violation # because of the ABORT that we're supposed to be # ignoring because of a first Violation that happened # earlier. continue try: # slightly silly way to do it, but nice and uniform raise Violation("ABORT received") except Violation: f = BananaFailure() self.handleViolation(f, "receive-abort") continue elif typebyte == ERROR: strlen = header if len(self.buffer) >= strlen: # the whole string is available obj = self.buffer.popleft(strlen) # handleError must drop the connection self.handleError(obj) return else: self.buffer.appendleft(first65[:pos + 1]) return # there is more to come elif typebyte == LIST: raise BananaError("oldbanana peer detected, " + "compatibility code not yet written") #listStack.append((header, [])) elif typebyte == STRING: strlen = header if len(self.buffer) >= strlen: # the whole string is available obj = self.buffer.popleft(strlen) # although it might be rejected else: # there is more to come if rejected: # drop all we have and note how much more should be # dropped if self.debugReceive: print "DROPPED some string bits" self.skipBytes = strlen - len(self.buffer) self.buffer.clear() else: self.buffer.appendleft(first65[:pos + 1]) return elif typebyte == INT: obj = int(header) elif typebyte == NEG: # -2**31 is too large for a positive int, so go through # LongType first obj = int(-long(header)) elif typebyte == LONGINT or typebyte == LONGNEG: strlen = header if len(self.buffer) >= strlen: # the whole number is available obj = bytes_to_long(self.buffer.popleft(strlen)) if typebyte == LONGNEG: obj = -obj # although it might be rejected else: # there is more to come if rejected: # drop all we have and note how much more should be # dropped self.skipBytes = strlen - len(self.buffer) self.buffer.clear() else: self.buffer.appendleft(first65[:pos + 1]) return elif typebyte == VOCAB: obj = self.incomingVocabulary[header] # TODO: bail if expanded string is too big # this actually means doing self.checkToken(VOCAB, len(obj)) # but we have to make sure we handle the rejection properly elif typebyte == FLOAT: if len(self.buffer) >= 8: obj = struct.unpack("!d", self.buffer.popleft(8))[0] else: # this case is easier than STRING, because it is only 8 # bytes. We don't bother skipping anything. self.buffer.appendleft(first65[:pos + 1]) return elif typebyte == PING: self.sendPONG(header) continue # otherwise ignored elif typebyte == PONG: continue # otherwise ignored else: raise BananaError("Invalid Type Byte 0x%x" % ord(typebyte)) if not rejected: if self.inOpen: self.handleOpen(self.inboundOpenCount, self.inboundObjectCount, obj) # handleOpen might push a new unslicer and clear # .inOpen, or leave .inOpen true and append the object # to .indexOpen else: self.handleToken(obj) else: if self.debugReceive: print "DROP", type(obj), obj pass # drop the object # while loop ends here # note: this is redundant, as there are no 'break' statements in that # loop, and the loop exit condition is 'while len(self.buffer)' self.buffer.clear()