def _parseMessage(self, now, message):
message = parseMessage(message)
sentAt = self._sentMessageAt.pop(message.previousID, None)
if sentAt is not None:
self._congestion.processDelta(now, now - sentAt)
if message.id:
self._clock.callLater(0, self._sendAMessage, ack=message.id)
self._theyAcked.update(message.ranges)
for qd in list(self._sentMessages):
if qd.interval & self._theyAcked:
qd.interval.difference_update(self._theyAcked)
if not qd.interval:
for d in qd.deferreds:
d.callback(now)
self._cancel(qd)
self._outstandingMessages -= 1
self._sentMessages.remove(qd)
if message.resolution and self._theirStreamEnd is None:
self._theirStreamEnd = message.dataPos
self._theirResolution = message.resolution
self._received.add(halfOpen(message.dataPos, message.dataPos + 1))
self._reads = 'closing'
self._checkTheirResolution()
return
elif not message.data:
return
i = halfOpen(message.dataPos, message.dataPos + len(message.data))
new = IntervalSet([i]) - self._received
if not new:
return
self._received.add(i)
newData = message.data[new.lower_bound() -
i.lower_bound:new.upper_bound() -
i.upper_bound or None]
bisect.insort(self._fragment, (i.lower_bound, newData))
if len(self._received) > 1 or self._received.lower_bound() != 0:
return
newData = ''.join([d for _, d in self._fragment])
self._protocol.dataReceived(newData)
self._fragment = []
self._checkTheirResolution()
def _parseMessage(self, now, message):
message = parseMessage(message)
sentAt = self._sentMessageAt.pop(message.previousID, None)
if sentAt is not None:
self._congestion.processDelta(now, now - sentAt)
if message.id:
self._clock.callLater(0, self._sendAMessage, ack=message.id)
self._theyAcked.update(message.ranges)
for qd in list(self._sentMessages):
if qd.interval & self._theyAcked:
qd.interval.difference_update(self._theyAcked)
if not qd.interval:
for d in qd.deferreds:
d.callback(now)
self._cancel(qd)
self._outstandingMessages -= 1
self._sentMessages.remove(qd)
if message.resolution and self._theirStreamEnd is None:
self._theirStreamEnd = message.dataPos
self._theirResolution = message.resolution
self._received.add(halfOpen(message.dataPos, message.dataPos + 1))
self._reads = 'closing'
self._checkTheirResolution()
return
elif not message.data:
return
i = halfOpen(message.dataPos, message.dataPos + len(message.data))
new = IntervalSet([i]) - self._received
if not new:
return
self._received.add(i)
newData = message.data[new.lower_bound() - i.lower_bound:new.upper_bound() - i.upper_bound or None]
bisect.insort(self._fragment, (i.lower_bound, newData))
if len(self._received) > 1 or self._received.lower_bound() != 0:
return
newData = ''.join([d for _, d in self._fragment])
self._protocol.dataReceived(newData)
self._fragment = []
self._checkTheirResolution()
class RangeItem:
items = IntervalSet.empty()
def __init__(self, low, high):
"""
Initialize a range item with lower bound and higher bound.
"""
self.items = IntervalSet([Interval(low, high)])
def __eq__(self, rhs):
"""
Redefine == operator
"""
assert isinstance(rhs, RangeItem)
return self.items == rhs.items
def empty(self):
if len(self.items) == 0:
return True
return False
def dump(self, type='ip'):
if self.empty():
print 'Empty RangeItem'
else:
if type == 'int':
print 'RangeItem %s - %s' % (self.items.lower_bound(),
self.items.upper_bound())
elif type == 'ip':
print 'RangeItem %s - %s' % (long2ip(self.items.lower_bound()),
long2ip(self.items.upper_bound()))
def produce(self):
return RangeItem(self.items.lower_bound(), self.items.upper_bound())
def overlaps(self, rhs):
"""
If this item is overlapped with the other
"""
if len(self.items & rhs.items) != 0:
return True
return False
def includes(self, rhs):
"""
If this item includes the other
"""
if self.items & rhs.items == rhs.items:
return True
return False
def decouples(self, rhs):
"""
Decouple this item with the other
"""
assert isinstance(rhs, RangeItem)
# print self.items, rhs.items
new_values = [
self.items - rhs.items, rhs.items - self.items,
self.items & rhs.items
]
new_items = []
for value in new_values:
# print value
# print '%s - %s' % str(value.lower_bound()), str(value.upper_bound)
if len(value) != 0:
new_items.append(
RangeItem(value.lower_bound(), value.upper_bound()))
else:
empty_range = RangeItem(0, 0)
empty_range.items = IntervalSet.empty()
new_items.append(empty_range)
return new_items
class _CurveCPBaseTransport(DatagramProtocol):
timeouts = 1, 1, 2, 3, 5, 8, 13
_generateKey = staticmethod(PrivateKey.generate)
_generateKeydir = staticmethod(EphemeralKey)
def __init__(self, clock, serverKey, factory):
self._clock = clock
self._serverKey = serverKey
self._factory = factory
self._received = IntervalSet()
self._weAcked = IntervalSet()
self._sent = IntervalSet()
self._theyAcked = IntervalSet()
self._sentMessages = set()
self._previousID = 0
self._fragment = []
self._congestion = Chicago()
self._sentMessageAt = {}
self._delayedCalls = {}
self._messageQueue = []
self._enqueuedMessages = set()
self._deferred = defer.Deferred()
self._nonce = 0
self._theirLastNonce = 0
self._counter = 1
self._ourStreamEnd = None
self._theirStreamEnd = None
self._reads = self._writes = None
self._done = False
self._outstandingMessages = 0
self._onDone = []
def _now(self):
return self._clock.seconds()
def _timedOutHandshaking(self):
self._deferred.errback(e.HandshakeTimeout())
def _write(self, data):
self.transport.write(data, self._peerHost)
def _retrySendingForHandshake(self, data):
mt = MultiTimeout(self._clock, self.timeouts,
self._timedOutHandshaking, self._write, data)
mt.reset()
return mt
messageMap = {}
def datagramReceived(self, data, host_port):
if self._done:
return
handler = self._messageMap.get(data[:8])
if not handler:
return
meth = getattr(self, '_datagram_' + handler)
meth(data, host_port)
_nonceInfix = ''
def _encryptForNonce(self, which, box, data):
packedNonce = _nonceStruct.pack(self._nonce)
self._nonce += 1
nonce = 'CurveCP-%s-%s%s' % (self._nonceInfix, which, packedNonce)
return packedNonce + box.encrypt(data, nonce).ciphertext
def _verifyNonce(self, nonce):
unpacked, = _nonceStruct.unpack(nonce)
ret = unpacked >= self._theirLastNonce
self._theirLastNonce = unpacked
return ret
def _serializeMessage(self, message):
return ''
def _sendMessage(self, message):
packet = self._serializeMessage(message)
self._write(packet)
if message.id:
self._sentMessageAt[
message.id] = self._congestion.lastSentAt = self._now()
self._weAcked.update(message.ranges)
def _parseMessage(self, now, message):
message = parseMessage(message)
sentAt = self._sentMessageAt.pop(message.previousID, None)
if sentAt is not None:
self._congestion.processDelta(now, now - sentAt)
if message.id:
self._clock.callLater(0, self._sendAMessage, ack=message.id)
self._theyAcked.update(message.ranges)
for qd in list(self._sentMessages):
if qd.interval & self._theyAcked:
qd.interval.difference_update(self._theyAcked)
if not qd.interval:
for d in qd.deferreds:
d.callback(now)
self._cancel(qd)
self._outstandingMessages -= 1
self._sentMessages.remove(qd)
if message.resolution and self._theirStreamEnd is None:
self._theirStreamEnd = message.dataPos
self._theirResolution = message.resolution
self._received.add(halfOpen(message.dataPos, message.dataPos + 1))
self._reads = 'closing'
self._checkTheirResolution()
return
elif not message.data:
return
i = halfOpen(message.dataPos, message.dataPos + len(message.data))
new = IntervalSet([i]) - self._received
if not new:
return
self._received.add(i)
newData = message.data[new.lower_bound() -
i.lower_bound:new.upper_bound() -
i.upper_bound or None]
bisect.insort(self._fragment, (i.lower_bound, newData))
if len(self._received) > 1 or self._received.lower_bound() != 0:
return
newData = ''.join([d for _, d in self._fragment])
self._protocol.dataReceived(newData)
self._fragment = []
self._checkTheirResolution()
def _checkTheirResolution(self):
if self._theirStreamEnd is None:
return
if len(self._received) != 1 or self._received.lower_bound() != 0:
return
self._reads = 'closed'
self._protocol.readConnectionLost()
self._checkBothResolutions()
def notifyFinish(self):
if self._done:
return defer.succeed(None)
d = defer.Deferred()
self._onDone.append(d)
return d
def _checkBothResolutions(self):
if self._reads == self._writes == 'closed' and not self._done:
self._protocol.connectionLost(
Failure(e.resolution_map[self._theirResolution]()))
self._cancel('message')
deferreds, self._onDone = self._onDone, None
for d in deferreds:
d.callback(None)
# this used to be done on a callLater, but I can't remember why
self._done = True
def _sendAMessage(self, ack=None):
now = self._now()
nextActionIn = None
message = Message(
id=self._counter,
previousID=0,
ranges=list(self._received)[:6],
resolution=None,
dataPos=0,
data='',
)
if ack is not None:
message = message._replace(id=0, previousID=ack)
elif self._messageQueue:
_, _, qd = heapq.heappop(self._messageQueue)
self._enqueuedMessages.remove(qd)
message = qd.fillInMessage(message)
self._counter += 1
if qd.sentAt:
self._congestion.timedOut(now)
self._sentMessageAt.pop(qd.messageIDs[-1], None)
elif self._congestion.window is not None and self._outstandingMessages > self._congestion.window:
self._enqueue(1, qd)
return
else:
self._outstandingMessages += 1
qd.sentAt.append(now)
qd.messageIDs.append(message.id)
self._sentMessages.add(qd)
self._reschedule(qd)
else:
return 60
self._sendMessage(message)
return nextActionIn
def _reschedule(self, what, nextActionIn=None):
now = self._now()
if nextActionIn is None:
if what == 'message':
nextActionIn = self._congestion.nextMessageIn(now)
else:
nextActionIn = self._congestion.nextTimeoutIn(now, what)
delayedCall = self._delayedCalls.get(what)
if delayedCall is not None and delayedCall.active():
delayedCall.reset(nextActionIn)
else:
self._delayedCalls[what] = self._clock.callLater(
nextActionIn, self._scheduledAction, what)
def _cancel(self, what):
delayedCall = self._delayedCalls.pop(what, None)
if delayedCall is not None and delayedCall.active():
delayedCall.cancel()
def _scheduledAction(self, what):
nextActionIn = None
if what == 'message':
nextActionIn = self._sendAMessage()
self._reschedule(what, nextActionIn=nextActionIn)
else:
self._sentMessages.remove(what)
if what.interval:
self._enqueue(0, what)
def _enqueue(self, priority, *data):
self._reschedule('message')
for datum in data:
if datum not in self._enqueuedMessages and datum.interval:
heapq.heappush(self._messageQueue,
(priority, datum.lowerBound, datum))
self._enqueuedMessages.add(datum)
def write(self, data):
if not data:
return defer.succeed(None)
elif self._writes in ('closing', 'closed'):
return defer.fail(
e.CurveCPConnectionDone(
'attempted a write after closing writes'))
d = defer.Deferred()
qds = []
lowerBound = self._sent.upper_bound() if self._sent else 0
while data:
ds = []
queueableData = data[:1024]
dataRange = halfOpen(lowerBound, lowerBound + len(queueableData))
qd = QueuedData(IntervalSet([dataRange]), lowerBound, ds, [], [],
queueableData)
self._sent.add(dataRange)
lowerBound += len(queueableData)
data = data[1024:]
qds.append(qd)
ds.append(d)
self._enqueue(1, *qds)
return d
def _peerEstablished(self):
self._protocol = self._factory.buildProtocol(self.getPeer())
self._protocol.makeConnection(self)
self._deferred.callback(self._protocol)
self._reads = 'open'
self._writes = 'open'
def _doneWritingAcked(self, when):
self._writes = 'closed'
self._checkBothResolutions()
return when
def loseConnection(self, success=True):
d = defer.Deferred()
d.addCallback(self._doneWritingAcked)
streamEnd = self._ourStreamEnd = self._sent.upper_bound(
) if self._sent else 0
resolution = self._ourResolution = 'success' if success else 'failure'
interval = IntervalSet([halfOpen(streamEnd, streamEnd + 1)])
self._enqueue(
1, QueuedResolution(interval, streamEnd, [d], [], [], resolution))
self._writes = 'closing'
return d
class RangeItem:
items = IntervalSet.empty()
def __init__(self, low, high):
"""
Initialize a range item with lower bound and higher bound.
"""
self.items = IntervalSet([Interval(low, high)])
def __eq__(self, rhs):
"""
Redefine == operator
"""
assert isinstance(rhs, RangeItem)
return self.items == rhs.items
def empty(self):
if len(self.items) == 0:
return True
return False
def dump(self, type='ip'):
if self.empty():
print 'Empty RangeItem'
else:
if type == 'int':
print 'RangeItem %s - %s' % (self.items.lower_bound(), self.items.upper_bound())
elif type == 'ip':
print 'RangeItem %s - %s' % (long2ip(self.items.lower_bound()), long2ip(self.items.upper_bound()))
def produce(self):
return RangeItem(self.items.lower_bound(), self.items.upper_bound())
def overlaps(self, rhs):
"""
If this item is overlapped with the other
"""
if len(self.items & rhs.items) != 0:
return True
return False
def includes(self, rhs):
"""
If this item includes the other
"""
if self.items & rhs.items == rhs.items:
return True
return False
def decouples(self, rhs):
"""
Decouple this item with the other
"""
assert isinstance(rhs, RangeItem)
# print self.items, rhs.items
new_values = [self.items - rhs.items, rhs.items - self.items, self.items & rhs.items]
new_items = []
for value in new_values:
# print value
# print '%s - %s' % str(value.lower_bound()), str(value.upper_bound)
if len(value) != 0:
new_items.append(RangeItem(value.lower_bound(), value.upper_bound()))
else:
empty_range = RangeItem(0, 0)
empty_range.items = IntervalSet.empty()
new_items.append(empty_range)
return new_items
class _CurveCPBaseTransport(DatagramProtocol):
timeouts = 1, 1, 2, 3, 5, 8, 13
_generateKey = staticmethod(PrivateKey.generate)
_generateKeydir = staticmethod(EphemeralKey)
def __init__(self, clock, serverKey, factory):
self._clock = clock
self._serverKey = serverKey
self._factory = factory
self._received = IntervalSet()
self._weAcked = IntervalSet()
self._sent = IntervalSet()
self._theyAcked = IntervalSet()
self._sentMessages = set()
self._previousID = 0
self._fragment = []
self._congestion = Chicago()
self._sentMessageAt = {}
self._delayedCalls = {}
self._messageQueue = []
self._enqueuedMessages = set()
self._deferred = defer.Deferred()
self._nonce = 0
self._theirLastNonce = 0
self._counter = 1
self._ourStreamEnd = None
self._theirStreamEnd = None
self._reads = self._writes = None
self._done = False
self._outstandingMessages = 0
self._onDone = []
def _now(self):
return self._clock.seconds()
def _timedOutHandshaking(self):
self._deferred.errback(e.HandshakeTimeout())
def _write(self, data):
self.transport.write(data, self._peerHost)
def _retrySendingForHandshake(self, data):
mt = MultiTimeout(
self._clock, self.timeouts, self._timedOutHandshaking, self._write, data)
mt.reset()
return mt
messageMap = {}
def datagramReceived(self, data, host_port):
if self._done:
return
handler = self._messageMap.get(data[:8])
if not handler:
return
meth = getattr(self, '_datagram_' + handler)
meth(data, host_port)
_nonceInfix = ''
def _encryptForNonce(self, which, box, data):
packedNonce = _nonceStruct.pack(self._nonce)
self._nonce += 1
nonce = 'CurveCP-%s-%s%s' % (self._nonceInfix, which, packedNonce)
return packedNonce + box.encrypt(data, nonce).ciphertext
def _verifyNonce(self, nonce):
unpacked, = _nonceStruct.unpack(nonce)
ret = unpacked >= self._theirLastNonce
self._theirLastNonce = unpacked
return ret
def _serializeMessage(self, message):
return ''
def _sendMessage(self, message):
packet = self._serializeMessage(message)
self._write(packet)
if message.id:
self._sentMessageAt[message.id] = self._congestion.lastSentAt = self._now()
self._weAcked.update(message.ranges)
def _parseMessage(self, now, message):
message = parseMessage(message)
sentAt = self._sentMessageAt.pop(message.previousID, None)
if sentAt is not None:
self._congestion.processDelta(now, now - sentAt)
if message.id:
self._clock.callLater(0, self._sendAMessage, ack=message.id)
self._theyAcked.update(message.ranges)
for qd in list(self._sentMessages):
if qd.interval & self._theyAcked:
qd.interval.difference_update(self._theyAcked)
if not qd.interval:
for d in qd.deferreds:
d.callback(now)
self._cancel(qd)
self._outstandingMessages -= 1
self._sentMessages.remove(qd)
if message.resolution and self._theirStreamEnd is None:
self._theirStreamEnd = message.dataPos
self._theirResolution = message.resolution
self._received.add(halfOpen(message.dataPos, message.dataPos + 1))
self._reads = 'closing'
self._checkTheirResolution()
return
elif not message.data:
return
i = halfOpen(message.dataPos, message.dataPos + len(message.data))
new = IntervalSet([i]) - self._received
if not new:
return
self._received.add(i)
newData = message.data[new.lower_bound() - i.lower_bound:new.upper_bound() - i.upper_bound or None]
bisect.insort(self._fragment, (i.lower_bound, newData))
if len(self._received) > 1 or self._received.lower_bound() != 0:
return
newData = ''.join([d for _, d in self._fragment])
self._protocol.dataReceived(newData)
self._fragment = []
self._checkTheirResolution()
def _checkTheirResolution(self):
if self._theirStreamEnd is None:
return
if len(self._received) != 1 or self._received.lower_bound() != 0:
return
self._reads = 'closed'
self._protocol.readConnectionLost()
self._checkBothResolutions()
def notifyFinish(self):
if self._done:
return defer.succeed(None)
d = defer.Deferred()
self._onDone.append(d)
return d
def _checkBothResolutions(self):
if self._reads == self._writes == 'closed' and not self._done:
self._protocol.connectionLost(Failure(e.resolution_map[self._theirResolution]()))
self._cancel('message')
deferreds, self._onDone = self._onDone, None
for d in deferreds:
d.callback(None)
# this used to be done on a callLater, but I can't remember why
self._done = True
def _sendAMessage(self, ack=None):
now = self._now()
nextActionIn = None
message = Message(
id=self._counter,
previousID=0,
ranges=list(self._received)[:6],
resolution=None,
dataPos=0,
data='',
)
if ack is not None:
message = message._replace(id=0, previousID=ack)
elif self._messageQueue:
_, _, qd = heapq.heappop(self._messageQueue)
self._enqueuedMessages.remove(qd)
message = qd.fillInMessage(message)
self._counter += 1
if qd.sentAt:
self._congestion.timedOut(now)
self._sentMessageAt.pop(qd.messageIDs[-1], None)
elif self._congestion.window is not None and self._outstandingMessages > self._congestion.window:
self._enqueue(1, qd)
return
else:
self._outstandingMessages += 1
qd.sentAt.append(now)
qd.messageIDs.append(message.id)
self._sentMessages.add(qd)
self._reschedule(qd)
else:
return 60
self._sendMessage(message)
return nextActionIn
def _reschedule(self, what, nextActionIn=None):
now = self._now()
if nextActionIn is None:
if what == 'message':
nextActionIn = self._congestion.nextMessageIn(now)
else:
nextActionIn = self._congestion.nextTimeoutIn(now, what)
delayedCall = self._delayedCalls.get(what)
if delayedCall is not None and delayedCall.active():
delayedCall.reset(nextActionIn)
else:
self._delayedCalls[what] = self._clock.callLater(
nextActionIn, self._scheduledAction, what)
def _cancel(self, what):
delayedCall = self._delayedCalls.pop(what, None)
if delayedCall is not None and delayedCall.active():
delayedCall.cancel()
def _scheduledAction(self, what):
nextActionIn = None
if what == 'message':
nextActionIn = self._sendAMessage()
self._reschedule(what, nextActionIn=nextActionIn)
else:
self._sentMessages.remove(what)
if what.interval:
self._enqueue(0, what)
def _enqueue(self, priority, *data):
self._reschedule('message')
for datum in data:
if datum not in self._enqueuedMessages and datum.interval:
heapq.heappush(self._messageQueue, (priority, datum.lowerBound, datum))
self._enqueuedMessages.add(datum)
def write(self, data):
if not data:
return defer.succeed(None)
elif self._writes in ('closing', 'closed'):
return defer.fail(e.CurveCPConnectionDone(
'attempted a write after closing writes'))
d = defer.Deferred()
qds = []
lowerBound = self._sent.upper_bound() if self._sent else 0
while data:
ds = []
queueableData = data[:1024]
dataRange = halfOpen(lowerBound, lowerBound + len(queueableData))
qd = QueuedData(
IntervalSet([dataRange]), lowerBound, ds, [], [], queueableData)
self._sent.add(dataRange)
lowerBound += len(queueableData)
data = data[1024:]
qds.append(qd)
ds.append(d)
self._enqueue(1, *qds)
return d
def _peerEstablished(self):
self._protocol = self._factory.buildProtocol(self.getPeer())
self._protocol.makeConnection(self)
self._deferred.callback(self._protocol)
self._reads = 'open'
self._writes = 'open'
def _doneWritingAcked(self, when):
self._writes = 'closed'
self._checkBothResolutions()
return when
def loseConnection(self, success=True):
d = defer.Deferred()
d.addCallback(self._doneWritingAcked)
streamEnd = self._ourStreamEnd = self._sent.upper_bound() if self._sent else 0
resolution = self._ourResolution = 'success' if success else 'failure'
interval = IntervalSet([halfOpen(streamEnd, streamEnd + 1)])
self._enqueue(1, QueuedResolution(interval, streamEnd, [d], [], [], resolution))
self._writes = 'closing'
return d
