def pack_ranges(self):
ret = []
prev = 0
ranges = self.ranges
if ranges and ranges[0].lower_bound != 0:
ranges = [halfOpen(0, 0)] + ranges[:5]
rangeIter = iter(ranges)
remainingDelta = remainingSpan = None
for deltaPack, spanPack in self.rangePackers:
if not (remainingSpan or remainingDelta):
i = next(rangeIter, None) or halfOpen(prev, prev)
if not i.lower_closed or i.upper_closed:
raise ValueError('every interval must be half-open')
if deltaPack is None and i.lower_bound != 0:
raise ValueError('first interval must start at 0')
remainingDelta = i.lower_bound - prev
remainingSpan = i.upper_bound - i.lower_bound
prev = i.upper_bound
if deltaPack is not None:
remainingDelta = packInto(deltaPack, remainingDelta, ret)
if not remainingDelta:
remainingSpan = packInto(spanPack, remainingSpan, ret)
else:
packInto(spanPack, 0, ret)
return ''.join(ret)
def pack_ranges(self):
ret = []
prev = 0
ranges = self.ranges
if ranges and ranges[0].lower_bound != 0:
ranges = [halfOpen(0, 0)] + ranges[:5]
rangeIter = iter(ranges)
remainingDelta = remainingSpan = None
for deltaPack, spanPack in self.rangePackers:
if not (remainingSpan or remainingDelta):
i = next(rangeIter, None) or halfOpen(prev, prev)
if not i.lower_closed or i.upper_closed:
raise ValueError('every interval must be half-open')
if deltaPack is None and i.lower_bound != 0:
raise ValueError('first interval must start at 0')
remainingDelta = i.lower_bound - prev
remainingSpan = i.upper_bound - i.lower_bound
prev = i.upper_bound
if deltaPack is not None:
remainingDelta = packInto(deltaPack, remainingDelta, ret)
if not remainingDelta:
remainingSpan = packInto(spanPack, remainingSpan, ret)
else:
packInto(spanPack, 0, ret)
return ''.join(ret)
def test_closeDeferredFiresAfterSendingData(messageTransport):
t = messageTransport
t.write('hello')
t.write('world')
d = t.loseConnection()
fired = []
d.addCallback(fired.append)
assert not fired
t._clock.pump([1, 1, 1])
t._parseMessage(t._now(), Message(0, 1, [halfOpen(0, 1)], None, 0, '').pack())
assert not fired
t._parseMessage(t._now(), Message(0, 1, [halfOpen(0, 6)], None, 0, '').pack())
assert not fired
t._parseMessage(t._now(), Message(0, 1, [halfOpen(0, 11)], None, 0, '').pack())
assert fired[0] == t._now()
def test_writeDeferredFiresSendingLotsOfData(messageTransport):
t = messageTransport
d = t.write('hi' * 1023)
fired = []
d.addCallback(fired.append)
assert not fired
t._clock.pump([1, 1])
t._parseMessage(t._now(), Message(0, 1, [halfOpen(0, 2)], None, 0, '').pack())
assert not fired
t._parseMessage(t._now(), Message(0, 1, [halfOpen(0, 1024)], None, 0, '').pack())
assert not fired
t._parseMessage(t._now(), Message(0, 1, [halfOpen(0, 2045)], None, 0, '').pack())
assert not fired
t._parseMessage(t._now(), Message(0, 1, [halfOpen(0, 2046)], None, 0, '').pack())
assert fired[0] == t._now()
def test_emptyingTheMessageQueueWaitsForAWhile(messageTransport):
t = messageTransport
t.write('hi')
t._clock.pump([1, 10])
t._parseMessage(t._now(), Message(0, 1, [halfOpen(0, 2)], None, 0, '').pack())
t._clock.advance(10)
assert len(t._sendMessage.captured) == 2
assert nextCallIn(t._clock) == 60
def test_closeDeferredFiresAfterSendingData(messageTransport):
t = messageTransport
t.write('hello')
t.write('world')
d = t.loseConnection()
fired = []
d.addCallback(fired.append)
assert not fired
t._clock.pump([1, 1, 1])
t._parseMessage(t._now(),
Message(0, 1, [halfOpen(0, 1)], None, 0, '').pack())
assert not fired
t._parseMessage(t._now(),
Message(0, 1, [halfOpen(0, 6)], None, 0, '').pack())
assert not fired
t._parseMessage(t._now(),
Message(0, 1, [halfOpen(0, 11)], None, 0, '').pack())
assert fired[0] == t._now()
def test_emptyingTheMessageQueueWaitsForAWhile(messageTransport):
t = messageTransport
t.write('hi')
t._clock.pump([1, 10])
t._parseMessage(t._now(),
Message(0, 1, [halfOpen(0, 2)], None, 0, '').pack())
t._clock.advance(10)
assert len(t._sendMessage.captured) == 2
assert nextCallIn(t._clock) == 60
def test_closeDeferredFires(messageTransport):
t = messageTransport
d = t.loseConnection()
fired = []
d.addCallback(fired.append)
assert not fired
t._clock.advance(1)
t._parseMessage(t._now(), Message(0, 1, [halfOpen(0, 1)], None, 0, '').pack())
assert fired[0] == t._now()
def test_writeDeferredFires(messageTransport):
t = messageTransport
d = t.write('hi')
fired = []
d.addCallback(fired.append)
assert not fired
runUntilNext(t._clock)
t._parseMessage(t._now(), Message(0, 1, [halfOpen(0, 2)], None, 0, '').pack())
assert fired[0] == t._now()
def loseConnection(self, success=True):
d = defer.Deferred()
d.addCallback(self._doneWritingAcked)
streamEnd = self._ourStreamEnd = self._sent.upper_bound() if self._sent else 0
resolution = self._ourResolution = 'success' if success else 'failure'
interval = IntervalSet([halfOpen(streamEnd, streamEnd + 1)])
self._enqueue(1, QueuedResolution(interval, streamEnd, [d], [], [], resolution))
self._writes = 'closing'
return d
def _parseMessage(self, now, message):
message = parseMessage(message)
sentAt = self._sentMessageAt.pop(message.previousID, None)
if sentAt is not None:
self._congestion.processDelta(now, now - sentAt)
if message.id:
self._clock.callLater(0, self._sendAMessage, ack=message.id)
self._theyAcked.update(message.ranges)
for qd in list(self._sentMessages):
if qd.interval & self._theyAcked:
qd.interval.difference_update(self._theyAcked)
if not qd.interval:
for d in qd.deferreds:
d.callback(now)
self._cancel(qd)
self._outstandingMessages -= 1
self._sentMessages.remove(qd)
if message.resolution and self._theirStreamEnd is None:
self._theirStreamEnd = message.dataPos
self._theirResolution = message.resolution
self._received.add(halfOpen(message.dataPos, message.dataPos + 1))
self._reads = 'closing'
self._checkTheirResolution()
return
elif not message.data:
return
i = halfOpen(message.dataPos, message.dataPos + len(message.data))
new = IntervalSet([i]) - self._received
if not new:
return
self._received.add(i)
newData = message.data[new.lower_bound() -
i.lower_bound:new.upper_bound() -
i.upper_bound or None]
bisect.insort(self._fragment, (i.lower_bound, newData))
if len(self._received) > 1 or self._received.lower_bound() != 0:
return
newData = ''.join([d for _, d in self._fragment])
self._protocol.dataReceived(newData)
self._fragment = []
self._checkTheirResolution()
def _makeIntervals(ranges):
end = 0
ret = []
for delta, span in ranges:
start = end + delta
end = start + span
if start == end:
continue
ret.append(halfOpen(start, end))
return ret
def test_closeDeferredFires(messageTransport):
t = messageTransport
d = t.loseConnection()
fired = []
d.addCallback(fired.append)
assert not fired
t._clock.advance(1)
t._parseMessage(t._now(),
Message(0, 1, [halfOpen(0, 1)], None, 0, '').pack())
assert fired[0] == t._now()
def test_writeDeferredFires(messageTransport):
t = messageTransport
d = t.write('hi')
fired = []
d.addCallback(fired.append)
assert not fired
runUntilNext(t._clock)
t._parseMessage(t._now(),
Message(0, 1, [halfOpen(0, 2)], None, 0, '').pack())
assert fired[0] == t._now()
def test_writeDeferredFiresSendingLotsOfData(messageTransport):
t = messageTransport
d = t.write('hi' * 1023)
fired = []
d.addCallback(fired.append)
assert not fired
t._clock.pump([1, 1])
t._parseMessage(t._now(),
Message(0, 1, [halfOpen(0, 2)], None, 0, '').pack())
assert not fired
t._parseMessage(t._now(),
Message(0, 1, [halfOpen(0, 1024)], None, 0, '').pack())
assert not fired
t._parseMessage(t._now(),
Message(0, 1, [halfOpen(0, 2045)], None, 0, '').pack())
assert not fired
t._parseMessage(t._now(),
Message(0, 1, [halfOpen(0, 2046)], None, 0, '').pack())
assert fired[0] == t._now()
def _makeIntervals(ranges):
end = 0
ret = []
for delta, span in ranges:
start = end + delta
end = start + span
if start == end:
continue
ret.append(halfOpen(start, end))
return ret
def _parseMessage(self, now, message):
message = parseMessage(message)
sentAt = self._sentMessageAt.pop(message.previousID, None)
if sentAt is not None:
self._congestion.processDelta(now, now - sentAt)
if message.id:
self._clock.callLater(0, self._sendAMessage, ack=message.id)
self._theyAcked.update(message.ranges)
for qd in list(self._sentMessages):
if qd.interval & self._theyAcked:
qd.interval.difference_update(self._theyAcked)
if not qd.interval:
for d in qd.deferreds:
d.callback(now)
self._cancel(qd)
self._outstandingMessages -= 1
self._sentMessages.remove(qd)
if message.resolution and self._theirStreamEnd is None:
self._theirStreamEnd = message.dataPos
self._theirResolution = message.resolution
self._received.add(halfOpen(message.dataPos, message.dataPos + 1))
self._reads = 'closing'
self._checkTheirResolution()
return
elif not message.data:
return
i = halfOpen(message.dataPos, message.dataPos + len(message.data))
new = IntervalSet([i]) - self._received
if not new:
return
self._received.add(i)
newData = message.data[new.lower_bound() - i.lower_bound:new.upper_bound() - i.upper_bound or None]
bisect.insort(self._fragment, (i.lower_bound, newData))
if len(self._received) > 1 or self._received.lower_bound() != 0:
return
newData = ''.join([d for _, d in self._fragment])
self._protocol.dataReceived(newData)
self._fragment = []
self._checkTheirResolution()
def loseConnection(self, success=True):
d = defer.Deferred()
d.addCallback(self._doneWritingAcked)
streamEnd = self._ourStreamEnd = self._sent.upper_bound(
) if self._sent else 0
resolution = self._ourResolution = 'success' if success else 'failure'
interval = IntervalSet([halfOpen(streamEnd, streamEnd + 1)])
self._enqueue(
1, QueuedResolution(interval, streamEnd, [d], [], [], resolution))
self._writes = 'closing'
return d
def write(self, data):
if not data:
return defer.succeed(None)
elif self._writes in ('closing', 'closed'):
return defer.fail(e.CurveCPConnectionDone(
'attempted a write after closing writes'))
d = defer.Deferred()
qds = []
lowerBound = self._sent.upper_bound() if self._sent else 0
while data:
ds = []
queueableData = data[:1024]
dataRange = halfOpen(lowerBound, lowerBound + len(queueableData))
qd = QueuedData(
IntervalSet([dataRange]), lowerBound, ds, [], [], queueableData)
self._sent.add(dataRange)
lowerBound += len(queueableData)
data = data[1024:]
qds.append(qd)
ds.append(d)
self._enqueue(1, *qds)
return d
def write(self, data):
if not data:
return defer.succeed(None)
elif self._writes in ('closing', 'closed'):
return defer.fail(
e.CurveCPConnectionDone(
'attempted a write after closing writes'))
d = defer.Deferred()
qds = []
lowerBound = self._sent.upper_bound() if self._sent else 0
while data:
ds = []
queueableData = data[:1024]
dataRange = halfOpen(lowerBound, lowerBound + len(queueableData))
qd = QueuedData(IntervalSet([dataRange]), lowerBound, ds, [], [],
queueableData)
self._sent.add(dataRange)
lowerBound += len(queueableData)
data = data[1024:]
qds.append(qd)
ds.append(d)
self._enqueue(1, *qds)
return d
@pytest.mark.parametrize(('input', 'expected'), messagePackPairs)
def test_messagePack(input, expected):
assert input.pack_data().encode('hex') == expected.encode('hex')
def test_messagePackFailure():
m = Message(0, 0, [], None, 0, '\0' * 1025)
with pytest.raises(ValueError):
m.pack_data()
messagePackRangesPairs = [
([], '\0' * 30),
([halfOpen(0, 256)], '\0\1\0\0\0\0\0\0' + '\0' * 22),
([halfOpen(0, 257)], '\1\1\0\0\0\0\0\0' + '\0' * 22),
([halfOpen(0, 256),
halfOpen(512,
1024)], '\0\1\0\0\0\0\0\0' + '\0\1\0\0' + '\0\2' + '\0' * 16),
([
halfOpen(0, 18446744073709551615),
halfOpen(18446744078004518910, 18446744078004584445),
halfOpen(18446744078004649980, 18446744078004715515),
halfOpen(18446744078004781050, 18446744078004846585),
halfOpen(18446744078004912120, 18446744078004977655),
halfOpen(18446744078005043190, 18446744078005108725)
], '\xff' * 30),
# overflows
([halfOpen(0, 18446744073709551616)],
def test_ack(messageTransport):
t = messageTransport
t._parseMessage(t._now(), Message(1, 0, [], None, 0, 'hi').pack())
runUntilNext(t._clock)
assert t._sendMessage.captured[0] == Message(0, 1, [halfOpen(0, 2)], None, 0, '')
(Message(0, 0, [], 'success', 0, '\1' * 1024), '\0\14' + '\0' * 8 + '\1' * 1024),
(Message(0, 0, [], 'failure', 0, '\1' * 1024), '\0\24' + '\0' * 8 + '\1' * 1024),
]
@pytest.mark.parametrize(('input', 'expected'), messagePackPairs)
def test_messagePack(input, expected):
assert input.pack_data().encode('hex') == expected.encode('hex')
def test_messagePackFailure():
m = Message(0, 0, [], None, 0, '\0' * 1025)
with pytest.raises(ValueError):
m.pack_data()
messagePackRangesPairs = [
([], '\0' * 30),
([halfOpen(0, 256)], '\0\1\0\0\0\0\0\0' + '\0' * 22),
([halfOpen(0, 257)], '\1\1\0\0\0\0\0\0' + '\0' * 22),
([halfOpen(0, 256), halfOpen(512, 1024)], '\0\1\0\0\0\0\0\0' + '\0\1\0\0' + '\0\2' + '\0' * 16),
([
halfOpen(0, 18446744073709551615),
halfOpen(18446744078004518910, 18446744078004584445),
halfOpen(18446744078004649980, 18446744078004715515),
halfOpen(18446744078004781050, 18446744078004846585),
halfOpen(18446744078004912120, 18446744078004977655),
halfOpen(18446744078005043190, 18446744078005108725)
], '\xff' * 30),
# overflows
([halfOpen(0, 18446744073709551616)], '\xff' * 8 + '\0\0\0\0\1\0' + '\0' * 16),
([halfOpen(0, 18446744073709879290)], '\xff' * 8 + '\0\0\0\0\xff\xff' + '\0\0\xff\xff' * 4),
([halfOpen(0, 1), halfOpen(2, 65540)], '\1\0\0\0\0\0\0\0' '\1\0\0\0\xff\xff' '\0\0\3\0' + '\0' * 12),
def test_ack(messageTransport):
t = messageTransport
t._parseMessage(t._now(), Message(1, 0, [], None, 0, 'hi').pack())
runUntilNext(t._clock)
assert t._sendMessage.captured[0] == Message(0, 1, [halfOpen(0, 2)], None,
0, '')
def showMessage(tag, message, sentAt=None):
print tag, message.id, message.previousID, IntervalSet(message.ranges),
print halfOpen(message.dataPos, message.dataPos + len(message.data)),
print len(message.data), sentAt and len(sentAt)
def showMessage(tag, message, sentAt=None):
print tag, message.id, message.previousID, IntervalSet(message.ranges),
print halfOpen(message.dataPos, message.dataPos + len(message.data)),
print len(message.data), sentAt and len(sentAt)