def test_mock_httpresponse_stream(self):
# Mock out a HTTP Request that does enough to make it through urllib3's
# read() and close() calls, and also exhausts and underlying file
# object.
class MockHTTPRequest(object):
self.fp = None
@asyncio.coroutine
def read(self, amt=None):
data = yield From(self.fp.read(amt))
if not data:
self.fp = None
raise Return (data)
def close(self):
self.fp = None
#bio = BytesIO(b'foo')
bio = self._fake_fp(b'foo')
fp = MockHTTPRequest()
fp.fp = bio
resp = HTTPResponse(fp, preload_content=False)
stream = yield From(resp.stream(2))
self.assertEqual(next(stream), b'fo')
self.assertEqual(next(stream), b'o')
self.assertRaises(StopIteration, next, stream)
def test_streaming(self):
fp = BytesIO(b'foo')
resp = HTTPResponse(fp, preload_content=False)
stream = yield From(resp.stream(2, decode_content=False))
self.assertEqual(next(stream), b'fo')
self.assertEqual(next(stream), b'o')
self.assertRaises(StopIteration, next, stream)
def test_empty_stream(self):
fp = self._fake_fp(b'')
#fp = BytesIO(b'')
resp = HTTPResponse(fp, preload_content=False)
stream = yield From(resp.stream(2, decode_content=False))
self.assertRaises(StopIteration, next, stream)
def test_deflate_streaming(self):
import zlib
data = zlib.compress(b'foo')
fp = self._fake_fp(data)
resp = HTTPResponse(fp, headers={'content-encoding': 'deflate'},
preload_content=False)
stream = yield From(resp.stream(2))
self.assertEqual(next(stream), b'fo')
self.assertEqual(next(stream), b'o')
self.assertRaises(StopIteration, next, stream)
def test_deflate2_streaming(self):
import zlib
compress = zlib.compressobj(6, zlib.DEFLATED, -zlib.MAX_WBITS)
data = compress.compress(b'foo')
data += compress.flush()
fp = self._fake_fp(data)
resp = HTTPResponse(fp, headers={'content-encoding': 'deflate'},
preload_content=False)
stream = yield From(resp.stream(2))
self.assertEqual(next(stream), b'fo')
self.assertEqual(next(stream), b'o')
self.assertRaises(StopIteration, next, stream)
def test_streaming_tell(self):
fp = self._fake_fp(b'foo')
#fp = BytesIO(b'foo')
resp = HTTPResponse(fp, preload_content=False)
stream = yield From(resp.stream(2, decode_content=False))
position = 0
position += len(next(stream))
self.assertEqual(2, position)
self.assertEqual(position, resp.tell())
position += len(next(stream))
self.assertEqual(3, position)
self.assertEqual(position, resp.tell())
self.assertRaises(StopIteration, next, stream)
def test_gzipped_streaming_tell(self):
import zlib
compress = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS)
uncompressed_data = b'foo'
data = compress.compress(uncompressed_data)
data += compress.flush()
#fp = BytesIO(data)
fp = self._fake_fp(data)
resp = HTTPResponse(fp, headers={'content-encoding': 'gzip'},
preload_content=False)
stream = yield From(resp.stream())
# Read everything
payload = next(stream)
self.assertEqual(payload, uncompressed_data)
self.assertEqual(len(data), resp.tell())
self.assertRaises(StopIteration, next, stream)
def tst_deflate_streaming_tell_intermediate_point(self):
# test not relevant any longer, now that 'stream' is just a cached
# set of blocks
# Ensure that ``tell()`` returns the correct number of bytes when
# part-way through streaming compressed content.
import zlib
NUMBER_OF_READS = 10
class MockCompressedDataReading(BytesIO):
"""
A ByteIO-like reader returning ``payload`` in ``NUMBER_OF_READS``
calls to ``read``.
"""
def __init__(self, payload, payload_part_size):
self.payloads = [
payload[i*payload_part_size:(i+1)*payload_part_size]
for i in range(NUMBER_OF_READS+1)]
assert b"".join(self.payloads) == payload
def read(self, _=None):
# Amount is unused.
yield None
raise Return (b''.join(self.payloads))
#if len(self.payloads) > 0:
# return self.payloads.pop(0)
#return b""
uncompressed_data = zlib.decompress(ZLIB_PAYLOAD)
payload_part_size = len(ZLIB_PAYLOAD) // NUMBER_OF_READS
fp = MockCompressedDataReading(ZLIB_PAYLOAD, payload_part_size)
resp = HTTPResponse(fp, headers={'content-encoding': 'deflate'},
preload_content=False)
stream = yield From(resp.stream(payload_part_size))
parts_positions = []
for part in stream:
_t = resp.tell()
parts_positions.append((part, _t))
end_of_stream = resp.tell()
self.assertRaises(StopIteration, next, stream)
parts, positions = zip(*parts_positions)
# Check that the payload is equal to the uncompressed data
payload = b"".join(parts)
self.assertEqual(uncompressed_data, payload)
# Check that the positions in the stream are correct
expected = [(i+1)*payload_part_size for i in range(NUMBER_OF_READS)]
self.assertEqual(expected, list(positions))
# Check that the end of the stream is in the correct place
self.assertEqual(len(ZLIB_PAYLOAD), end_of_stream)
