def test_mock_httpresponse_stream(self): # Mock out a HTTP Request that does enough to make it through urllib3's # read() and close() calls, and also exhausts and underlying file # object. class MockHTTPRequest(object): self.fp = None @asyncio.coroutine def read(self, amt=None): data = yield From(self.fp.read(amt)) if not data: self.fp = None raise Return (data) def close(self): self.fp = None #bio = BytesIO(b'foo') bio = self._fake_fp(b'foo') fp = MockHTTPRequest() fp.fp = bio resp = HTTPResponse(fp, preload_content=False) stream = yield From(resp.stream(2)) self.assertEqual(next(stream), b'fo') self.assertEqual(next(stream), b'o') self.assertRaises(StopIteration, next, stream)
def test_streaming(self): fp = BytesIO(b'foo') resp = HTTPResponse(fp, preload_content=False) stream = yield From(resp.stream(2, decode_content=False)) self.assertEqual(next(stream), b'fo') self.assertEqual(next(stream), b'o') self.assertRaises(StopIteration, next, stream)
def test_empty_stream(self): fp = self._fake_fp(b'') #fp = BytesIO(b'') resp = HTTPResponse(fp, preload_content=False) stream = yield From(resp.stream(2, decode_content=False)) self.assertRaises(StopIteration, next, stream)
def test_deflate_streaming(self): import zlib data = zlib.compress(b'foo') fp = self._fake_fp(data) resp = HTTPResponse(fp, headers={'content-encoding': 'deflate'}, preload_content=False) stream = yield From(resp.stream(2)) self.assertEqual(next(stream), b'fo') self.assertEqual(next(stream), b'o') self.assertRaises(StopIteration, next, stream)
def test_deflate2_streaming(self): import zlib compress = zlib.compressobj(6, zlib.DEFLATED, -zlib.MAX_WBITS) data = compress.compress(b'foo') data += compress.flush() fp = self._fake_fp(data) resp = HTTPResponse(fp, headers={'content-encoding': 'deflate'}, preload_content=False) stream = yield From(resp.stream(2)) self.assertEqual(next(stream), b'fo') self.assertEqual(next(stream), b'o') self.assertRaises(StopIteration, next, stream)
def test_streaming_tell(self): fp = self._fake_fp(b'foo') #fp = BytesIO(b'foo') resp = HTTPResponse(fp, preload_content=False) stream = yield From(resp.stream(2, decode_content=False)) position = 0 position += len(next(stream)) self.assertEqual(2, position) self.assertEqual(position, resp.tell()) position += len(next(stream)) self.assertEqual(3, position) self.assertEqual(position, resp.tell()) self.assertRaises(StopIteration, next, stream)
def test_gzipped_streaming_tell(self): import zlib compress = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS) uncompressed_data = b'foo' data = compress.compress(uncompressed_data) data += compress.flush() #fp = BytesIO(data) fp = self._fake_fp(data) resp = HTTPResponse(fp, headers={'content-encoding': 'gzip'}, preload_content=False) stream = yield From(resp.stream()) # Read everything payload = next(stream) self.assertEqual(payload, uncompressed_data) self.assertEqual(len(data), resp.tell()) self.assertRaises(StopIteration, next, stream)
def tst_deflate_streaming_tell_intermediate_point(self): # test not relevant any longer, now that 'stream' is just a cached # set of blocks # Ensure that ``tell()`` returns the correct number of bytes when # part-way through streaming compressed content. import zlib NUMBER_OF_READS = 10 class MockCompressedDataReading(BytesIO): """ A ByteIO-like reader returning ``payload`` in ``NUMBER_OF_READS`` calls to ``read``. """ def __init__(self, payload, payload_part_size): self.payloads = [ payload[i*payload_part_size:(i+1)*payload_part_size] for i in range(NUMBER_OF_READS+1)] assert b"".join(self.payloads) == payload def read(self, _=None): # Amount is unused. yield None raise Return (b''.join(self.payloads)) #if len(self.payloads) > 0: # return self.payloads.pop(0) #return b"" uncompressed_data = zlib.decompress(ZLIB_PAYLOAD) payload_part_size = len(ZLIB_PAYLOAD) // NUMBER_OF_READS fp = MockCompressedDataReading(ZLIB_PAYLOAD, payload_part_size) resp = HTTPResponse(fp, headers={'content-encoding': 'deflate'}, preload_content=False) stream = yield From(resp.stream(payload_part_size)) parts_positions = [] for part in stream: _t = resp.tell() parts_positions.append((part, _t)) end_of_stream = resp.tell() self.assertRaises(StopIteration, next, stream) parts, positions = zip(*parts_positions) # Check that the payload is equal to the uncompressed data payload = b"".join(parts) self.assertEqual(uncompressed_data, payload) # Check that the positions in the stream are correct expected = [(i+1)*payload_part_size for i in range(NUMBER_OF_READS)] self.assertEqual(expected, list(positions)) # Check that the end of the stream is in the correct place self.assertEqual(len(ZLIB_PAYLOAD), end_of_stream)