def iter_raw(r: requests.Response, chunk_size: int = 1):
"""
Reimplementation of requests.Response.iter_content that doesn't try
to decode zipped content
:param chunk_size:
:type r: requests.Response
"""
def generate():
while True:
# urllib3.response.HTTPResponse.read
chunk = r.raw.read(amt=chunk_size, decode_content=False)
if not chunk:
break
log_item("Type of chunk", type(chunk))
yield chunk
r._content_consumed = True
# noinspection PyProtectedMember
if r._content_consumed and isinstance(r._content, bool):
raise StreamConsumedError()
elif chunk_size is not None and not isinstance(chunk_size, int):
raise TypeError("chunk_size must be an int, it is instead a %s." %
type(chunk_size))
# noinspection PyProtectedMember
# simulate reading small chunks of the content
reused_chunks = iter_slices(r._content, chunk_size)
stream_chunks = generate()
# noinspection PyProtectedMember
chunks = reused_chunks if r._content_consumed else stream_chunks
return chunks
def iter_content(self, chunk_size=1, decode_unicode=False):
"""Iterates over the response data. When stream=True is set on the
request, this avoids reading the content at once into memory for
large responses. The chunk size is the number of bytes it should
read into memory. This is not necessarily the length of each item
returned as decoding can take place.
chunk_size must be of type int or None. A value of None will
function differently depending on the value of `stream`.
stream=True will read data as it arrives in whatever size the
chunks are received. If stream=False, data is returned as
a single chunk.
If decode_unicode is True, content will be decoded using the best
available encoding based on the response.
"""
def generate():
# Special case for google app engine.
if hasattr(self.raw, 'stream'):
try:
if isinstance(self.raw._original_response._method, int):
while True:
chunk = self.raw.read(chunk_size, decode_content=True)
if not chunk:
break
yield chunk
except ProtocolError as e:
raise ChunkedEncodingError(e)
except DecodeError as e:
raise ContentDecodingError(e)
except ReadTimeoutError as e:
raise ConnectionError(e)
else:
# Standard file-like object.
while True:
chunk = self.raw.read(chunk_size)
if not chunk:
break
yield chunk
self._content_consumed = True
if self._content_consumed and isinstance(self._content, bool):
raise StreamConsumedError()
elif chunk_size is not None and not isinstance(chunk_size, int):
raise TypeError("chunk_size must be an int, it is instead a %s." %
type(chunk_size))
# simulate reading small chunks of the content
reused_chunks = iter_slices(self._content, chunk_size)
stream_chunks = generate()
chunks = reused_chunks if self._content_consumed else stream_chunks
if decode_unicode:
chunks = stream_decode_response_unicode(chunks, self)
return chunks
async def iter_content(self, chunk_size=1, decode_unicode=False):
if self._content_consumed and isinstance(self._content, bool):
raise StreamConsumedError()
elif chunk_size is not None and not isinstance(chunk_size, int):
raise TypeError(
"chunk_size must be an int, it is instead a %s." % type(chunk_size)
)
# simulate reading small chunks of the content
reused_chunks = iter_slices(self._content, chunk_size)
stream_chunks = self.generate(chunk_size)
chunks = reused_chunks if self._content_consumed else stream_chunks
if decode_unicode:
chunks = stream_decode_response_unicode(chunks, self)
async for c in chunks:
yield c
def iter_content(self, chunk_size=1, decode_unicode=False):
"""rewrite requests function, set decode_content with False"""
def generate():
# Special case for urllib3.
if hasattr(self.raw, 'stream'):
try:
for chunk in self.raw.stream(chunk_size, decode_content=False):
yield chunk
except ProtocolError as e:
raise ChunkedEncodingError(e)
except DecodeError as e:
raise ContentDecodingError(e)
except ReadTimeoutError as e:
raise ConnectionError(e)
else:
# Standard file-like object.
while True:
chunk = self.raw.read(chunk_size)
if not chunk:
break
yield chunk
self._content_consumed = True
if self._content_consumed and isinstance(self._content, bool):
raise StreamConsumedError()
elif chunk_size is not None and not isinstance(chunk_size, int):
raise TypeError("chunk_size must be an int, it is instead a %s." %
type(chunk_size))
# simulate reading small chunks of the content
reused_chunks = iter_slices(self._content, chunk_size)
stream_chunks = generate()
chunks = reused_chunks if self._content_consumed else stream_chunks
if decode_unicode:
chunks = stream_decode_response_unicode(chunks, self)
return chunks
def test_iter_slices(value, length):
if length is None or (length <= 0 and len(value) > 0):
# Reads all content at once
assert len(list(iter_slices(value, length))) == 1
else:
assert len(list(iter_slices(value, 1))) == length
def test_iter_slices(value, length):
if length is None or (length <= 0 and len(value) > 0):
# Reads all content at once
assert len(list(iter_slices(value, length))) == 1
else:
assert len(list(iter_slices(value, 1))) == length
def iter_content(self, chunk_size=1, decode_unicode=False):
"""Iterates over the response data. When stream=True is set on the
request, this avoids reading the content at once into memory for
large responses. The chunk size is the number of bytes it should
read into memory. This is not necessarily the length of each item
returned as decoding can take place.
chunk_size must be of type int or None. A value of None will
function differently depending on the value of `stream`.
stream=True will read data as it arrives in whatever size the
chunks are received. If stream=False, data is returned as
a single chunk.
If decode_unicode is True, content will be decoded using the best
available encoding based on the response.
"""
def generate():
decode = decode_unicode
if self.encoding is None:
decode = False
if decode:
decoder = codecs.getincrementaldecoder(
self.encoding)(errors='replace')
if self.raw.stream:
content_remain = {'': ''}
while content_remain:
future = Future()
def callback(status):
chunk = self.raw.body.getvalue()
self.raw.body.truncate(0)
self.raw.body.seek(0)
if decode:
chunk = decoder.decode(chunk)
if not status:
content_remain.clear()
future.set_result(chunk)
self.raw.connection.read_stream_body(self.raw,
chunk_size,
callback=callback)
yield future
while not future.done():
yield future
else:
if self.raw.body:
self.raw.body.seek(0)
while True:
chunk = self.raw.body.read(chunk_size)
if decode:
chunk = decoder.decode(chunk)
if not chunk:
break
else:
yield chunk
self._content_consumed = True
if self._content_consumed and isinstance(self._content, bool):
raise StreamConsumedError()
elif chunk_size is not None and not isinstance(chunk_size, int):
raise TypeError('chunk_size must be an int, it is instead a %s.' %
type(chunk_size))
elif not isinstance(self.raw, HTTPMessageDelegate):
raise TypeError('self.raw must be a trip.adapters.MessageDelegate')
if self._content_consumed:
# simulate reading small chunks of the content
if self.raw.stream:
return iter_slices_future(self, chunk_size, decode_unicode)
else:
return iter_slices(self._content, chunk_size)
else:
return generate()
def test_iter_slices(value, length):
assert len(list(iter_slices(value, 1))) == length
def test_iter_slices(value, length):
assert len(list(iter_slices(value, 1))) == length
