def iter_raw(r: requests.Response, chunk_size: int = 1): """ Reimplementation of requests.Response.iter_content that doesn't try to decode zipped content :param chunk_size: :type r: requests.Response """ def generate(): while True: # urllib3.response.HTTPResponse.read chunk = r.raw.read(amt=chunk_size, decode_content=False) if not chunk: break log_item("Type of chunk", type(chunk)) yield chunk r._content_consumed = True # noinspection PyProtectedMember if r._content_consumed and isinstance(r._content, bool): raise StreamConsumedError() elif chunk_size is not None and not isinstance(chunk_size, int): raise TypeError("chunk_size must be an int, it is instead a %s." % type(chunk_size)) # noinspection PyProtectedMember # simulate reading small chunks of the content reused_chunks = iter_slices(r._content, chunk_size) stream_chunks = generate() # noinspection PyProtectedMember chunks = reused_chunks if r._content_consumed else stream_chunks return chunks
def iter_content(self, chunk_size=1, decode_unicode=False): """Iterates over the response data. When stream=True is set on the request, this avoids reading the content at once into memory for large responses. The chunk size is the number of bytes it should read into memory. This is not necessarily the length of each item returned as decoding can take place. chunk_size must be of type int or None. A value of None will function differently depending on the value of `stream`. stream=True will read data as it arrives in whatever size the chunks are received. If stream=False, data is returned as a single chunk. If decode_unicode is True, content will be decoded using the best available encoding based on the response. """ def generate(): # Special case for google app engine. if hasattr(self.raw, 'stream'): try: if isinstance(self.raw._original_response._method, int): while True: chunk = self.raw.read(chunk_size, decode_content=True) if not chunk: break yield chunk except ProtocolError as e: raise ChunkedEncodingError(e) except DecodeError as e: raise ContentDecodingError(e) except ReadTimeoutError as e: raise ConnectionError(e) else: # Standard file-like object. while True: chunk = self.raw.read(chunk_size) if not chunk: break yield chunk self._content_consumed = True if self._content_consumed and isinstance(self._content, bool): raise StreamConsumedError() elif chunk_size is not None and not isinstance(chunk_size, int): raise TypeError("chunk_size must be an int, it is instead a %s." % type(chunk_size)) # simulate reading small chunks of the content reused_chunks = iter_slices(self._content, chunk_size) stream_chunks = generate() chunks = reused_chunks if self._content_consumed else stream_chunks if decode_unicode: chunks = stream_decode_response_unicode(chunks, self) return chunks
async def iter_content(self, chunk_size=1, decode_unicode=False): if self._content_consumed and isinstance(self._content, bool): raise StreamConsumedError() elif chunk_size is not None and not isinstance(chunk_size, int): raise TypeError( "chunk_size must be an int, it is instead a %s." % type(chunk_size) ) # simulate reading small chunks of the content reused_chunks = iter_slices(self._content, chunk_size) stream_chunks = self.generate(chunk_size) chunks = reused_chunks if self._content_consumed else stream_chunks if decode_unicode: chunks = stream_decode_response_unicode(chunks, self) async for c in chunks: yield c
def iter_content(self, chunk_size=1, decode_unicode=False): """rewrite requests function, set decode_content with False""" def generate(): # Special case for urllib3. if hasattr(self.raw, 'stream'): try: for chunk in self.raw.stream(chunk_size, decode_content=False): yield chunk except ProtocolError as e: raise ChunkedEncodingError(e) except DecodeError as e: raise ContentDecodingError(e) except ReadTimeoutError as e: raise ConnectionError(e) else: # Standard file-like object. while True: chunk = self.raw.read(chunk_size) if not chunk: break yield chunk self._content_consumed = True if self._content_consumed and isinstance(self._content, bool): raise StreamConsumedError() elif chunk_size is not None and not isinstance(chunk_size, int): raise TypeError("chunk_size must be an int, it is instead a %s." % type(chunk_size)) # simulate reading small chunks of the content reused_chunks = iter_slices(self._content, chunk_size) stream_chunks = generate() chunks = reused_chunks if self._content_consumed else stream_chunks if decode_unicode: chunks = stream_decode_response_unicode(chunks, self) return chunks
def test_iter_slices(value, length): if length is None or (length <= 0 and len(value) > 0): # Reads all content at once assert len(list(iter_slices(value, length))) == 1 else: assert len(list(iter_slices(value, 1))) == length
def iter_content(self, chunk_size=1, decode_unicode=False): """Iterates over the response data. When stream=True is set on the request, this avoids reading the content at once into memory for large responses. The chunk size is the number of bytes it should read into memory. This is not necessarily the length of each item returned as decoding can take place. chunk_size must be of type int or None. A value of None will function differently depending on the value of `stream`. stream=True will read data as it arrives in whatever size the chunks are received. If stream=False, data is returned as a single chunk. If decode_unicode is True, content will be decoded using the best available encoding based on the response. """ def generate(): decode = decode_unicode if self.encoding is None: decode = False if decode: decoder = codecs.getincrementaldecoder( self.encoding)(errors='replace') if self.raw.stream: content_remain = {'': ''} while content_remain: future = Future() def callback(status): chunk = self.raw.body.getvalue() self.raw.body.truncate(0) self.raw.body.seek(0) if decode: chunk = decoder.decode(chunk) if not status: content_remain.clear() future.set_result(chunk) self.raw.connection.read_stream_body(self.raw, chunk_size, callback=callback) yield future while not future.done(): yield future else: if self.raw.body: self.raw.body.seek(0) while True: chunk = self.raw.body.read(chunk_size) if decode: chunk = decoder.decode(chunk) if not chunk: break else: yield chunk self._content_consumed = True if self._content_consumed and isinstance(self._content, bool): raise StreamConsumedError() elif chunk_size is not None and not isinstance(chunk_size, int): raise TypeError('chunk_size must be an int, it is instead a %s.' % type(chunk_size)) elif not isinstance(self.raw, HTTPMessageDelegate): raise TypeError('self.raw must be a trip.adapters.MessageDelegate') if self._content_consumed: # simulate reading small chunks of the content if self.raw.stream: return iter_slices_future(self, chunk_size, decode_unicode) else: return iter_slices(self._content, chunk_size) else: return generate()
def test_iter_slices(value, length): assert len(list(iter_slices(value, 1))) == length