async def __norm_callback( response: aiohttp.ClientResponse, decode: bool = False, max_size: Optional[int] = None, intended_content_type: Optional[str] = None) -> Optional[AnyStr]: content_type = response.headers.get('Content-Type') if not intended_content_type or not content_type or content_type.startswith( intended_content_type): body: Optional[bytes] = None if max_size is None: body = await response.read() elif max_size > 0: body = await response.content.read(max_size) if decode and body: xml_header = body.split(b'\n', 1)[0] if xml_header.startswith( b'<?xml' ) and b'?>' in xml_header and b'encoding' in xml_header: try: encoding = BeautifulSoup(xml_header, 'lxml-xml').original_encoding return body.decode(encoding=encoding, errors='replace') except (LookupError, RuntimeError): pass try: encoding = response.get_encoding() return body.decode(encoding=encoding, errors='replace') except (LookupError, RuntimeError): return body.decode(encoding='utf-8', errors='replace') return body return None
async def from_client_response(cls, client_response: ClientResponse, expires: datetime = None): """Convert a ClientResponse into a CachedReponse""" # Response may not have been read yet, if fetched by something other than CachedSession if not client_response._released: await client_response.read() # Copy most attributes over as is copy_attrs = set(attr.fields_dict(cls).keys()) - EXCLUDE_ATTRS response = cls(**{k: getattr(client_response, k) for k in copy_attrs}) # Set some remaining attributes individually response._body = client_response._body response._links = [(k, _to_str_tuples(v)) for k, v in client_response.links.items()] response.expires = expires response.real_url = client_response.request_info.real_url # The encoding may be unset even if the response has been read try: response.encoding = client_response.get_encoding() except RuntimeError: pass response.url = str(client_response.url) if client_response.history: response.history = (*[ await cls.from_client_response(r) for r in client_response.history ], ) return response
async def from_client_response(cls, client_response: ClientResponse, expires: datetime = None): """Convert a ClientResponse into a CachedReponse""" if isinstance(client_response, cls): return client_response # Copy most attributes over as is copy_attrs = set(attr.fields_dict(cls).keys()) - EXCLUDE_ATTRS response = cls(**{k: getattr(client_response, k) for k in copy_attrs}) # Read response content, and reset StreamReader on original response if not client_response._released: await client_response.read() response._body = client_response._body client_response.content = CachedStreamReader(client_response._body) # Set remaining attributes individually response.expires = expires response.links = client_response.links response.real_url = client_response.request_info.real_url # The encoding may be unset even if the response has been read, and # get_encoding() does not handle certain edge cases like an empty response body try: response.encoding = client_response.get_encoding() except (RuntimeError, TypeError): pass if client_response.history: response.history = (*[ await cls.from_client_response(r) for r in client_response.history ], ) return response
async def wrap_async(response: ClientResponse) -> Response: """Build a ``requests`` response from a ``aiohttp`` response. A ``requests.Response`` instance is built to provide synchronous access to the original response's data. Note that the returned response does not have proper data for :attr:``elapsed`` or :attr:``request``. The response will be consumed if it has not already. """ # Ensure the response data is read so that the wrapped response # does not require any async methods. await response.read() wrapped = Response() wrapped._content = response._body # type: ignore wrapped._content_consumed = True # type: ignore wrapped.status_code = response.status wrapped.headers = CaseInsensitiveDict(response.headers) wrapped.url = str(response.url) # `aiohttp` uses a `URL` object. wrapped.encoding = response.get_encoding() wrapped.history = [await wrap_async(rsp) for rsp in response.history] wrapped.reason = response.reason or "" wrapped.cookies = cookiejar_from_dict(response.cookies) return wrapped
async def from_client_response(cls, client_response: ClientResponse): # Response may not have been read yet, if fetched by something other than CachedSession if not client_response._released: await client_response.read() # Copy most attributes over as is copy_attrs = set(attr.fields_dict(cls).keys()) - EXCLUDE_ATTRS response = cls(**{k: getattr(client_response, k) for k in copy_attrs}) # Set some remaining attributes individually response._body = client_response._body response.headers = dict(client_response.headers) response.encoding = client_response.get_encoding() response.request_info = RequestInfo.from_object(client_response.request_info) response.url = str(client_response.url) if client_response.history: response.history = (cls.from_client_response(r) for r in client_response.history) return response
async def _handle_stream_response(response: aiohttp.ClientResponse, chunk_size=500) -> AsyncIterable: """ Helper method that do next things: 1) Async iterating over response content by chunks 2) Decode bytes to text 3) Build string lines from text stream :param response: :param chunk_size: :return: """ if 400 <= response.status < 600: raise WrongHttpStatusCode(response.status) encoding = response.get_encoding() pending = None async for bytes_chunk in response.content.iter_chunked(chunk_size): chunk = bytes_chunk.decode(encoding) if pending is not None: chunk = pending + chunk lines = chunk.splitlines() if lines and lines[-1] and chunk and lines[-1][-1] == chunk[-1]: pending = lines.pop() else: pending = None for line in lines: yield line if pending is not None: yield pending