async def __norm_callback(
response: aiohttp.ClientResponse,
decode: bool = False,
max_size: Optional[int] = None,
intended_content_type: Optional[str] = None) -> Optional[AnyStr]:
content_type = response.headers.get('Content-Type')
if not intended_content_type or not content_type or content_type.startswith(
intended_content_type):
body: Optional[bytes] = None
if max_size is None:
body = await response.read()
elif max_size > 0:
body = await response.content.read(max_size)
if decode and body:
xml_header = body.split(b'\n', 1)[0]
if xml_header.startswith(
b'<?xml'
) and b'?>' in xml_header and b'encoding' in xml_header:
try:
encoding = BeautifulSoup(xml_header,
'lxml-xml').original_encoding
return body.decode(encoding=encoding, errors='replace')
except (LookupError, RuntimeError):
pass
try:
encoding = response.get_encoding()
return body.decode(encoding=encoding, errors='replace')
except (LookupError, RuntimeError):
return body.decode(encoding='utf-8', errors='replace')
return body
return None
async def from_client_response(cls,
client_response: ClientResponse,
expires: datetime = None):
"""Convert a ClientResponse into a CachedReponse"""
# Response may not have been read yet, if fetched by something other than CachedSession
if not client_response._released:
await client_response.read()
# Copy most attributes over as is
copy_attrs = set(attr.fields_dict(cls).keys()) - EXCLUDE_ATTRS
response = cls(**{k: getattr(client_response, k) for k in copy_attrs})
# Set some remaining attributes individually
response._body = client_response._body
response._links = [(k, _to_str_tuples(v))
for k, v in client_response.links.items()]
response.expires = expires
response.real_url = client_response.request_info.real_url
# The encoding may be unset even if the response has been read
try:
response.encoding = client_response.get_encoding()
except RuntimeError:
pass
response.url = str(client_response.url)
if client_response.history:
response.history = (*[
await cls.from_client_response(r)
for r in client_response.history
], )
return response
async def from_client_response(cls,
client_response: ClientResponse,
expires: datetime = None):
"""Convert a ClientResponse into a CachedReponse"""
if isinstance(client_response, cls):
return client_response
# Copy most attributes over as is
copy_attrs = set(attr.fields_dict(cls).keys()) - EXCLUDE_ATTRS
response = cls(**{k: getattr(client_response, k) for k in copy_attrs})
# Read response content, and reset StreamReader on original response
if not client_response._released:
await client_response.read()
response._body = client_response._body
client_response.content = CachedStreamReader(client_response._body)
# Set remaining attributes individually
response.expires = expires
response.links = client_response.links
response.real_url = client_response.request_info.real_url
# The encoding may be unset even if the response has been read, and
# get_encoding() does not handle certain edge cases like an empty response body
try:
response.encoding = client_response.get_encoding()
except (RuntimeError, TypeError):
pass
if client_response.history:
response.history = (*[
await cls.from_client_response(r)
for r in client_response.history
], )
return response
async def wrap_async(response: ClientResponse) -> Response:
"""Build a ``requests`` response from a ``aiohttp`` response.
A ``requests.Response`` instance is built to provide synchronous
access to the original response's data. Note that the returned
response does not have proper data for :attr:``elapsed`` or
:attr:``request``.
The response will be consumed if it has not already.
"""
# Ensure the response data is read so that the wrapped response
# does not require any async methods.
await response.read()
wrapped = Response()
wrapped._content = response._body # type: ignore
wrapped._content_consumed = True # type: ignore
wrapped.status_code = response.status
wrapped.headers = CaseInsensitiveDict(response.headers)
wrapped.url = str(response.url) # `aiohttp` uses a `URL` object.
wrapped.encoding = response.get_encoding()
wrapped.history = [await wrap_async(rsp) for rsp in response.history]
wrapped.reason = response.reason or ""
wrapped.cookies = cookiejar_from_dict(response.cookies)
return wrapped
async def from_client_response(cls, client_response: ClientResponse):
# Response may not have been read yet, if fetched by something other than CachedSession
if not client_response._released:
await client_response.read()
# Copy most attributes over as is
copy_attrs = set(attr.fields_dict(cls).keys()) - EXCLUDE_ATTRS
response = cls(**{k: getattr(client_response, k) for k in copy_attrs})
# Set some remaining attributes individually
response._body = client_response._body
response.headers = dict(client_response.headers)
response.encoding = client_response.get_encoding()
response.request_info = RequestInfo.from_object(client_response.request_info)
response.url = str(client_response.url)
if client_response.history:
response.history = (cls.from_client_response(r) for r in client_response.history)
return response
async def _handle_stream_response(response: aiohttp.ClientResponse,
chunk_size=500) -> AsyncIterable:
"""
Helper method that do next things:
1) Async iterating over response content by chunks
2) Decode bytes to text
3) Build string lines from text stream
:param response:
:param chunk_size:
:return:
"""
if 400 <= response.status < 600:
raise WrongHttpStatusCode(response.status)
encoding = response.get_encoding()
pending = None
async for bytes_chunk in response.content.iter_chunked(chunk_size):
chunk = bytes_chunk.decode(encoding)
if pending is not None:
chunk = pending + chunk
lines = chunk.splitlines()
if lines and lines[-1] and chunk and lines[-1][-1] == chunk[-1]:
pending = lines.pop()
else:
pending = None
for line in lines:
yield line
if pending is not None:
yield pending
