class ReverseProxyResponse(aiohttp.client.ClientResponse): """ aiohttp's default response class decompresses the response body on the fly; a reverse proxy wants to return it to its client verbatim. so this class exists in order that the payload parser instantiated in this method can be instantiated without the decompression feature. it isn't otherwise interesting. """ async def start(self, connection, read_until_eof=False): """Start response processing.""" self._setup_connection(connection) message = None while True: httpstream = self._reader.set_parser(self._response_parser) # read response message = await httpstream.read() if message.code != 100: break if self._continue is not None and not self._continue.done(): self._continue.set_result(True) self._continue = None # response status self.version = message.version self.status = message.code self.reason = message.reason self._should_close = message.should_close # headers self.headers = CIMultiDictProxy(message.headers) # payload response_with_body = self._need_parse_response_body() self._reader.set_parser( aiohttp.HttpPayloadParser( message, compression=False, readall=read_until_eof, response_with_body=response_with_body), self.content ) # cookies self.cookies = http.cookies.SimpleCookie() if aiohttp.client.hdrs.SET_COOKIE in self.headers: for hdr in self.headers.getall(aiohttp.client.hdrs.SET_COOKIE): try: self.cookies.load(hdr) except http.cookies.CookieError as exc: aiohttp.log.client_logger.warning( 'Can not load response cookies: %s', exc) return self
class ClientResponse(HeadersMixin): # from the Status-Line of the response version = None # HTTP-Version status = None # Status-Code reason = None # Reason-Phrase cookies = None # Response cookies (Set-Cookie) content = None # Payload stream headers = None # Response headers, CIMultiDictProxy raw_headers = None # Response raw headers, a sequence of pairs _connection = None # current connection flow_control_class = FlowControlStreamReader # reader flow control _reader = None # input stream _response_parser = aiohttp.HttpResponseParser() _source_traceback = None # setted up by ClientRequest after ClientResponse object creation # post-init stage allows to not change ctor signature _loop = None _closed = True # to allow __del__ for non-initialized properly response def __init__(self, method, url, *, writer=None, continue100=None, timeout=5*60): assert isinstance(url, URL) self.method = method self._url_obj = url self._content = None self._writer = writer self._continue = continue100 self._closed = False self._should_close = True # override by message.should_close later self._history = () self._timeout = timeout @property def url_obj(self): return self._url_obj @property def url(self): return str(self._url_obj) @property def host(self): warnings.warn("Deprecated, use .url_obj.host", DeprecationWarning, stacklevel=2) return self._url_obj.host def _post_init(self, loop): self._loop = loop if loop.get_debug(): self._source_traceback = traceback.extract_stack(sys._getframe(1)) def __del__(self, _warnings=warnings): if self._loop is None: return # not started if self._closed: return self.close() _warnings.warn("Unclosed response {!r}".format(self), ResourceWarning) context = {'client_response': self, 'message': 'Unclosed response'} if self._source_traceback: context['source_traceback'] = self._source_traceback self._loop.call_exception_handler(context) def __repr__(self): out = io.StringIO() ascii_encodable_url = str(self.url) if self.reason: ascii_encodable_reason = self.reason.encode('ascii', 'backslashreplace') \ .decode('ascii') else: ascii_encodable_reason = self.reason print('<ClientResponse({}) [{} {}]>'.format( ascii_encodable_url, self.status, ascii_encodable_reason), file=out) print(self.headers, file=out) return out.getvalue() @property def connection(self): return self._connection @property def history(self): """A sequence of of responses, if redirects occurred.""" return self._history def waiting_for_continue(self): return self._continue is not None def _setup_connection(self, connection): self._reader = connection.reader self._connection = connection self.content = self.flow_control_class( connection.reader, loop=connection.loop, timeout=self._timeout) def _need_parse_response_body(self): return (self.method.lower() != 'head' and self.status not in [204, 304]) @asyncio.coroutine def start(self, connection, read_until_eof=False): """Start response processing.""" self._setup_connection(connection) while True: httpstream = self._reader.set_parser(self._response_parser) # read response with Timeout(self._timeout, loop=self._loop): message = yield from httpstream.read() if message.code != 100: break if self._continue is not None and not self._continue.done(): self._continue.set_result(True) self._continue = None # response status self.version = message.version self.status = message.code self.reason = message.reason self._should_close = message.should_close # headers self.headers = CIMultiDictProxy(message.headers) self.raw_headers = tuple(message.raw_headers) # payload rwb = self._need_parse_response_body() self._reader.set_parser( aiohttp.HttpPayloadParser(message, readall=read_until_eof, response_with_body=rwb), self.content) # cookies self.cookies = http.cookies.SimpleCookie() if hdrs.SET_COOKIE in self.headers: for hdr in self.headers.getall(hdrs.SET_COOKIE): try: self.cookies.load(hdr) except http.cookies.CookieError as exc: client_logger.warning( 'Can not load response cookies: %s', exc) return self def close(self): if self._closed: return self._closed = True if self._loop is None or self._loop.is_closed(): return if self._connection is not None: self._connection.close() self._connection = None self._cleanup_writer() self._notify_content() @asyncio.coroutine def release(self): if self._closed: return try: content = self.content if content is not None and not content.at_eof(): chunk = yield from content.readany() while chunk is not EOF_MARKER or chunk: chunk = yield from content.readany() except Exception: self._connection.close() self._connection = None raise finally: self._closed = True if self._connection is not None: self._connection.release() if self._reader is not None: self._reader.unset_parser() self._connection = None self._cleanup_writer() self._notify_content() def raise_for_status(self): if 400 <= self.status: raise aiohttp.HttpProcessingError( code=self.status, message=self.reason) def _cleanup_writer(self): if self._writer is not None and not self._writer.done(): self._writer.cancel() self._writer = None def _notify_content(self): content = self.content if content and content.exception() is None and not content.is_eof(): content.set_exception( aiohttp.ClientDisconnectedError('Connection closed')) @asyncio.coroutine def wait_for_close(self): if self._writer is not None: try: yield from self._writer finally: self._writer = None yield from self.release() @asyncio.coroutine def read(self): """Read response payload.""" if self._content is None: try: self._content = yield from self.content.read() except: self.close() raise else: yield from self.release() return self._content def _get_encoding(self): ctype = self.headers.get(hdrs.CONTENT_TYPE, '').lower() mtype, stype, _, params = helpers.parse_mimetype(ctype) encoding = params.get('charset') if not encoding: if mtype == 'application' and stype == 'json': # RFC 7159 states that the default encoding is UTF-8. encoding = 'utf-8' else: encoding = chardet.detect(self._content)['encoding'] if not encoding: encoding = 'utf-8' return encoding @asyncio.coroutine def text(self, encoding=None): """Read response payload and decode.""" if self._content is None: yield from self.read() if encoding is None: encoding = self._get_encoding() return self._content.decode(encoding) @asyncio.coroutine def json(self, *, encoding=None, loads=json.loads): """Read and decodes JSON response.""" if self._content is None: yield from self.read() ctype = self.headers.get(hdrs.CONTENT_TYPE, '').lower() if 'json' not in ctype: client_logger.warning( 'Attempt to decode JSON with unexpected mimetype: %s', ctype) stripped = self._content.strip() if not stripped: return None if encoding is None: encoding = self._get_encoding() return loads(stripped.decode(encoding)) if PY_35: @asyncio.coroutine def __aenter__(self): return self @asyncio.coroutine def __aexit__(self, exc_type, exc_val, exc_tb): if exc_type is None: yield from self.release() else: self.close()
class ClientResponse: # from the Status-Line of the response version = None # HTTP-Version status = None # Status-Code reason = None # Reason-Phrase cookies = None # Response cookies (Set-Cookie) content = None # Payload stream headers = None # Response headers, CIMultiDictProxy raw_headers = None # Response raw headers, a sequence of pairs _connection = None # current connection flow_control_class = FlowControlStreamReader # reader flow control _reader = None # input stream _response_parser = aiohttp.HttpResponseParser() _source_traceback = None # setted up by ClientRequest after ClientResponse object creation # post-init stage allows to not change ctor signature _loop = None _closed = True # to allow __del__ for non-initialized properly response def __init__(self, method, url, host='', *, writer=None, continue100=None, timeout=5 * 60): super().__init__() self.method = method self.url = url self.host = host self._content = None self._writer = writer self._continue = continue100 self._closed = False self._should_close = True # override by message.should_close later self._history = () self._timeout = timeout def _post_init(self, loop): self._loop = loop if loop.get_debug(): self._source_traceback = traceback.extract_stack(sys._getframe(1)) def __del__(self, _warnings=warnings): if self._loop is None: return # not started if self._closed: return self.close() _warnings.warn("Unclosed response {!r}".format(self), ResourceWarning) context = {'client_response': self, 'message': 'Unclosed response'} if self._source_traceback: context['source_traceback'] = self._source_traceback self._loop.call_exception_handler(context) def __repr__(self): out = io.StringIO() ascii_encodable_url = self.url.encode('ascii', 'backslashreplace') \ .decode('ascii') if self.reason: ascii_encodable_reason = self.reason.encode('ascii', 'backslashreplace') \ .decode('ascii') else: ascii_encodable_reason = self.reason print('<ClientResponse({}) [{} {}]>'.format(ascii_encodable_url, self.status, ascii_encodable_reason), file=out) print(self.headers, file=out) return out.getvalue() @property def connection(self): return self._connection @property def history(self): """A sequence of of responses, if redirects occured.""" return self._history def waiting_for_continue(self): return self._continue is not None def _setup_connection(self, connection): self._reader = connection.reader self._connection = connection self.content = self.flow_control_class(connection.reader, loop=connection.loop, timeout=self._timeout) def _need_parse_response_body(self): return (self.method.lower() != 'head' and self.status not in [204, 304]) @asyncio.coroutine def start(self, connection, read_until_eof=False): """Start response processing.""" self._setup_connection(connection) while True: httpstream = self._reader.set_parser(self._response_parser) # read response with Timeout(self._timeout, loop=self._loop): message = yield from httpstream.read() if message.code != 100: break if self._continue is not None and not self._continue.done(): self._continue.set_result(True) self._continue = None # response status self.version = message.version self.status = message.code self.reason = message.reason self._should_close = message.should_close # headers self.headers = CIMultiDictProxy(message.headers) self.raw_headers = tuple(message.raw_headers) # payload rwb = self._need_parse_response_body() self._reader.set_parser( aiohttp.HttpPayloadParser(message, readall=read_until_eof, response_with_body=rwb), self.content) # cookies self.cookies = http.cookies.SimpleCookie() if hdrs.SET_COOKIE in self.headers: for hdr in self.headers.getall(hdrs.SET_COOKIE): try: self.cookies.load(hdr) except http.cookies.CookieError as exc: client_logger.warning('Can not load response cookies: %s', exc) return self def close(self): if self._closed: return self._closed = True if self._loop is None or self._loop.is_closed(): return if self._connection is not None: self._connection.close() self._connection = None self._cleanup_writer() @asyncio.coroutine def release(self): if self._closed: return try: content = self.content if content is not None and not content.at_eof(): chunk = yield from content.readany() while chunk is not EOF_MARKER or chunk: chunk = yield from content.readany() except Exception: self._connection.close() self._connection = None raise finally: self._closed = True if self._connection is not None: self._connection.release() if self._reader is not None: self._reader.unset_parser() self._connection = None self._cleanup_writer() def raise_for_status(self): if 400 <= self.status: raise aiohttp.HttpProcessingError(code=self.status, message=self.reason) def _cleanup_writer(self): if self._writer is not None and not self._writer.done(): self._writer.cancel() self._writer = None @asyncio.coroutine def wait_for_close(self): if self._writer is not None: try: yield from self._writer finally: self._writer = None yield from self.release() @asyncio.coroutine def read(self): """Read response payload.""" if self._content is None: try: self._content = yield from self.content.read() except: self.close() raise else: yield from self.release() return self._content def _get_encoding(self): ctype = self.headers.get(hdrs.CONTENT_TYPE, '').lower() mtype, stype, _, params = helpers.parse_mimetype(ctype) encoding = params.get('charset') if not encoding: encoding = chardet.detect(self._content)['encoding'] if not encoding: encoding = 'utf-8' return encoding @asyncio.coroutine def text(self, encoding=None): """Read response payload and decode.""" if self._content is None: yield from self.read() if encoding is None: encoding = self._get_encoding() return self._content.decode(encoding) @asyncio.coroutine def json(self, *, encoding=None, loads=json.loads): """Read and decodes JSON response.""" if self._content is None: yield from self.read() ctype = self.headers.get(hdrs.CONTENT_TYPE, '').lower() if 'json' not in ctype: client_logger.warning( 'Attempt to decode JSON with unexpected mimetype: %s', ctype) stripped = self._content.strip() if not stripped: return None if encoding is None: encoding = self._get_encoding() return loads(stripped.decode(encoding)) if PY_35: @asyncio.coroutine def __aenter__(self): return self @asyncio.coroutine def __aexit__(self, exc_type, exc_val, exc_tb): if exc_type is None: yield from self.release() else: self.close()
class ClientResponse(HeadersMixin): # from the Status-Line of the response version = None # HTTP-Version status = None # Status-Code reason = None # Reason-Phrase content = None # Payload stream headers = None # Response headers, CIMultiDictProxy raw_headers = None # Response raw headers, a sequence of pairs _connection = None # current connection flow_control_class = FlowControlStreamReader # reader flow control _reader = None # input stream _source_traceback = None # setted up by ClientRequest after ClientResponse object creation # post-init stage allows to not change ctor signature _loop = None _closed = True # to allow __del__ for non-initialized properly response def __init__(self, method, url, *, writer=None, continue100=None, timer=None): assert isinstance(url, URL) self.method = method self._url = url self._content = None self._writer = writer self._continue = continue100 self._closed = False self._should_close = True # override by message.should_close later self._history = () self.headers = None self._timer = timer if timer is not None else _TimeServiceTimeoutNoop() self.cookies = SimpleCookie() @property def url(self): return self._url @property def url_obj(self): warnings.warn("Deprecated, use .url #1654", DeprecationWarning, stacklevel=2) return self._url @property def host(self): warnings.warn("Deprecated, use .url.host", DeprecationWarning, stacklevel=2) return self._url.host @property def _headers(self): return self.headers def _post_init(self, loop): self._loop = loop if loop.get_debug(): self._source_traceback = traceback.extract_stack(sys._getframe(1)) def __del__(self, _warnings=warnings): if self._loop is None: return # not started if self._closed: return self.close() _warnings.warn("Unclosed response {!r}".format(self), ResourceWarning) context = {'client_response': self, 'message': 'Unclosed response'} if self._source_traceback: context['source_traceback'] = self._source_traceback self._loop.call_exception_handler(context) def __repr__(self): out = io.StringIO() ascii_encodable_url = str(self.url) if self.reason: ascii_encodable_reason = self.reason.encode('ascii', 'backslashreplace') \ .decode('ascii') else: ascii_encodable_reason = self.reason print('<ClientResponse({}) [{} {}]>'.format(ascii_encodable_url, self.status, ascii_encodable_reason), file=out) print(self.headers, file=out) return out.getvalue() @property def connection(self): return self._connection @property def history(self): """A sequence of of responses, if redirects occurred.""" return self._history @asyncio.coroutine def start(self, connection, read_until_eof=False): """Start response processing.""" self._protocol = connection.protocol self._connection = connection connection.protocol.set_response_params( timer=self._timer, skip_payload=self.method.lower() == 'head', skip_status_codes=(204, 304), read_until_eof=read_until_eof) with self._timer: while True: # read response (message, payload) = yield from self._protocol.read() if (message.code < 100 or message.code > 199 or message.code == 101): break if self._continue is not None and not self._continue.done(): self._continue.set_result(True) self._continue = None # response status self.version = message.version self.status = message.code self.reason = message.reason self._should_close = message.should_close # headers self.headers = CIMultiDictProxy(message.headers) self.raw_headers = tuple(message.raw_headers) # payload self.content = payload # cookies for hdr in self.headers.getall(hdrs.SET_COOKIE, ()): try: self.cookies.load(hdr) except CookieError as exc: client_logger.warning('Can not load response cookies: %s', exc) return self def close(self): if self._closed: return self._closed = True if self._loop is None or self._loop.is_closed(): return if self._connection is not None: self._connection.close() self._connection = None self._cleanup_writer() self._notify_content() @asyncio.coroutine def release(self): if self._closed: return try: content = self.content if content is not None: close = False if content.exception() is not None: close = True else: content.read_nowait() if not content.at_eof(): close = True if close: self.close() except Exception: self._connection.close() self._connection = None raise finally: self._closed = True if self._connection is not None: self._connection.release() self._connection = None self._cleanup_writer() self._notify_content() def raise_for_status(self): if 400 <= self.status: raise aiohttp.ClientResponseError(code=self.status, message=self.reason, headers=self.headers) def _cleanup_writer(self): if self._writer is not None and not self._writer.done(): self._writer.cancel() self._writer = None def _notify_content(self): content = self.content if content and content.exception() is None and not content.is_eof(): content.set_exception( aiohttp.ClientDisconnectedError('Connection closed')) @asyncio.coroutine def wait_for_close(self): if self._writer is not None: try: yield from self._writer finally: self._writer = None yield from self.release() @asyncio.coroutine def read(self): """Read response payload.""" if self._content is None: try: self._content = yield from self.content.read() except: self.close() raise else: yield from self.release() return self._content def _get_encoding(self): ctype = self.headers.get(hdrs.CONTENT_TYPE, '').lower() mtype, stype, _, params = helpers.parse_mimetype(ctype) encoding = params.get('charset') if not encoding: if mtype == 'application' and stype == 'json': # RFC 7159 states that the default encoding is UTF-8. encoding = 'utf-8' else: encoding = chardet.detect(self._content)['encoding'] if not encoding: encoding = 'utf-8' return encoding @asyncio.coroutine def text(self, encoding=None, errors='strict'): """Read response payload and decode.""" if self._content is None: yield from self.read() if encoding is None: encoding = self._get_encoding() return self._content.decode(encoding, errors=errors) @asyncio.coroutine def json(self, *, encoding=None, loads=json.loads): """Read and decodes JSON response.""" if self._content is None: yield from self.read() ctype = self.headers.get(hdrs.CONTENT_TYPE, '').lower() if 'json' not in ctype: client_logger.warning( 'Attempt to decode JSON with unexpected mimetype: %s', ctype) stripped = self._content.strip() if not stripped: return None if encoding is None: encoding = self._get_encoding() return loads(stripped.decode(encoding)) if PY_35: @asyncio.coroutine def __aenter__(self): return self @asyncio.coroutine def __aexit__(self, exc_type, exc_val, exc_tb): # similar to _RequestContextManager, we do not need to check # for exceptions, response object can closes connection # is state is broken yield from self.release()
class ClientResponse(HeadersMixin): # from the Status-Line of the response version = None # HTTP-Version status = None # Status-Code reason = None # Reason-Phrase content = None # Payload stream headers = None # Response headers, CIMultiDictProxy raw_headers = None # Response raw headers, a sequence of pairs _connection = None # current connection flow_control_class = StreamReader # reader flow control _reader = None # input stream _source_traceback = None # setted up by ClientRequest after ClientResponse object creation # post-init stage allows to not change ctor signature _loop = None _closed = True # to allow __del__ for non-initialized properly response _session = None def __init__(self, method, url, *, writer=None, continue100=None, timer=None, request_info=None, auto_decompress=True): assert isinstance(url, URL) self.method = method self.headers = None self.cookies = SimpleCookie() self._url = url self._content = None self._writer = writer self._continue = continue100 self._closed = True self._history = () self._request_info = request_info self._timer = timer if timer is not None else TimerNoop() self._auto_decompress = auto_decompress self._cache = {} # reqired for @reify method decorator @property def url(self): return self._url @property def url_obj(self): warnings.warn("Deprecated, use .url #1654", DeprecationWarning, stacklevel=2) return self._url @property def host(self): return self._url.host @property def _headers(self): return self.headers @property def request_info(self): return self._request_info @reify def content_disposition(self): raw = self._headers.get(hdrs.CONTENT_DISPOSITION) if raw is None: return None disposition_type, params = multipart.parse_content_disposition(raw) params = MappingProxyType(params) filename = multipart.content_disposition_filename(params) return ContentDisposition(disposition_type, params, filename) def _post_init(self, loop, session): self._loop = loop self._session = session # store a reference to session #1985 if loop.get_debug(): self._source_traceback = traceback.extract_stack(sys._getframe(1)) def __del__(self, _warnings=warnings): if self._loop is None: return # not started if self._closed: return if self._connection is not None: self._connection.release() self._cleanup_writer() if self._loop.get_debug(): if PY_36: kwargs = {'source': self} else: kwargs = {} _warnings.warn("Unclosed response {!r}".format(self), ResourceWarning, **kwargs) context = { 'client_response': self, 'message': 'Unclosed response' } if self._source_traceback: context['source_traceback'] = self._source_traceback self._loop.call_exception_handler(context) def __repr__(self): out = io.StringIO() ascii_encodable_url = str(self.url) if self.reason: ascii_encodable_reason = self.reason.encode('ascii', 'backslashreplace') \ .decode('ascii') else: ascii_encodable_reason = self.reason print('<ClientResponse({}) [{} {}]>'.format(ascii_encodable_url, self.status, ascii_encodable_reason), file=out) print(self.headers, file=out) return out.getvalue() @property def connection(self): return self._connection @property def history(self): """A sequence of of responses, if redirects occurred.""" return self._history async def start(self, connection, read_until_eof=False): """Start response processing.""" self._closed = False self._protocol = connection.protocol self._connection = connection connection.protocol.set_response_params( timer=self._timer, skip_payload=self.method.lower() == 'head', skip_status_codes=(204, 304), read_until_eof=read_until_eof, auto_decompress=self._auto_decompress) with self._timer: while True: # read response try: (message, payload) = await self._protocol.read() except http.HttpProcessingError as exc: raise ClientResponseError(self.request_info, self.history, code=exc.code, message=exc.message, headers=exc.headers) from exc if (message.code < 100 or message.code > 199 or message.code == 101): break if self._continue is not None: set_result(self._continue, True) self._continue = None # payload eof handler payload.on_eof(self._response_eof) # response status self.version = message.version self.status = message.code self.reason = message.reason # headers self.headers = CIMultiDictProxy(message.headers) self.raw_headers = tuple(message.raw_headers) # payload self.content = payload # cookies for hdr in self.headers.getall(hdrs.SET_COOKIE, ()): try: self.cookies.load(hdr) except CookieError as exc: client_logger.warning('Can not load response cookies: %s', exc) return self def _response_eof(self): if self._closed: return if self._connection is not None: # websocket, protocol could be None because # connection could be detached if (self._connection.protocol is not None and self._connection.protocol.upgraded): return self._connection.release() self._connection = None self._closed = True self._cleanup_writer() @property def closed(self): return self._closed def close(self): if self._closed: return self._closed = True if self._loop is None or self._loop.is_closed(): return if self._connection is not None: self._connection.close() self._connection = None self._cleanup_writer() self._notify_content() def release(self): if self._closed: return noop() self._closed = True if self._connection is not None: self._connection.release() self._connection = None self._cleanup_writer() self._notify_content() return noop() def raise_for_status(self): if 400 <= self.status: raise ClientResponseError(self.request_info, self.history, code=self.status, message=self.reason, headers=self.headers) def _cleanup_writer(self): if self._writer is not None: self._writer.cancel() self._writer = None self._session = None def _notify_content(self): content = self.content if content and content.exception() is None and not content.is_eof(): content.set_exception(ClientConnectionError('Connection closed')) async def wait_for_close(self): if self._writer is not None: try: await self._writer finally: self._writer = None self.release() async def read(self): """Read response payload.""" if self._content is None: try: self._content = await self.content.read() except Exception: self.close() raise return self._content def get_encoding(self): ctype = self.headers.get(hdrs.CONTENT_TYPE, '').lower() mimetype = helpers.parse_mimetype(ctype) encoding = mimetype.parameters.get('charset') if encoding: try: codecs.lookup(encoding) except LookupError: encoding = None if not encoding: if mimetype.type == 'application' and mimetype.subtype == 'json': # RFC 7159 states that the default encoding is UTF-8. encoding = 'utf-8' else: encoding = chardet.detect(self._content)['encoding'] if not encoding: encoding = 'utf-8' return encoding async def text(self, encoding=None, errors='strict'): """Read response payload and decode.""" if self._content is None: await self.read() if encoding is None: encoding = self.get_encoding() return self._content.decode(encoding, errors=errors) async def json(self, *, encoding=None, loads=json.loads, content_type='application/json'): """Read and decodes JSON response.""" if self._content is None: await self.read() if content_type: ctype = self.headers.get(hdrs.CONTENT_TYPE, '').lower() if content_type not in ctype: raise ContentTypeError(self.request_info, self.history, message=('Attempt to decode JSON with ' 'unexpected mimetype: %s' % ctype), headers=self.headers) stripped = self._content.strip() if not stripped: return None if encoding is None: encoding = self.get_encoding() return loads(stripped.decode(encoding)) async def __aenter__(self): return self async def __aexit__(self, exc_type, exc_val, exc_tb): # similar to _RequestContextManager, we do not need to check # for exceptions, response object can closes connection # is state is broken self.release()
class ClientResponse: # from the Status-Line of the response version = None # HTTP-Version status = None # Status-Code reason = None # Reason-Phrase cookies = None # Response cookies (Set-Cookie) content = None # Payload stream headers = None # Response headers, CIMultiDictProxy raw_headers = None # Response raw headers, a sequence of pairs _connection = None # current connection flow_control_class = FlowControlStreamReader # reader flow control _reader = None # input stream _response_parser = aiohttp.HttpResponseParser() _source_traceback = None # setted up by ClientRequest after ClientResponse object creation # post-init stage allows to not change ctor signature _loop = None _closed = True # to allow __del__ for non-initialized properly response def __init__(self, method, url, host="", *, writer=None, continue100=None): super().__init__() self.method = method self.url = url self.host = host self._content = None self._writer = writer self._continue = continue100 self._closed = False self._should_close = True # override by message.should_close later self._history = () def _post_init(self, loop): self._loop = loop if loop.get_debug(): self._source_traceback = traceback.extract_stack(sys._getframe(1)) def __del__(self, _warnings=warnings): if self._closed: return self.close() _warnings.warn("Unclosed response {!r}".format(self), ResourceWarning) context = {"client_response": self, "message": "Unclosed response"} if self._source_traceback: context["source_traceback"] = self._source_traceback self._loop.call_exception_handler(context) def __repr__(self): out = io.StringIO() print("<ClientResponse({}) [{} {}]>".format(self.url, self.status, self.reason), file=out) print(self.headers, file=out) return out.getvalue() @property def connection(self): return self._connection @property def history(self): """A sequence of of responses, if redirects occured.""" return self._history def waiting_for_continue(self): return self._continue is not None def _setup_connection(self, connection): self._reader = connection.reader self._connection = connection self.content = self.flow_control_class(connection.reader, loop=connection.loop) def _need_parse_response_body(self): return self.method.lower() != "head" and self.status not in [204, 304] @asyncio.coroutine def start(self, connection, read_until_eof=False): """Start response processing.""" self._setup_connection(connection) while True: httpstream = self._reader.set_parser(self._response_parser) # read response message = yield from httpstream.read() if message.code != 100: break if self._continue is not None and not self._continue.done(): self._continue.set_result(True) self._continue = None # response status self.version = message.version self.status = message.code self.reason = message.reason self._should_close = message.should_close # headers self.headers = CIMultiDictProxy(message.headers) self.raw_headers = tuple(message.raw_headers) # payload response_with_body = self._need_parse_response_body() self._reader.set_parser( aiohttp.HttpPayloadParser(message, readall=read_until_eof, response_with_body=response_with_body), self.content, ) # cookies self.cookies = http.cookies.SimpleCookie() if hdrs.SET_COOKIE in self.headers: for hdr in self.headers.getall(hdrs.SET_COOKIE): try: self.cookies.load(hdr) except http.cookies.CookieError as exc: client_logger.warning("Can not load response cookies: %s", exc) return self def close(self, force=True): if not force: warnings.warn("force parameter should be True", DeprecationWarning, stacklevel=2) if self._closed: return self._closed = True if hasattr(self._loop, "is_closed"): if self._loop.is_closed(): return if self._connection is not None: self._connection.close() self._connection = None self._cleanup_writer() @asyncio.coroutine def release(self): if self._closed: return try: content = self.content if content is not None and not content.at_eof(): chunk = yield from content.readany() while chunk is not EOF_MARKER or chunk: chunk = yield from content.readany() except Exception: self._connection.close() self._connection = None raise finally: self._closed = True if self._connection is not None: self._connection.release() if self._reader is not None: self._reader.unset_parser() self._connection = None self._cleanup_writer() def _cleanup_writer(self): if self._writer is not None and not self._writer.done(): self._writer.cancel() self._writer = None @asyncio.coroutine def wait_for_close(self): if self._writer is not None: try: yield from self._writer finally: self._writer = None yield from self.release() @asyncio.coroutine def read(self, decode=False): """Read response payload.""" if self._content is None: try: self._content = yield from self.content.read() except: self.close() raise else: yield from self.release() data = self._content if decode: warnings.warn(".read(True) is deprecated. use .json() instead", DeprecationWarning) return (yield from self.json()) return data def _get_encoding(self): ctype = self.headers.get(hdrs.CONTENT_TYPE, "").lower() mtype, stype, _, params = helpers.parse_mimetype(ctype) encoding = params.get("charset") if not encoding: encoding = chardet.detect(self._content)["encoding"] if not encoding: encoding = "utf-8" return encoding @asyncio.coroutine def text(self, encoding=None): """Read response payload and decode.""" if self._content is None: yield from self.read() if encoding is None: encoding = self._get_encoding() return self._content.decode(encoding) @asyncio.coroutine def json(self, *, encoding=None, loads=json.loads): """Read and decodes JSON response.""" if self._content is None: yield from self.read() ctype = self.headers.get(hdrs.CONTENT_TYPE, "").lower() if "json" not in ctype: client_logger.warning("Attempt to decode JSON with unexpected mimetype: %s", ctype) stripped = self._content.strip() if not stripped: return None if encoding is None: encoding = self._get_encoding() return loads(stripped.decode(encoding)) if PY_35: @asyncio.coroutine def __aenter__(self): return self @asyncio.coroutine def __aexit__(self, exc_type, exc_val, exc_tb): if exc_type is None: yield from self.release() else: self.close()
class ClientResponse(HeadersMixin): # from the Status-Line of the response version = None # HTTP-Version status = None # Status-Code reason = None # Reason-Phrase content = None # Payload stream headers = None # Response headers, CIMultiDictProxy raw_headers = None # Response raw headers, a sequence of pairs _connection = None # current connection flow_control_class = StreamReader # reader flow control _reader = None # input stream _source_traceback = None # setted up by ClientRequest after ClientResponse object creation # post-init stage allows to not change ctor signature _loop = None _closed = True # to allow __del__ for non-initialized properly response _session = None def __init__(self, method, url, *, writer=None, continue100=None, timer=None, request_info=None, auto_decompress=True): assert isinstance(url, URL) self.method = method self.headers = None self.cookies = SimpleCookie() self._url = url self._content = None self._writer = writer self._continue = continue100 self._closed = True self._history = () self._request_info = request_info self._timer = timer if timer is not None else TimerNoop() self._auto_decompress = auto_decompress self._cache = {} # reqired for @reify method decorator @property def url(self): return self._url @property def url_obj(self): warnings.warn( "Deprecated, use .url #1654", DeprecationWarning, stacklevel=2) return self._url @property def host(self): return self._url.host @property def _headers(self): return self.headers @property def request_info(self): return self._request_info @reify def content_disposition(self): raw = self._headers.get(hdrs.CONTENT_DISPOSITION) if raw is None: return None disposition_type, params = multipart.parse_content_disposition(raw) params = MappingProxyType(params) filename = multipart.content_disposition_filename(params) return ContentDisposition(disposition_type, params, filename) def _post_init(self, loop, session): self._loop = loop self._session = session # store a reference to session #1985 if loop.get_debug(): self._source_traceback = traceback.extract_stack(sys._getframe(1)) def __del__(self, _warnings=warnings): if self._loop is None: return # not started if self._closed: return if self._connection is not None: self._connection.release() self._cleanup_writer() if self._loop.get_debug(): if PY_36: kwargs = {'source': self} else: kwargs = {} _warnings.warn("Unclosed response {!r}".format(self), ResourceWarning, **kwargs) context = {'client_response': self, 'message': 'Unclosed response'} if self._source_traceback: context['source_traceback'] = self._source_traceback self._loop.call_exception_handler(context) def __repr__(self): out = io.StringIO() ascii_encodable_url = str(self.url) if self.reason: ascii_encodable_reason = self.reason.encode('ascii', 'backslashreplace') \ .decode('ascii') else: ascii_encodable_reason = self.reason print('<ClientResponse({}) [{} {}]>'.format( ascii_encodable_url, self.status, ascii_encodable_reason), file=out) print(self.headers, file=out) return out.getvalue() @property def connection(self): return self._connection @property def history(self): """A sequence of of responses, if redirects occurred.""" return self._history async def start(self, connection, read_until_eof=False): """Start response processing.""" self._closed = False self._protocol = connection.protocol self._connection = connection connection.protocol.set_response_params( timer=self._timer, skip_payload=self.method.lower() == 'head', skip_status_codes=(204, 304), read_until_eof=read_until_eof, auto_decompress=self._auto_decompress) with self._timer: while True: # read response try: (message, payload) = await self._protocol.read() except http.HttpProcessingError as exc: raise ClientResponseError( self.request_info, self.history, code=exc.code, message=exc.message, headers=exc.headers) from exc if (message.code < 100 or message.code > 199 or message.code == 101): break if self._continue is not None: set_result(self._continue, True) self._continue = None # payload eof handler payload.on_eof(self._response_eof) # response status self.version = message.version self.status = message.code self.reason = message.reason # headers self.headers = CIMultiDictProxy(message.headers) self.raw_headers = tuple(message.raw_headers) # payload self.content = payload # cookies for hdr in self.headers.getall(hdrs.SET_COOKIE, ()): try: self.cookies.load(hdr) except CookieError as exc: client_logger.warning( 'Can not load response cookies: %s', exc) return self def _response_eof(self): if self._closed: return if self._connection is not None: # websocket, protocol could be None because # connection could be detached if (self._connection.protocol is not None and self._connection.protocol.upgraded): return self._connection.release() self._connection = None self._closed = True self._cleanup_writer() @property def closed(self): return self._closed def close(self): if self._closed: return self._closed = True if self._loop is None or self._loop.is_closed(): return if self._connection is not None: self._connection.close() self._connection = None self._cleanup_writer() self._notify_content() def release(self): if self._closed: return noop() self._closed = True if self._connection is not None: self._connection.release() self._connection = None self._cleanup_writer() self._notify_content() return noop() def raise_for_status(self): if 400 <= self.status: raise ClientResponseError( self.request_info, self.history, code=self.status, message=self.reason, headers=self.headers) def _cleanup_writer(self): if self._writer is not None: self._writer.cancel() self._writer = None self._session = None def _notify_content(self): content = self.content if content and content.exception() is None and not content.is_eof(): content.set_exception( ClientConnectionError('Connection closed')) async def wait_for_close(self): if self._writer is not None: try: await self._writer finally: self._writer = None self.release() async def read(self): """Read response payload.""" if self._content is None: try: self._content = await self.content.read() except Exception: self.close() raise return self._content def get_encoding(self): ctype = self.headers.get(hdrs.CONTENT_TYPE, '').lower() mimetype = helpers.parse_mimetype(ctype) encoding = mimetype.parameters.get('charset') if encoding: try: codecs.lookup(encoding) except LookupError: encoding = None if not encoding: if mimetype.type == 'application' and mimetype.subtype == 'json': # RFC 7159 states that the default encoding is UTF-8. encoding = 'utf-8' else: encoding = chardet.detect(self._content)['encoding'] if not encoding: encoding = 'utf-8' return encoding async def text(self, encoding=None, errors='strict'): """Read response payload and decode.""" if self._content is None: await self.read() if encoding is None: encoding = self.get_encoding() return self._content.decode(encoding, errors=errors) async def json(self, *, encoding=None, loads=json.loads, content_type='application/json'): """Read and decodes JSON response.""" if self._content is None: await self.read() if content_type: ctype = self.headers.get(hdrs.CONTENT_TYPE, '').lower() if content_type not in ctype: raise ContentTypeError( self.request_info, self.history, message=('Attempt to decode JSON with ' 'unexpected mimetype: %s' % ctype), headers=self.headers) stripped = self._content.strip() if not stripped: return None if encoding is None: encoding = self.get_encoding() return loads(stripped.decode(encoding)) async def __aenter__(self): return self async def __aexit__(self, exc_type, exc_val, exc_tb): # similar to _RequestContextManager, we do not need to check # for exceptions, response object can closes connection # is state is broken self.release()