async def handlers(parsed: ParseResult, text: str, headers: multidict.CIMultiDictProxy, session: aiohttp.client.ClientSession, transaction: Hub.current.scope.transaction): youtube_check = False bitly_warning = False adfly_warning = False url = None with transaction.start_child(op="task", description="Handlers"): query_parse = parse_qs(parsed.query) log.info("Checking handlers") if "youtube.com" in parsed.netloc and parsed.path == "/redirect": check = youtube.youtube(parsed) if check: url = check[0] youtube_check = True elif "google.com" in parsed.netloc and parsed.path == "/url": check = google.google(parsed) if check: url = check elif "bitly.com" in parsed.netloc and parsed.path == "/a/warning": check = bitly.bitly(parsed) if check: url = check[0] bitly_warning = True elif headers.get("x-powered-by") == "adfly" and len(parsed.path) > 0: check = adfly.adfly(text) if check: url = check adfly_warning = True elif "duckduckgo.com" in parsed.netloc and "q" in query_parse.keys(): check = await duckduckgo.duckduckgo(parsed, session) if check: url = check elif "justpaste.it" in parsed.netloc and "redirect" in parsed.path: check = justpasteit.justpasteit(text) if check: url = check elif parsed.netloc in linkvertise.linkvertise_domains(): check = await linkvertise.linkvertise(parsed, session) if check: url = check elif "goo.su" in parsed.netloc: check = goo_su.goo_su(text) if check: url = check elif parsed.netloc in privatebin.privatebin_domains(): check = await privatebin.privatebin(parsed, session) if check: url = check return {"url": url, "youtube": youtube_check, "bitly": bitly_warning, "adfly": adfly_warning}
class ClientResponse(HeadersMixin): # from the Status-Line of the response version = None # HTTP-Version status = None # Status-Code reason = None # Reason-Phrase cookies = None # Response cookies (Set-Cookie) content = None # Payload stream headers = None # Response headers, CIMultiDictProxy raw_headers = None # Response raw headers, a sequence of pairs _connection = None # current connection flow_control_class = FlowControlStreamReader # reader flow control _reader = None # input stream _response_parser = aiohttp.HttpResponseParser() _source_traceback = None # setted up by ClientRequest after ClientResponse object creation # post-init stage allows to not change ctor signature _loop = None _closed = True # to allow __del__ for non-initialized properly response def __init__(self, method, url, *, writer=None, continue100=None, timeout=5*60): assert isinstance(url, URL) self.method = method self._url_obj = url self._content = None self._writer = writer self._continue = continue100 self._closed = False self._should_close = True # override by message.should_close later self._history = () self._timeout = timeout @property def url_obj(self): return self._url_obj @property def url(self): return str(self._url_obj) @property def host(self): warnings.warn("Deprecated, use .url_obj.host", DeprecationWarning, stacklevel=2) return self._url_obj.host def _post_init(self, loop): self._loop = loop if loop.get_debug(): self._source_traceback = traceback.extract_stack(sys._getframe(1)) def __del__(self, _warnings=warnings): if self._loop is None: return # not started if self._closed: return self.close() _warnings.warn("Unclosed response {!r}".format(self), ResourceWarning) context = {'client_response': self, 'message': 'Unclosed response'} if self._source_traceback: context['source_traceback'] = self._source_traceback self._loop.call_exception_handler(context) def __repr__(self): out = io.StringIO() ascii_encodable_url = str(self.url) if self.reason: ascii_encodable_reason = self.reason.encode('ascii', 'backslashreplace') \ .decode('ascii') else: ascii_encodable_reason = self.reason print('<ClientResponse({}) [{} {}]>'.format( ascii_encodable_url, self.status, ascii_encodable_reason), file=out) print(self.headers, file=out) return out.getvalue() @property def connection(self): return self._connection @property def history(self): """A sequence of of responses, if redirects occurred.""" return self._history def waiting_for_continue(self): return self._continue is not None def _setup_connection(self, connection): self._reader = connection.reader self._connection = connection self.content = self.flow_control_class( connection.reader, loop=connection.loop, timeout=self._timeout) def _need_parse_response_body(self): return (self.method.lower() != 'head' and self.status not in [204, 304]) @asyncio.coroutine def start(self, connection, read_until_eof=False): """Start response processing.""" self._setup_connection(connection) while True: httpstream = self._reader.set_parser(self._response_parser) # read response with Timeout(self._timeout, loop=self._loop): message = yield from httpstream.read() if message.code != 100: break if self._continue is not None and not self._continue.done(): self._continue.set_result(True) self._continue = None # response status self.version = message.version self.status = message.code self.reason = message.reason self._should_close = message.should_close # headers self.headers = CIMultiDictProxy(message.headers) self.raw_headers = tuple(message.raw_headers) # payload rwb = self._need_parse_response_body() self._reader.set_parser( aiohttp.HttpPayloadParser(message, readall=read_until_eof, response_with_body=rwb), self.content) # cookies self.cookies = http.cookies.SimpleCookie() if hdrs.SET_COOKIE in self.headers: for hdr in self.headers.getall(hdrs.SET_COOKIE): try: self.cookies.load(hdr) except http.cookies.CookieError as exc: client_logger.warning( 'Can not load response cookies: %s', exc) return self def close(self): if self._closed: return self._closed = True if self._loop is None or self._loop.is_closed(): return if self._connection is not None: self._connection.close() self._connection = None self._cleanup_writer() self._notify_content() @asyncio.coroutine def release(self): if self._closed: return try: content = self.content if content is not None and not content.at_eof(): chunk = yield from content.readany() while chunk is not EOF_MARKER or chunk: chunk = yield from content.readany() except Exception: self._connection.close() self._connection = None raise finally: self._closed = True if self._connection is not None: self._connection.release() if self._reader is not None: self._reader.unset_parser() self._connection = None self._cleanup_writer() self._notify_content() def raise_for_status(self): if 400 <= self.status: raise aiohttp.HttpProcessingError( code=self.status, message=self.reason) def _cleanup_writer(self): if self._writer is not None and not self._writer.done(): self._writer.cancel() self._writer = None def _notify_content(self): content = self.content if content and content.exception() is None and not content.is_eof(): content.set_exception( aiohttp.ClientDisconnectedError('Connection closed')) @asyncio.coroutine def wait_for_close(self): if self._writer is not None: try: yield from self._writer finally: self._writer = None yield from self.release() @asyncio.coroutine def read(self): """Read response payload.""" if self._content is None: try: self._content = yield from self.content.read() except: self.close() raise else: yield from self.release() return self._content def _get_encoding(self): ctype = self.headers.get(hdrs.CONTENT_TYPE, '').lower() mtype, stype, _, params = helpers.parse_mimetype(ctype) encoding = params.get('charset') if not encoding: if mtype == 'application' and stype == 'json': # RFC 7159 states that the default encoding is UTF-8. encoding = 'utf-8' else: encoding = chardet.detect(self._content)['encoding'] if not encoding: encoding = 'utf-8' return encoding @asyncio.coroutine def text(self, encoding=None): """Read response payload and decode.""" if self._content is None: yield from self.read() if encoding is None: encoding = self._get_encoding() return self._content.decode(encoding) @asyncio.coroutine def json(self, *, encoding=None, loads=json.loads): """Read and decodes JSON response.""" if self._content is None: yield from self.read() ctype = self.headers.get(hdrs.CONTENT_TYPE, '').lower() if 'json' not in ctype: client_logger.warning( 'Attempt to decode JSON with unexpected mimetype: %s', ctype) stripped = self._content.strip() if not stripped: return None if encoding is None: encoding = self._get_encoding() return loads(stripped.decode(encoding)) if PY_35: @asyncio.coroutine def __aenter__(self): return self @asyncio.coroutine def __aexit__(self, exc_type, exc_val, exc_tb): if exc_type is None: yield from self.release() else: self.close()
class ClientResponse: # from the Status-Line of the response version = None # HTTP-Version status = None # Status-Code reason = None # Reason-Phrase cookies = None # Response cookies (Set-Cookie) content = None # Payload stream headers = None # Response headers, CIMultiDictProxy raw_headers = None # Response raw headers, a sequence of pairs _connection = None # current connection flow_control_class = FlowControlStreamReader # reader flow control _reader = None # input stream _response_parser = aiohttp.HttpResponseParser() _source_traceback = None # setted up by ClientRequest after ClientResponse object creation # post-init stage allows to not change ctor signature _loop = None _closed = True # to allow __del__ for non-initialized properly response def __init__(self, method, url, host='', *, writer=None, continue100=None, timeout=5 * 60): super().__init__() self.method = method self.url = url self.host = host self._content = None self._writer = writer self._continue = continue100 self._closed = False self._should_close = True # override by message.should_close later self._history = () self._timeout = timeout def _post_init(self, loop): self._loop = loop if loop.get_debug(): self._source_traceback = traceback.extract_stack(sys._getframe(1)) def __del__(self, _warnings=warnings): if self._loop is None: return # not started if self._closed: return self.close() _warnings.warn("Unclosed response {!r}".format(self), ResourceWarning) context = {'client_response': self, 'message': 'Unclosed response'} if self._source_traceback: context['source_traceback'] = self._source_traceback self._loop.call_exception_handler(context) def __repr__(self): out = io.StringIO() ascii_encodable_url = self.url.encode('ascii', 'backslashreplace') \ .decode('ascii') if self.reason: ascii_encodable_reason = self.reason.encode('ascii', 'backslashreplace') \ .decode('ascii') else: ascii_encodable_reason = self.reason print('<ClientResponse({}) [{} {}]>'.format(ascii_encodable_url, self.status, ascii_encodable_reason), file=out) print(self.headers, file=out) return out.getvalue() @property def connection(self): return self._connection @property def history(self): """A sequence of of responses, if redirects occured.""" return self._history def waiting_for_continue(self): return self._continue is not None def _setup_connection(self, connection): self._reader = connection.reader self._connection = connection self.content = self.flow_control_class(connection.reader, loop=connection.loop, timeout=self._timeout) def _need_parse_response_body(self): return (self.method.lower() != 'head' and self.status not in [204, 304]) @asyncio.coroutine def start(self, connection, read_until_eof=False): """Start response processing.""" self._setup_connection(connection) while True: httpstream = self._reader.set_parser(self._response_parser) # read response with Timeout(self._timeout, loop=self._loop): message = yield from httpstream.read() if message.code != 100: break if self._continue is not None and not self._continue.done(): self._continue.set_result(True) self._continue = None # response status self.version = message.version self.status = message.code self.reason = message.reason self._should_close = message.should_close # headers self.headers = CIMultiDictProxy(message.headers) self.raw_headers = tuple(message.raw_headers) # payload rwb = self._need_parse_response_body() self._reader.set_parser( aiohttp.HttpPayloadParser(message, readall=read_until_eof, response_with_body=rwb), self.content) # cookies self.cookies = http.cookies.SimpleCookie() if hdrs.SET_COOKIE in self.headers: for hdr in self.headers.getall(hdrs.SET_COOKIE): try: self.cookies.load(hdr) except http.cookies.CookieError as exc: client_logger.warning('Can not load response cookies: %s', exc) return self def close(self): if self._closed: return self._closed = True if self._loop is None or self._loop.is_closed(): return if self._connection is not None: self._connection.close() self._connection = None self._cleanup_writer() @asyncio.coroutine def release(self): if self._closed: return try: content = self.content if content is not None and not content.at_eof(): chunk = yield from content.readany() while chunk is not EOF_MARKER or chunk: chunk = yield from content.readany() except Exception: self._connection.close() self._connection = None raise finally: self._closed = True if self._connection is not None: self._connection.release() if self._reader is not None: self._reader.unset_parser() self._connection = None self._cleanup_writer() def raise_for_status(self): if 400 <= self.status: raise aiohttp.HttpProcessingError(code=self.status, message=self.reason) def _cleanup_writer(self): if self._writer is not None and not self._writer.done(): self._writer.cancel() self._writer = None @asyncio.coroutine def wait_for_close(self): if self._writer is not None: try: yield from self._writer finally: self._writer = None yield from self.release() @asyncio.coroutine def read(self): """Read response payload.""" if self._content is None: try: self._content = yield from self.content.read() except: self.close() raise else: yield from self.release() return self._content def _get_encoding(self): ctype = self.headers.get(hdrs.CONTENT_TYPE, '').lower() mtype, stype, _, params = helpers.parse_mimetype(ctype) encoding = params.get('charset') if not encoding: encoding = chardet.detect(self._content)['encoding'] if not encoding: encoding = 'utf-8' return encoding @asyncio.coroutine def text(self, encoding=None): """Read response payload and decode.""" if self._content is None: yield from self.read() if encoding is None: encoding = self._get_encoding() return self._content.decode(encoding) @asyncio.coroutine def json(self, *, encoding=None, loads=json.loads): """Read and decodes JSON response.""" if self._content is None: yield from self.read() ctype = self.headers.get(hdrs.CONTENT_TYPE, '').lower() if 'json' not in ctype: client_logger.warning( 'Attempt to decode JSON with unexpected mimetype: %s', ctype) stripped = self._content.strip() if not stripped: return None if encoding is None: encoding = self._get_encoding() return loads(stripped.decode(encoding)) if PY_35: @asyncio.coroutine def __aenter__(self): return self @asyncio.coroutine def __aexit__(self, exc_type, exc_val, exc_tb): if exc_type is None: yield from self.release() else: self.close()
class ClientResponse(HeadersMixin): # from the Status-Line of the response version = None # HTTP-Version status = None # Status-Code reason = None # Reason-Phrase content = None # Payload stream headers = None # Response headers, CIMultiDictProxy raw_headers = None # Response raw headers, a sequence of pairs _connection = None # current connection flow_control_class = FlowControlStreamReader # reader flow control _reader = None # input stream _source_traceback = None # setted up by ClientRequest after ClientResponse object creation # post-init stage allows to not change ctor signature _loop = None _closed = True # to allow __del__ for non-initialized properly response def __init__(self, method, url, *, writer=None, continue100=None, timer=None): assert isinstance(url, URL) self.method = method self._url = url self._content = None self._writer = writer self._continue = continue100 self._closed = False self._should_close = True # override by message.should_close later self._history = () self.headers = None self._timer = timer if timer is not None else _TimeServiceTimeoutNoop() self.cookies = SimpleCookie() @property def url(self): return self._url @property def url_obj(self): warnings.warn("Deprecated, use .url #1654", DeprecationWarning, stacklevel=2) return self._url @property def host(self): warnings.warn("Deprecated, use .url.host", DeprecationWarning, stacklevel=2) return self._url.host @property def _headers(self): return self.headers def _post_init(self, loop): self._loop = loop if loop.get_debug(): self._source_traceback = traceback.extract_stack(sys._getframe(1)) def __del__(self, _warnings=warnings): if self._loop is None: return # not started if self._closed: return self.close() _warnings.warn("Unclosed response {!r}".format(self), ResourceWarning) context = {'client_response': self, 'message': 'Unclosed response'} if self._source_traceback: context['source_traceback'] = self._source_traceback self._loop.call_exception_handler(context) def __repr__(self): out = io.StringIO() ascii_encodable_url = str(self.url) if self.reason: ascii_encodable_reason = self.reason.encode('ascii', 'backslashreplace') \ .decode('ascii') else: ascii_encodable_reason = self.reason print('<ClientResponse({}) [{} {}]>'.format(ascii_encodable_url, self.status, ascii_encodable_reason), file=out) print(self.headers, file=out) return out.getvalue() @property def connection(self): return self._connection @property def history(self): """A sequence of of responses, if redirects occurred.""" return self._history @asyncio.coroutine def start(self, connection, read_until_eof=False): """Start response processing.""" self._protocol = connection.protocol self._connection = connection connection.protocol.set_response_params( timer=self._timer, skip_payload=self.method.lower() == 'head', skip_status_codes=(204, 304), read_until_eof=read_until_eof) with self._timer: while True: # read response (message, payload) = yield from self._protocol.read() if (message.code < 100 or message.code > 199 or message.code == 101): break if self._continue is not None and not self._continue.done(): self._continue.set_result(True) self._continue = None # response status self.version = message.version self.status = message.code self.reason = message.reason self._should_close = message.should_close # headers self.headers = CIMultiDictProxy(message.headers) self.raw_headers = tuple(message.raw_headers) # payload self.content = payload # cookies for hdr in self.headers.getall(hdrs.SET_COOKIE, ()): try: self.cookies.load(hdr) except CookieError as exc: client_logger.warning('Can not load response cookies: %s', exc) return self def close(self): if self._closed: return self._closed = True if self._loop is None or self._loop.is_closed(): return if self._connection is not None: self._connection.close() self._connection = None self._cleanup_writer() self._notify_content() @asyncio.coroutine def release(self): if self._closed: return try: content = self.content if content is not None: close = False if content.exception() is not None: close = True else: content.read_nowait() if not content.at_eof(): close = True if close: self.close() except Exception: self._connection.close() self._connection = None raise finally: self._closed = True if self._connection is not None: self._connection.release() self._connection = None self._cleanup_writer() self._notify_content() def raise_for_status(self): if 400 <= self.status: raise aiohttp.ClientResponseError(code=self.status, message=self.reason, headers=self.headers) def _cleanup_writer(self): if self._writer is not None and not self._writer.done(): self._writer.cancel() self._writer = None def _notify_content(self): content = self.content if content and content.exception() is None and not content.is_eof(): content.set_exception( aiohttp.ClientDisconnectedError('Connection closed')) @asyncio.coroutine def wait_for_close(self): if self._writer is not None: try: yield from self._writer finally: self._writer = None yield from self.release() @asyncio.coroutine def read(self): """Read response payload.""" if self._content is None: try: self._content = yield from self.content.read() except: self.close() raise else: yield from self.release() return self._content def _get_encoding(self): ctype = self.headers.get(hdrs.CONTENT_TYPE, '').lower() mtype, stype, _, params = helpers.parse_mimetype(ctype) encoding = params.get('charset') if not encoding: if mtype == 'application' and stype == 'json': # RFC 7159 states that the default encoding is UTF-8. encoding = 'utf-8' else: encoding = chardet.detect(self._content)['encoding'] if not encoding: encoding = 'utf-8' return encoding @asyncio.coroutine def text(self, encoding=None, errors='strict'): """Read response payload and decode.""" if self._content is None: yield from self.read() if encoding is None: encoding = self._get_encoding() return self._content.decode(encoding, errors=errors) @asyncio.coroutine def json(self, *, encoding=None, loads=json.loads): """Read and decodes JSON response.""" if self._content is None: yield from self.read() ctype = self.headers.get(hdrs.CONTENT_TYPE, '').lower() if 'json' not in ctype: client_logger.warning( 'Attempt to decode JSON with unexpected mimetype: %s', ctype) stripped = self._content.strip() if not stripped: return None if encoding is None: encoding = self._get_encoding() return loads(stripped.decode(encoding)) if PY_35: @asyncio.coroutine def __aenter__(self): return self @asyncio.coroutine def __aexit__(self, exc_type, exc_val, exc_tb): # similar to _RequestContextManager, we do not need to check # for exceptions, response object can closes connection # is state is broken yield from self.release()
def make_mocked_request(method, path, headers=None, *, match_info=sentinel, version=HttpVersion(1, 1), closing=False, app=None, writer=sentinel, protocol=sentinel, transport=sentinel, payload=sentinel, sslcontext=None, client_max_size=1024**2, loop=...): """Creates mocked web.Request testing purposes. Useful in unit tests, when spinning full web server is overkill or specific conditions and errors are hard to trigger. """ task = mock.Mock() if loop is ...: loop = mock.Mock() loop.create_future.return_value = () if version < HttpVersion(1, 1): closing = True if headers: headers = CIMultiDictProxy(CIMultiDict(headers)) raw_hdrs = tuple( (k.encode('utf-8'), v.encode('utf-8')) for k, v in headers.items()) else: headers = CIMultiDictProxy(CIMultiDict()) raw_hdrs = () chunked = 'chunked' in headers.get(hdrs.TRANSFER_ENCODING, '').lower() message = RawRequestMessage(method, path, version, headers, raw_hdrs, closing, False, False, chunked, URL(path)) if app is None: app = _create_app_mock() if transport is sentinel: transport = _create_transport(sslcontext) if protocol is sentinel: protocol = mock.Mock() protocol.transport = transport if writer is sentinel: writer = mock.Mock() writer.write_headers = make_mocked_coro(None) writer.write = make_mocked_coro(None) writer.write_eof = make_mocked_coro(None) writer.drain = make_mocked_coro(None) writer.transport = transport protocol.transport = transport protocol.writer = writer if payload is sentinel: payload = mock.Mock() req = Request(message, payload, protocol, writer, task, loop, client_max_size=client_max_size) match_info = UrlMappingMatchInfo( {} if match_info is sentinel else match_info, mock.Mock()) match_info.add_app(app) req._match_info = match_info return req
def make_mocked_request(method, path, headers=None, *, match_info=sentinel, version=HttpVersion(1, 1), closing=False, app=None, writer=sentinel, protocol=sentinel, transport=sentinel, payload=sentinel, sslcontext=None, client_max_size=1024**2, loop=...): """Creates mocked web.Request testing purposes. Useful in unit tests, when spinning full web server is overkill or specific conditions and errors are hard to trigger. """ task = mock.Mock() if loop is ...: loop = mock.Mock() loop.create_future.return_value = () if version < HttpVersion(1, 1): closing = True if headers: headers = CIMultiDictProxy(CIMultiDict(headers)) raw_hdrs = tuple( (k.encode('utf-8'), v.encode('utf-8')) for k, v in headers.items()) else: headers = CIMultiDictProxy(CIMultiDict()) raw_hdrs = () chunked = 'chunked' in headers.get(hdrs.TRANSFER_ENCODING, '').lower() message = RawRequestMessage( method, path, version, headers, raw_hdrs, closing, False, False, chunked, URL(path)) if app is None: app = _create_app_mock() if transport is sentinel: transport = _create_transport(sslcontext) if protocol is sentinel: protocol = mock.Mock() protocol.transport = transport if writer is sentinel: writer = mock.Mock() writer.write_headers = make_mocked_coro(None) writer.write = make_mocked_coro(None) writer.write_eof = make_mocked_coro(None) writer.drain = make_mocked_coro(None) writer.transport = transport protocol.transport = transport protocol.writer = writer if payload is sentinel: payload = mock.Mock() req = Request(message, payload, protocol, writer, task, loop, client_max_size=client_max_size) match_info = UrlMappingMatchInfo( {} if match_info is sentinel else match_info, mock.Mock()) match_info.add_app(app) req._match_info = match_info return req
class ClientResponse(HeadersMixin): # from the Status-Line of the response version = None # HTTP-Version status = None # Status-Code reason = None # Reason-Phrase content = None # Payload stream headers = None # Response headers, CIMultiDictProxy raw_headers = None # Response raw headers, a sequence of pairs _connection = None # current connection flow_control_class = StreamReader # reader flow control _reader = None # input stream _source_traceback = None # setted up by ClientRequest after ClientResponse object creation # post-init stage allows to not change ctor signature _loop = None _closed = True # to allow __del__ for non-initialized properly response _session = None def __init__(self, method, url, *, writer=None, continue100=None, timer=None, request_info=None, auto_decompress=True): assert isinstance(url, URL) self.method = method self.headers = None self.cookies = SimpleCookie() self._url = url self._content = None self._writer = writer self._continue = continue100 self._closed = True self._history = () self._request_info = request_info self._timer = timer if timer is not None else TimerNoop() self._auto_decompress = auto_decompress self._cache = {} # reqired for @reify method decorator @property def url(self): return self._url @property def url_obj(self): warnings.warn("Deprecated, use .url #1654", DeprecationWarning, stacklevel=2) return self._url @property def host(self): return self._url.host @property def _headers(self): return self.headers @property def request_info(self): return self._request_info @reify def content_disposition(self): raw = self._headers.get(hdrs.CONTENT_DISPOSITION) if raw is None: return None disposition_type, params = multipart.parse_content_disposition(raw) params = MappingProxyType(params) filename = multipart.content_disposition_filename(params) return ContentDisposition(disposition_type, params, filename) def _post_init(self, loop, session): self._loop = loop self._session = session # store a reference to session #1985 if loop.get_debug(): self._source_traceback = traceback.extract_stack(sys._getframe(1)) def __del__(self, _warnings=warnings): if self._loop is None: return # not started if self._closed: return if self._connection is not None: self._connection.release() self._cleanup_writer() if self._loop.get_debug(): if PY_36: kwargs = {'source': self} else: kwargs = {} _warnings.warn("Unclosed response {!r}".format(self), ResourceWarning, **kwargs) context = { 'client_response': self, 'message': 'Unclosed response' } if self._source_traceback: context['source_traceback'] = self._source_traceback self._loop.call_exception_handler(context) def __repr__(self): out = io.StringIO() ascii_encodable_url = str(self.url) if self.reason: ascii_encodable_reason = self.reason.encode('ascii', 'backslashreplace') \ .decode('ascii') else: ascii_encodable_reason = self.reason print('<ClientResponse({}) [{} {}]>'.format(ascii_encodable_url, self.status, ascii_encodable_reason), file=out) print(self.headers, file=out) return out.getvalue() @property def connection(self): return self._connection @property def history(self): """A sequence of of responses, if redirects occurred.""" return self._history async def start(self, connection, read_until_eof=False): """Start response processing.""" self._closed = False self._protocol = connection.protocol self._connection = connection connection.protocol.set_response_params( timer=self._timer, skip_payload=self.method.lower() == 'head', skip_status_codes=(204, 304), read_until_eof=read_until_eof, auto_decompress=self._auto_decompress) with self._timer: while True: # read response try: (message, payload) = await self._protocol.read() except http.HttpProcessingError as exc: raise ClientResponseError(self.request_info, self.history, code=exc.code, message=exc.message, headers=exc.headers) from exc if (message.code < 100 or message.code > 199 or message.code == 101): break if self._continue is not None: set_result(self._continue, True) self._continue = None # payload eof handler payload.on_eof(self._response_eof) # response status self.version = message.version self.status = message.code self.reason = message.reason # headers self.headers = CIMultiDictProxy(message.headers) self.raw_headers = tuple(message.raw_headers) # payload self.content = payload # cookies for hdr in self.headers.getall(hdrs.SET_COOKIE, ()): try: self.cookies.load(hdr) except CookieError as exc: client_logger.warning('Can not load response cookies: %s', exc) return self def _response_eof(self): if self._closed: return if self._connection is not None: # websocket, protocol could be None because # connection could be detached if (self._connection.protocol is not None and self._connection.protocol.upgraded): return self._connection.release() self._connection = None self._closed = True self._cleanup_writer() @property def closed(self): return self._closed def close(self): if self._closed: return self._closed = True if self._loop is None or self._loop.is_closed(): return if self._connection is not None: self._connection.close() self._connection = None self._cleanup_writer() self._notify_content() def release(self): if self._closed: return noop() self._closed = True if self._connection is not None: self._connection.release() self._connection = None self._cleanup_writer() self._notify_content() return noop() def raise_for_status(self): if 400 <= self.status: raise ClientResponseError(self.request_info, self.history, code=self.status, message=self.reason, headers=self.headers) def _cleanup_writer(self): if self._writer is not None: self._writer.cancel() self._writer = None self._session = None def _notify_content(self): content = self.content if content and content.exception() is None and not content.is_eof(): content.set_exception(ClientConnectionError('Connection closed')) async def wait_for_close(self): if self._writer is not None: try: await self._writer finally: self._writer = None self.release() async def read(self): """Read response payload.""" if self._content is None: try: self._content = await self.content.read() except Exception: self.close() raise return self._content def get_encoding(self): ctype = self.headers.get(hdrs.CONTENT_TYPE, '').lower() mimetype = helpers.parse_mimetype(ctype) encoding = mimetype.parameters.get('charset') if encoding: try: codecs.lookup(encoding) except LookupError: encoding = None if not encoding: if mimetype.type == 'application' and mimetype.subtype == 'json': # RFC 7159 states that the default encoding is UTF-8. encoding = 'utf-8' else: encoding = chardet.detect(self._content)['encoding'] if not encoding: encoding = 'utf-8' return encoding async def text(self, encoding=None, errors='strict'): """Read response payload and decode.""" if self._content is None: await self.read() if encoding is None: encoding = self.get_encoding() return self._content.decode(encoding, errors=errors) async def json(self, *, encoding=None, loads=json.loads, content_type='application/json'): """Read and decodes JSON response.""" if self._content is None: await self.read() if content_type: ctype = self.headers.get(hdrs.CONTENT_TYPE, '').lower() if content_type not in ctype: raise ContentTypeError(self.request_info, self.history, message=('Attempt to decode JSON with ' 'unexpected mimetype: %s' % ctype), headers=self.headers) stripped = self._content.strip() if not stripped: return None if encoding is None: encoding = self.get_encoding() return loads(stripped.decode(encoding)) async def __aenter__(self): return self async def __aexit__(self, exc_type, exc_val, exc_tb): # similar to _RequestContextManager, we do not need to check # for exceptions, response object can closes connection # is state is broken self.release()
class ClientResponse: # from the Status-Line of the response version = None # HTTP-Version status = None # Status-Code reason = None # Reason-Phrase cookies = None # Response cookies (Set-Cookie) content = None # Payload stream headers = None # Response headers, CIMultiDictProxy raw_headers = None # Response raw headers, a sequence of pairs _connection = None # current connection flow_control_class = FlowControlStreamReader # reader flow control _reader = None # input stream _response_parser = aiohttp.HttpResponseParser() _source_traceback = None # setted up by ClientRequest after ClientResponse object creation # post-init stage allows to not change ctor signature _loop = None _closed = True # to allow __del__ for non-initialized properly response def __init__(self, method, url, host="", *, writer=None, continue100=None): super().__init__() self.method = method self.url = url self.host = host self._content = None self._writer = writer self._continue = continue100 self._closed = False self._should_close = True # override by message.should_close later self._history = () def _post_init(self, loop): self._loop = loop if loop.get_debug(): self._source_traceback = traceback.extract_stack(sys._getframe(1)) def __del__(self, _warnings=warnings): if self._closed: return self.close() _warnings.warn("Unclosed response {!r}".format(self), ResourceWarning) context = {"client_response": self, "message": "Unclosed response"} if self._source_traceback: context["source_traceback"] = self._source_traceback self._loop.call_exception_handler(context) def __repr__(self): out = io.StringIO() print("<ClientResponse({}) [{} {}]>".format(self.url, self.status, self.reason), file=out) print(self.headers, file=out) return out.getvalue() @property def connection(self): return self._connection @property def history(self): """A sequence of of responses, if redirects occured.""" return self._history def waiting_for_continue(self): return self._continue is not None def _setup_connection(self, connection): self._reader = connection.reader self._connection = connection self.content = self.flow_control_class(connection.reader, loop=connection.loop) def _need_parse_response_body(self): return self.method.lower() != "head" and self.status not in [204, 304] @asyncio.coroutine def start(self, connection, read_until_eof=False): """Start response processing.""" self._setup_connection(connection) while True: httpstream = self._reader.set_parser(self._response_parser) # read response message = yield from httpstream.read() if message.code != 100: break if self._continue is not None and not self._continue.done(): self._continue.set_result(True) self._continue = None # response status self.version = message.version self.status = message.code self.reason = message.reason self._should_close = message.should_close # headers self.headers = CIMultiDictProxy(message.headers) self.raw_headers = tuple(message.raw_headers) # payload response_with_body = self._need_parse_response_body() self._reader.set_parser( aiohttp.HttpPayloadParser(message, readall=read_until_eof, response_with_body=response_with_body), self.content, ) # cookies self.cookies = http.cookies.SimpleCookie() if hdrs.SET_COOKIE in self.headers: for hdr in self.headers.getall(hdrs.SET_COOKIE): try: self.cookies.load(hdr) except http.cookies.CookieError as exc: client_logger.warning("Can not load response cookies: %s", exc) return self def close(self, force=True): if not force: warnings.warn("force parameter should be True", DeprecationWarning, stacklevel=2) if self._closed: return self._closed = True if hasattr(self._loop, "is_closed"): if self._loop.is_closed(): return if self._connection is not None: self._connection.close() self._connection = None self._cleanup_writer() @asyncio.coroutine def release(self): if self._closed: return try: content = self.content if content is not None and not content.at_eof(): chunk = yield from content.readany() while chunk is not EOF_MARKER or chunk: chunk = yield from content.readany() except Exception: self._connection.close() self._connection = None raise finally: self._closed = True if self._connection is not None: self._connection.release() if self._reader is not None: self._reader.unset_parser() self._connection = None self._cleanup_writer() def _cleanup_writer(self): if self._writer is not None and not self._writer.done(): self._writer.cancel() self._writer = None @asyncio.coroutine def wait_for_close(self): if self._writer is not None: try: yield from self._writer finally: self._writer = None yield from self.release() @asyncio.coroutine def read(self, decode=False): """Read response payload.""" if self._content is None: try: self._content = yield from self.content.read() except: self.close() raise else: yield from self.release() data = self._content if decode: warnings.warn(".read(True) is deprecated. use .json() instead", DeprecationWarning) return (yield from self.json()) return data def _get_encoding(self): ctype = self.headers.get(hdrs.CONTENT_TYPE, "").lower() mtype, stype, _, params = helpers.parse_mimetype(ctype) encoding = params.get("charset") if not encoding: encoding = chardet.detect(self._content)["encoding"] if not encoding: encoding = "utf-8" return encoding @asyncio.coroutine def text(self, encoding=None): """Read response payload and decode.""" if self._content is None: yield from self.read() if encoding is None: encoding = self._get_encoding() return self._content.decode(encoding) @asyncio.coroutine def json(self, *, encoding=None, loads=json.loads): """Read and decodes JSON response.""" if self._content is None: yield from self.read() ctype = self.headers.get(hdrs.CONTENT_TYPE, "").lower() if "json" not in ctype: client_logger.warning("Attempt to decode JSON with unexpected mimetype: %s", ctype) stripped = self._content.strip() if not stripped: return None if encoding is None: encoding = self._get_encoding() return loads(stripped.decode(encoding)) if PY_35: @asyncio.coroutine def __aenter__(self): return self @asyncio.coroutine def __aexit__(self, exc_type, exc_val, exc_tb): if exc_type is None: yield from self.release() else: self.close()
class ClientResponse(HeadersMixin): # from the Status-Line of the response version = None # HTTP-Version status = None # Status-Code reason = None # Reason-Phrase content = None # Payload stream headers = None # Response headers, CIMultiDictProxy raw_headers = None # Response raw headers, a sequence of pairs _connection = None # current connection flow_control_class = StreamReader # reader flow control _reader = None # input stream _source_traceback = None # setted up by ClientRequest after ClientResponse object creation # post-init stage allows to not change ctor signature _loop = None _closed = True # to allow __del__ for non-initialized properly response _session = None def __init__(self, method, url, *, writer=None, continue100=None, timer=None, request_info=None, auto_decompress=True): assert isinstance(url, URL) self.method = method self.headers = None self.cookies = SimpleCookie() self._url = url self._content = None self._writer = writer self._continue = continue100 self._closed = True self._history = () self._request_info = request_info self._timer = timer if timer is not None else TimerNoop() self._auto_decompress = auto_decompress self._cache = {} # reqired for @reify method decorator @property def url(self): return self._url @property def url_obj(self): warnings.warn( "Deprecated, use .url #1654", DeprecationWarning, stacklevel=2) return self._url @property def host(self): return self._url.host @property def _headers(self): return self.headers @property def request_info(self): return self._request_info @reify def content_disposition(self): raw = self._headers.get(hdrs.CONTENT_DISPOSITION) if raw is None: return None disposition_type, params = multipart.parse_content_disposition(raw) params = MappingProxyType(params) filename = multipart.content_disposition_filename(params) return ContentDisposition(disposition_type, params, filename) def _post_init(self, loop, session): self._loop = loop self._session = session # store a reference to session #1985 if loop.get_debug(): self._source_traceback = traceback.extract_stack(sys._getframe(1)) def __del__(self, _warnings=warnings): if self._loop is None: return # not started if self._closed: return if self._connection is not None: self._connection.release() self._cleanup_writer() if self._loop.get_debug(): if PY_36: kwargs = {'source': self} else: kwargs = {} _warnings.warn("Unclosed response {!r}".format(self), ResourceWarning, **kwargs) context = {'client_response': self, 'message': 'Unclosed response'} if self._source_traceback: context['source_traceback'] = self._source_traceback self._loop.call_exception_handler(context) def __repr__(self): out = io.StringIO() ascii_encodable_url = str(self.url) if self.reason: ascii_encodable_reason = self.reason.encode('ascii', 'backslashreplace') \ .decode('ascii') else: ascii_encodable_reason = self.reason print('<ClientResponse({}) [{} {}]>'.format( ascii_encodable_url, self.status, ascii_encodable_reason), file=out) print(self.headers, file=out) return out.getvalue() @property def connection(self): return self._connection @property def history(self): """A sequence of of responses, if redirects occurred.""" return self._history async def start(self, connection, read_until_eof=False): """Start response processing.""" self._closed = False self._protocol = connection.protocol self._connection = connection connection.protocol.set_response_params( timer=self._timer, skip_payload=self.method.lower() == 'head', skip_status_codes=(204, 304), read_until_eof=read_until_eof, auto_decompress=self._auto_decompress) with self._timer: while True: # read response try: (message, payload) = await self._protocol.read() except http.HttpProcessingError as exc: raise ClientResponseError( self.request_info, self.history, code=exc.code, message=exc.message, headers=exc.headers) from exc if (message.code < 100 or message.code > 199 or message.code == 101): break if self._continue is not None: set_result(self._continue, True) self._continue = None # payload eof handler payload.on_eof(self._response_eof) # response status self.version = message.version self.status = message.code self.reason = message.reason # headers self.headers = CIMultiDictProxy(message.headers) self.raw_headers = tuple(message.raw_headers) # payload self.content = payload # cookies for hdr in self.headers.getall(hdrs.SET_COOKIE, ()): try: self.cookies.load(hdr) except CookieError as exc: client_logger.warning( 'Can not load response cookies: %s', exc) return self def _response_eof(self): if self._closed: return if self._connection is not None: # websocket, protocol could be None because # connection could be detached if (self._connection.protocol is not None and self._connection.protocol.upgraded): return self._connection.release() self._connection = None self._closed = True self._cleanup_writer() @property def closed(self): return self._closed def close(self): if self._closed: return self._closed = True if self._loop is None or self._loop.is_closed(): return if self._connection is not None: self._connection.close() self._connection = None self._cleanup_writer() self._notify_content() def release(self): if self._closed: return noop() self._closed = True if self._connection is not None: self._connection.release() self._connection = None self._cleanup_writer() self._notify_content() return noop() def raise_for_status(self): if 400 <= self.status: raise ClientResponseError( self.request_info, self.history, code=self.status, message=self.reason, headers=self.headers) def _cleanup_writer(self): if self._writer is not None: self._writer.cancel() self._writer = None self._session = None def _notify_content(self): content = self.content if content and content.exception() is None and not content.is_eof(): content.set_exception( ClientConnectionError('Connection closed')) async def wait_for_close(self): if self._writer is not None: try: await self._writer finally: self._writer = None self.release() async def read(self): """Read response payload.""" if self._content is None: try: self._content = await self.content.read() except Exception: self.close() raise return self._content def get_encoding(self): ctype = self.headers.get(hdrs.CONTENT_TYPE, '').lower() mimetype = helpers.parse_mimetype(ctype) encoding = mimetype.parameters.get('charset') if encoding: try: codecs.lookup(encoding) except LookupError: encoding = None if not encoding: if mimetype.type == 'application' and mimetype.subtype == 'json': # RFC 7159 states that the default encoding is UTF-8. encoding = 'utf-8' else: encoding = chardet.detect(self._content)['encoding'] if not encoding: encoding = 'utf-8' return encoding async def text(self, encoding=None, errors='strict'): """Read response payload and decode.""" if self._content is None: await self.read() if encoding is None: encoding = self.get_encoding() return self._content.decode(encoding, errors=errors) async def json(self, *, encoding=None, loads=json.loads, content_type='application/json'): """Read and decodes JSON response.""" if self._content is None: await self.read() if content_type: ctype = self.headers.get(hdrs.CONTENT_TYPE, '').lower() if content_type not in ctype: raise ContentTypeError( self.request_info, self.history, message=('Attempt to decode JSON with ' 'unexpected mimetype: %s' % ctype), headers=self.headers) stripped = self._content.strip() if not stripped: return None if encoding is None: encoding = self.get_encoding() return loads(stripped.decode(encoding)) async def __aenter__(self): return self async def __aexit__(self, exc_type, exc_val, exc_tb): # similar to _RequestContextManager, we do not need to check # for exceptions, response object can closes connection # is state is broken self.release()