class ClientRequest: GET_METHODS = { hdrs.METH_GET, hdrs.METH_HEAD, hdrs.METH_OPTIONS, hdrs.METH_TRACE, } POST_METHODS = {hdrs.METH_PATCH, hdrs.METH_POST, hdrs.METH_PUT} ALL_METHODS = GET_METHODS.union(POST_METHODS).union({hdrs.METH_DELETE}) DEFAULT_HEADERS = { hdrs.ACCEPT: '*/*', hdrs.ACCEPT_ENCODING: 'gzip, deflate', } body = b'' auth = None response = None response_class = None _writer = None # async task for streaming data _continue = None # waiter future for '100 Continue' response # N.B. # Adding __del__ method with self._writer closing doesn't make sense # because _writer is instance method, thus it keeps a reference to self. # Until writer has finished finalizer will not be called. def __init__(self, method, url, *, params=None, headers=None, skip_auto_headers=frozenset(), data=None, cookies=None, auth=None, version=http.HttpVersion11, compress=None, chunked=None, expect100=False, loop=None, response_class=None, proxy=None, proxy_auth=None, timer=None, session=None, auto_decompress=True, verify_ssl=None, fingerprint=None, ssl_context=None, proxy_headers=None): if verify_ssl is False and ssl_context is not None: raise ValueError( "Either disable ssl certificate validation by " "verify_ssl=False or specify ssl_context, not both.") if loop is None: loop = asyncio.get_event_loop() assert isinstance(url, URL), url assert isinstance(proxy, (URL, type(None))), proxy self._session = session if params: q = MultiDict(url.query) url2 = url.with_query(params) q.extend(url2.query) url = url.with_query(q) self.url = url.with_fragment(None) self.original_url = url self.method = method.upper() self.chunked = chunked self.compress = compress self.loop = loop self.length = None self.response_class = response_class or ClientResponse self._timer = timer if timer is not None else TimerNoop() self._auto_decompress = auto_decompress self._verify_ssl = verify_ssl self._ssl_context = ssl_context if loop.get_debug(): self._source_traceback = traceback.extract_stack(sys._getframe(1)) self.update_version(version) self.update_host(url) self.update_headers(headers) self.update_auto_headers(skip_auto_headers) self.update_cookies(cookies) self.update_content_encoding(data) self.update_auth(auth) self.update_proxy(proxy, proxy_auth, proxy_headers) self.update_fingerprint(fingerprint) self.update_body_from_data(data) if data or self.method not in self.GET_METHODS: self.update_transfer_encoding() self.update_expect_continue(expect100) @property def connection_key(self): return ConnectionKey(self.host, self.port, self.ssl) @property def host(self): return self.url.host @property def port(self): return self.url.port @property def request_info(self): return RequestInfo(self.url, self.method, self.headers) def update_host(self, url): """Update destination host, port and connection type (ssl).""" # get host/port if not url.host: raise InvalidURL(url) # basic auth info username, password = url.user, url.password if username: self.auth = helpers.BasicAuth(username, password or '') # Record entire netloc for usage in host header scheme = url.scheme self.ssl = scheme in ('https', 'wss') def update_version(self, version): """Convert request version to two elements tuple. parser HTTP version '1.1' => (1, 1) """ if isinstance(version, str): v = [l.strip() for l in version.split('.', 1)] try: version = int(v[0]), int(v[1]) except ValueError: raise ValueError( 'Can not parse http version number: {}' .format(version)) from None self.version = version def update_headers(self, headers): """Update request headers.""" self.headers = CIMultiDict() if headers: if isinstance(headers, (dict, MultiDictProxy, MultiDict)): headers = headers.items() for key, value in headers: self.headers.add(key, value) def update_auto_headers(self, skip_auto_headers): self.skip_auto_headers = CIMultiDict( (hdr, None) for hdr in sorted(skip_auto_headers)) used_headers = self.headers.copy() used_headers.extend(self.skip_auto_headers) for hdr, val in self.DEFAULT_HEADERS.items(): if hdr not in used_headers: self.headers.add(hdr, val) # add host if hdrs.HOST not in used_headers: netloc = self.url.raw_host if not self.url.is_default_port(): netloc += ':' + str(self.url.port) self.headers[hdrs.HOST] = netloc if hdrs.USER_AGENT not in used_headers: self.headers[hdrs.USER_AGENT] = SERVER_SOFTWARE def update_cookies(self, cookies): """Update request cookies header.""" if not cookies: return c = SimpleCookie() if hdrs.COOKIE in self.headers: c.load(self.headers.get(hdrs.COOKIE, '')) del self.headers[hdrs.COOKIE] for name, value in cookies.items(): if isinstance(value, Morsel): # Preserve coded_value mrsl_val = value.get(value.key, Morsel()) mrsl_val.set(value.key, value.value, value.coded_value) c[name] = mrsl_val else: c[name] = value self.headers[hdrs.COOKIE] = c.output(header='', sep=';').strip() def update_content_encoding(self, data): """Set request content encoding.""" if not data: return enc = self.headers.get(hdrs.CONTENT_ENCODING, '').lower() if enc: if self.compress: raise ValueError( 'compress can not be set ' 'if Content-Encoding header is set') elif self.compress: if not isinstance(self.compress, str): self.compress = 'deflate' self.headers[hdrs.CONTENT_ENCODING] = self.compress self.chunked = True # enable chunked, no need to deal with length def update_transfer_encoding(self): """Analyze transfer-encoding header.""" te = self.headers.get(hdrs.TRANSFER_ENCODING, '').lower() if 'chunked' in te: if self.chunked: raise ValueError( 'chunked can not be set ' 'if "Transfer-Encoding: chunked" header is set') elif self.chunked: if hdrs.CONTENT_LENGTH in self.headers: raise ValueError( 'chunked can not be set ' 'if Content-Length header is set') self.headers[hdrs.TRANSFER_ENCODING] = 'chunked' else: if hdrs.CONTENT_LENGTH not in self.headers: self.headers[hdrs.CONTENT_LENGTH] = str(len(self.body)) def update_auth(self, auth): """Set basic auth.""" if auth is None: auth = self.auth if auth is None: return if not isinstance(auth, helpers.BasicAuth): raise TypeError('BasicAuth() tuple is required instead') self.headers[hdrs.AUTHORIZATION] = auth.encode() def update_body_from_data(self, body): if not body: return # FormData if isinstance(body, FormData): body = body() try: body = payload.PAYLOAD_REGISTRY.get(body, disposition=None) except payload.LookupError: body = FormData(body)() self.body = body # enable chunked encoding if needed if not self.chunked: if hdrs.CONTENT_LENGTH not in self.headers: size = body.size if size is None: self.chunked = True else: if hdrs.CONTENT_LENGTH not in self.headers: self.headers[hdrs.CONTENT_LENGTH] = str(size) # set content-type if (hdrs.CONTENT_TYPE not in self.headers and hdrs.CONTENT_TYPE not in self.skip_auto_headers): self.headers[hdrs.CONTENT_TYPE] = body.content_type # copy payload headers if body.headers: for (key, value) in body.headers.items(): if key not in self.headers: self.headers[key] = value def update_expect_continue(self, expect=False): if expect: self.headers[hdrs.EXPECT] = '100-continue' elif self.headers.get(hdrs.EXPECT, '').lower() == '100-continue': expect = True if expect: self._continue = helpers.create_future(self.loop) def update_proxy(self, proxy, proxy_auth, proxy_headers): if proxy and not proxy.scheme == 'http': raise ValueError("Only http proxies are supported") if proxy_auth and not isinstance(proxy_auth, helpers.BasicAuth): raise ValueError("proxy_auth must be None or BasicAuth() tuple") self.proxy = proxy self.proxy_auth = proxy_auth self.proxy_headers = proxy_headers def update_fingerprint(self, fingerprint): if fingerprint: digestlen = len(fingerprint) hashfunc = HASHFUNC_BY_DIGESTLEN.get(digestlen) if not hashfunc: raise ValueError('fingerprint has invalid length') elif hashfunc is md5 or hashfunc is sha1: warnings.warn('md5 and sha1 are insecure and deprecated. ' 'Use sha256.', DeprecationWarning, stacklevel=2) client_logger.warn('md5 and sha1 are insecure and deprecated. ' 'Use sha256.') self._hashfunc = hashfunc self._fingerprint = fingerprint @property def verify_ssl(self): """Do check for ssl certifications?""" return self._verify_ssl @property def fingerprint(self): """Expected ssl certificate fingerprint.""" return self._fingerprint @property def ssl_context(self): """SSLContext instance for https requests.""" return self._ssl_context def keep_alive(self): if self.version < HttpVersion10: # keep alive not supported at all return False if self.version == HttpVersion10: if self.headers.get(hdrs.CONNECTION) == 'keep-alive': return True else: # no headers means we close for Http 1.0 return False elif self.headers.get(hdrs.CONNECTION) == 'close': return False return True @asyncio.coroutine def write_bytes(self, writer, conn): """Support coroutines that yields bytes objects.""" # 100 response if self._continue is not None: yield from writer.drain() yield from self._continue try: if isinstance(self.body, payload.Payload): yield from self.body.write(writer) else: if isinstance(self.body, (bytes, bytearray)): self.body = (self.body,) for chunk in self.body: writer.write(chunk) yield from writer.write_eof() except OSError as exc: new_exc = ClientOSError( exc.errno, 'Can not write request body for %s' % self.url) new_exc.__context__ = exc new_exc.__cause__ = exc conn.protocol.set_exception(new_exc) except asyncio.CancelledError as exc: if not conn.closed: conn.protocol.set_exception(exc) except Exception as exc: conn.protocol.set_exception(exc) finally: self._writer = None def send(self, conn): # Specify request target: # - CONNECT request must send authority form URI # - not CONNECT proxy must send absolute form URI # - most common is origin form URI if self.method == hdrs.METH_CONNECT: path = '{}:{}'.format(self.url.raw_host, self.url.port) elif self.proxy and not self.ssl: path = str(self.url) else: path = self.url.raw_path if self.url.raw_query_string: path += '?' + self.url.raw_query_string writer = PayloadWriter(conn.writer, self.loop) if self.compress: writer.enable_compression(self.compress) if self.chunked is not None: writer.enable_chunking() # set default content-type if (self.method in self.POST_METHODS and hdrs.CONTENT_TYPE not in self.skip_auto_headers and hdrs.CONTENT_TYPE not in self.headers): self.headers[hdrs.CONTENT_TYPE] = 'application/octet-stream' # set the connection header connection = self.headers.get(hdrs.CONNECTION) if not connection: if self.keep_alive(): if self.version == HttpVersion10: connection = 'keep-alive' else: if self.version == HttpVersion11: connection = 'close' if connection is not None: self.headers[hdrs.CONNECTION] = connection # status + headers status_line = '{0} {1} HTTP/{2[0]}.{2[1]}\r\n'.format( self.method, path, self.version) writer.write_headers(status_line, self.headers) self._writer = asyncio.ensure_future( self.write_bytes(writer, conn), loop=self.loop) self.response = self.response_class( self.method, self.original_url, writer=self._writer, continue100=self._continue, timer=self._timer, request_info=self.request_info, auto_decompress=self._auto_decompress ) self.response._post_init(self.loop, self._session) return self.response @asyncio.coroutine def close(self): if self._writer is not None: try: yield from self._writer finally: self._writer = None def terminate(self): if self._writer is not None: if not self.loop.is_closed(): self._writer.cancel() self._writer = None
class ClientRequest: GET_METHODS = { hdrs.METH_GET, hdrs.METH_HEAD, hdrs.METH_OPTIONS, hdrs.METH_TRACE, } POST_METHODS = {hdrs.METH_PATCH, hdrs.METH_POST, hdrs.METH_PUT} ALL_METHODS = GET_METHODS.union(POST_METHODS).union({hdrs.METH_DELETE}) DEFAULT_HEADERS = { hdrs.ACCEPT: '*/*', hdrs.ACCEPT_ENCODING: 'gzip, deflate', } body = b'' auth = None response = None response_class = None _writer = None # async task for streaming data _continue = None # waiter future for '100 Continue' response # N.B. # Adding __del__ method with self._writer closing doesn't make sense # because _writer is instance method, thus it keeps a reference to self. # Until writer has finished finalizer will not be called. def __init__(self, method: str, url: URL, *, params: Optional[Mapping[str, str]] = None, headers: Optional[LooseHeaders] = None, skip_auto_headers: Iterable[str] = frozenset(), data: Any = None, cookies: Optional[LooseCookies] = None, auth: Optional[BasicAuth] = None, version: http.HttpVersion = http.HttpVersion11, compress: Optional[str] = None, chunked: Optional[bool] = None, expect100: bool = False, loop: Optional[asyncio.AbstractEventLoop] = None, response_class: Optional[Type['ClientResponse']] = None, proxy: Optional[URL] = None, proxy_auth: Optional[BasicAuth] = None, timer: Optional[BaseTimerContext] = None, session: Optional['ClientSession'] = None, ssl: Union[SSLContext, bool, Fingerprint, None] = None, proxy_headers: Optional[LooseHeaders] = None, traces: Optional[List['Trace']] = None): if loop is None: loop = asyncio.get_event_loop() assert isinstance(url, URL), url assert isinstance(proxy, (URL, type(None))), proxy # FIXME: session is None in tests only, need to fix tests # assert session is not None self._session = cast('ClientSession', session) if params: q = MultiDict(url.query) url2 = url.with_query(params) q.extend(url2.query) url = url.with_query(q) self.original_url = url self.url = url.with_fragment(None) self.method = method.upper() self.chunked = chunked self.compress = compress self.loop = loop self.length = None if response_class is None: real_response_class = ClientResponse else: real_response_class = response_class self.response_class = real_response_class # type: Type[ClientResponse] self._timer = timer if timer is not None else TimerNoop() self._ssl = ssl if loop.get_debug(): self._source_traceback = traceback.extract_stack(sys._getframe(1)) self.update_version(version) self.update_host(url) self.update_headers(headers) self.update_auto_headers(skip_auto_headers) self.update_cookies(cookies) self.update_content_encoding(data) self.update_auth(auth) self.update_proxy(proxy, proxy_auth, proxy_headers) self.update_body_from_data(data) if data or self.method not in self.GET_METHODS: self.update_transfer_encoding() self.update_expect_continue(expect100) if traces is None: traces = [] self._traces = traces def is_ssl(self) -> bool: return self.url.scheme in ('https', 'wss') @property def ssl(self) -> Union['SSLContext', None, bool, Fingerprint]: return self._ssl @property def connection_key(self) -> ConnectionKey: proxy_headers = self.proxy_headers if proxy_headers: h = hash(tuple( (k, v) for k, v in proxy_headers.items())) # type: Optional[int] # noqa else: h = None return ConnectionKey(self.host, self.port, self.is_ssl(), self.ssl, self.proxy, self.proxy_auth, h) @property def host(self) -> str: ret = self.url.host assert ret is not None return ret @property def port(self) -> Optional[int]: return self.url.port @property def request_info(self) -> RequestInfo: headers = CIMultiDictProxy(self.headers) # type: CIMultiDictProxy[str] return RequestInfo(self.url, self.method, headers, self.original_url) def update_host(self, url: URL) -> None: """Update destination host, port and connection type (ssl).""" # get host/port if not url.host: raise InvalidURL(url) # basic auth info username, password = url.user, url.password if username: self.auth = helpers.BasicAuth(username, password or '') def update_version(self, version: Union[http.HttpVersion, str]) -> None: """Convert request version to two elements tuple. parser HTTP version '1.1' => (1, 1) """ if isinstance(version, str): v = [l.strip() for l in version.split('.', 1)] try: version = http.HttpVersion(int(v[0]), int(v[1])) except ValueError: raise ValueError( 'Can not parse http version number: {}'.format( version)) from None self.version = version def update_headers(self, headers: Optional[LooseHeaders]) -> None: """Update request headers.""" self.headers = CIMultiDict() # type: CIMultiDict[str] # add host netloc = cast(str, self.url.raw_host) if helpers.is_ipv6_address(netloc): netloc = '[{}]'.format(netloc) if not self.url.is_default_port(): netloc += ':' + str(self.url.port) self.headers[hdrs.HOST] = netloc if headers: if isinstance(headers, (dict, MultiDictProxy, MultiDict)): headers = headers.items() # type: ignore for key, value in headers: # A special case for Host header if key.lower() == 'host': self.headers[key] = value else: self.headers.add(key, value) def update_auto_headers(self, skip_auto_headers: Iterable[str]) -> None: self.skip_auto_headers = CIMultiDict( (hdr, None) for hdr in sorted(skip_auto_headers)) used_headers = self.headers.copy() used_headers.extend(self.skip_auto_headers) # type: ignore for hdr, val in self.DEFAULT_HEADERS.items(): if hdr not in used_headers: self.headers.add(hdr, val) if hdrs.USER_AGENT not in used_headers: self.headers[hdrs.USER_AGENT] = SERVER_SOFTWARE def update_cookies(self, cookies: Optional[LooseCookies]) -> None: """Update request cookies header.""" if not cookies: return c = SimpleCookie() if hdrs.COOKIE in self.headers: c.load(self.headers.get(hdrs.COOKIE, '')) del self.headers[hdrs.COOKIE] if isinstance(cookies, Mapping): iter_cookies = cookies.items() else: iter_cookies = cookies # type: ignore for name, value in iter_cookies: if isinstance(value, Morsel): # Preserve coded_value mrsl_val = value.get(value.key, Morsel()) mrsl_val.set(value.key, value.value, value.coded_value) # type: ignore # noqa c[name] = mrsl_val else: c[name] = value # type: ignore self.headers[hdrs.COOKIE] = c.output(header='', sep=';').strip() def update_content_encoding(self, data: Any) -> None: """Set request content encoding.""" if not data: return enc = self.headers.get(hdrs.CONTENT_ENCODING, '').lower() if enc: if self.compress: raise ValueError('compress can not be set ' 'if Content-Encoding header is set') elif self.compress: if not isinstance(self.compress, str): self.compress = 'deflate' self.headers[hdrs.CONTENT_ENCODING] = self.compress self.chunked = True # enable chunked, no need to deal with length def update_transfer_encoding(self) -> None: """Analyze transfer-encoding header.""" te = self.headers.get(hdrs.TRANSFER_ENCODING, '').lower() if 'chunked' in te: if self.chunked: raise ValueError( 'chunked can not be set ' 'if "Transfer-Encoding: chunked" header is set') elif self.chunked: if hdrs.CONTENT_LENGTH in self.headers: raise ValueError('chunked can not be set ' 'if Content-Length header is set') self.headers[hdrs.TRANSFER_ENCODING] = 'chunked' else: if hdrs.CONTENT_LENGTH not in self.headers: self.headers[hdrs.CONTENT_LENGTH] = str(len(self.body)) def update_auth(self, auth: Optional[BasicAuth]) -> None: """Set basic auth.""" if auth is None: auth = self.auth if auth is None: return if not isinstance(auth, helpers.BasicAuth): raise TypeError('BasicAuth() tuple is required instead') self.headers[hdrs.AUTHORIZATION] = auth.encode() def update_body_from_data(self, body: Any) -> None: if not body: return # FormData if isinstance(body, FormData): body = body() try: body = payload.PAYLOAD_REGISTRY.get(body, disposition=None) except payload.LookupError: body = FormData(body)() self.body = body # enable chunked encoding if needed if not self.chunked: if hdrs.CONTENT_LENGTH not in self.headers: size = body.size if size is None: self.chunked = True else: if hdrs.CONTENT_LENGTH not in self.headers: self.headers[hdrs.CONTENT_LENGTH] = str(size) # set content-type if (hdrs.CONTENT_TYPE not in self.headers and hdrs.CONTENT_TYPE not in self.skip_auto_headers): self.headers[hdrs.CONTENT_TYPE] = body.content_type # copy payload headers if body.headers: for (key, value) in body.headers.items(): if key not in self.headers: self.headers[key] = value def update_expect_continue(self, expect: bool = False) -> None: if expect: self.headers[hdrs.EXPECT] = '100-continue' elif self.headers.get(hdrs.EXPECT, '').lower() == '100-continue': expect = True if expect: self._continue = self.loop.create_future() def update_proxy(self, proxy: Optional[URL], proxy_auth: Optional[BasicAuth], proxy_headers: Optional[LooseHeaders]) -> None: if proxy and not proxy.scheme == 'http': raise ValueError("Only http proxies are supported") if proxy_auth and not isinstance(proxy_auth, helpers.BasicAuth): raise ValueError("proxy_auth must be None or BasicAuth() tuple") self.proxy = proxy self.proxy_auth = proxy_auth self.proxy_headers = proxy_headers def keep_alive(self) -> bool: if self.version < HttpVersion10: # keep alive not supported at all return False if self.version == HttpVersion10: if self.headers.get(hdrs.CONNECTION) == 'keep-alive': return True else: # no headers means we close for Http 1.0 return False elif self.headers.get(hdrs.CONNECTION) == 'close': return False return True async def write_bytes(self, writer: AbstractStreamWriter, conn: 'Connection') -> None: """Support coroutines that yields bytes objects.""" # 100 response if self._continue is not None: await writer.drain() await self._continue protocol = conn.protocol assert protocol is not None try: if isinstance(self.body, payload.Payload): await self.body.write(writer) else: if isinstance(self.body, (bytes, bytearray)): self.body = (self.body, ) # type: ignore for chunk in self.body: await writer.write(chunk) # type: ignore await writer.write_eof() except OSError as exc: new_exc = ClientOSError( exc.errno, 'Can not write request body for %s' % self.url) new_exc.__context__ = exc new_exc.__cause__ = exc protocol.set_exception(new_exc) except asyncio.CancelledError as exc: if not conn.closed: protocol.set_exception(exc) except Exception as exc: protocol.set_exception(exc) finally: self._writer = None async def send(self, conn: 'Connection') -> 'ClientResponse': # Specify request target: # - CONNECT request must send authority form URI # - not CONNECT proxy must send absolute form URI # - most common is origin form URI if self.method == hdrs.METH_CONNECT: path = '{}:{}'.format(self.url.raw_host, self.url.port) elif self.proxy and not self.is_ssl(): path = str(self.url) else: path = self.url.raw_path if self.url.raw_query_string: path += '?' + self.url.raw_query_string protocol = conn.protocol assert protocol is not None writer = StreamWriter(protocol, self.loop, on_chunk_sent=self._on_chunk_request_sent) if self.compress: writer.enable_compression(self.compress) if self.chunked is not None: writer.enable_chunking() # set default content-type if (self.method in self.POST_METHODS and hdrs.CONTENT_TYPE not in self.skip_auto_headers and hdrs.CONTENT_TYPE not in self.headers): self.headers[hdrs.CONTENT_TYPE] = 'application/octet-stream' # set the connection header connection = self.headers.get(hdrs.CONNECTION) if not connection: if self.keep_alive(): if self.version == HttpVersion10: connection = 'keep-alive' else: if self.version == HttpVersion11: connection = 'close' if connection is not None: self.headers[hdrs.CONNECTION] = connection # status + headers status_line = '{0} {1} HTTP/{2[0]}.{2[1]}'.format( self.method, path, self.version) await writer.write_headers(status_line, self.headers) self._writer = self.loop.create_task(self.write_bytes(writer, conn)) response_class = self.response_class assert response_class is not None self.response = response_class(self.method, self.original_url, writer=self._writer, continue100=self._continue, timer=self._timer, request_info=self.request_info, traces=self._traces, loop=self.loop, session=self._session) return self.response async def close(self) -> None: if self._writer is not None: try: await self._writer finally: self._writer = None def terminate(self) -> None: if self._writer is not None: if not self.loop.is_closed(): self._writer.cancel() self._writer = None async def _on_chunk_request_sent(self, chunk: bytes) -> None: for trace in self._traces: await trace.send_request_chunk_sent(chunk)
class ClientRequest: GET_METHODS = { hdrs.METH_GET, hdrs.METH_HEAD, hdrs.METH_OPTIONS, hdrs.METH_TRACE, } POST_METHODS = {hdrs.METH_PATCH, hdrs.METH_POST, hdrs.METH_PUT} ALL_METHODS = GET_METHODS.union(POST_METHODS).union({hdrs.METH_DELETE}) DEFAULT_HEADERS = { hdrs.ACCEPT: '*/*', hdrs.ACCEPT_ENCODING: 'gzip, deflate', } body = b'' auth = None response = None response_class = None _writer = None # async task for streaming data _continue = None # waiter future for '100 Continue' response # N.B. # Adding __del__ method with self._writer closing doesn't make sense # because _writer is instance method, thus it keeps a reference to self. # Until writer has finished finalizer will not be called. def __init__(self, method: str, url: URL, *, params: Optional[Mapping[str, str]]=None, headers: Optional[LooseHeaders]=None, skip_auto_headers: Iterable[str]=frozenset(), data: Any=None, cookies: Optional[LooseCookies]=None, auth: Optional[BasicAuth]=None, version: http.HttpVersion=http.HttpVersion11, compress: Optional[str]=None, chunked: Optional[bool]=None, expect100: bool=False, loop: Optional[asyncio.AbstractEventLoop]=None, response_class: Optional[Type['ClientResponse']]=None, proxy: Optional[URL]=None, proxy_auth: Optional[BasicAuth]=None, timer: Optional[BaseTimerContext]=None, session: Optional['ClientSession']=None, ssl: Union[SSLContext, bool, Fingerprint, None]=None, proxy_headers: Optional[LooseHeaders]=None, traces: Optional[List['Trace']]=None): if loop is None: loop = asyncio.get_event_loop() assert isinstance(url, URL), url assert isinstance(proxy, (URL, type(None))), proxy # FIXME: session is None in tests only, need to fix tests # assert session is not None self._session = cast('ClientSession', session) if params: q = MultiDict(url.query) url2 = url.with_query(params) q.extend(url2.query) url = url.with_query(q) self.original_url = url self.url = url.with_fragment(None) self.method = method.upper() self.chunked = chunked self.compress = compress self.loop = loop self.length = None if response_class is None: real_response_class = ClientResponse else: real_response_class = response_class self.response_class = real_response_class # type: Type[ClientResponse] self._timer = timer if timer is not None else TimerNoop() self._ssl = ssl if loop.get_debug(): self._source_traceback = traceback.extract_stack(sys._getframe(1)) self.update_version(version) self.update_host(url) self.update_headers(headers) self.update_auto_headers(skip_auto_headers) self.update_cookies(cookies) self.update_content_encoding(data) self.update_auth(auth) self.update_proxy(proxy, proxy_auth, proxy_headers) self.update_body_from_data(data) if data or self.method not in self.GET_METHODS: self.update_transfer_encoding() self.update_expect_continue(expect100) if traces is None: traces = [] self._traces = traces def is_ssl(self) -> bool: return self.url.scheme in ('https', 'wss') @property def ssl(self) -> Union['SSLContext', None, bool, Fingerprint]: return self._ssl @property def connection_key(self) -> ConnectionKey: proxy_headers = self.proxy_headers if proxy_headers: h = hash(tuple((k, v) for k, v in proxy_headers.items())) # type: Optional[int] # noqa else: h = None return ConnectionKey(self.host, self.port, self.is_ssl(), self.ssl, self.proxy, self.proxy_auth, h) @property def host(self) -> str: ret = self.url.host assert ret is not None return ret @property def port(self) -> Optional[int]: return self.url.port @property def request_info(self) -> RequestInfo: headers = CIMultiDictProxy(self.headers) # type: CIMultiDictProxy[str] return RequestInfo(self.url, self.method, headers, self.original_url) def update_host(self, url: URL) -> None: """Update destination host, port and connection type (ssl).""" # get host/port if not url.host: raise InvalidURL(url) # basic auth info username, password = url.user, url.password if username: self.auth = helpers.BasicAuth(username, password or '') def update_version(self, version: Union[http.HttpVersion, str]) -> None: """Convert request version to two elements tuple. parser HTTP version '1.1' => (1, 1) """ if isinstance(version, str): v = [l.strip() for l in version.split('.', 1)] try: version = http.HttpVersion(int(v[0]), int(v[1])) except ValueError: raise ValueError( 'Can not parse http version number: {}' .format(version)) from None self.version = version def update_headers(self, headers: Optional[LooseHeaders]) -> None: """Update request headers.""" self.headers = CIMultiDict() # type: CIMultiDict[str] # add host netloc = cast(str, self.url.raw_host) if helpers.is_ipv6_address(netloc): netloc = '[{}]'.format(netloc) if not self.url.is_default_port(): netloc += ':' + str(self.url.port) self.headers[hdrs.HOST] = netloc if headers: if isinstance(headers, (dict, MultiDictProxy, MultiDict)): headers = headers.items() # type: ignore for key, value in headers: # A special case for Host header if key.lower() == 'host': self.headers[key] = value else: self.headers.add(key, value) def update_auto_headers(self, skip_auto_headers: Iterable[str]) -> None: self.skip_auto_headers = CIMultiDict( (hdr, None) for hdr in sorted(skip_auto_headers)) used_headers = self.headers.copy() used_headers.extend(self.skip_auto_headers) # type: ignore for hdr, val in self.DEFAULT_HEADERS.items(): if hdr not in used_headers: self.headers.add(hdr, val) if hdrs.USER_AGENT not in used_headers: self.headers[hdrs.USER_AGENT] = SERVER_SOFTWARE def update_cookies(self, cookies: Optional[LooseCookies]) -> None: """Update request cookies header.""" if not cookies: return c = SimpleCookie() if hdrs.COOKIE in self.headers: c.load(self.headers.get(hdrs.COOKIE, '')) del self.headers[hdrs.COOKIE] if isinstance(cookies, Mapping): iter_cookies = cookies.items() else: iter_cookies = cookies # type: ignore for name, value in iter_cookies: if isinstance(value, Morsel): # Preserve coded_value mrsl_val = value.get(value.key, Morsel()) mrsl_val.set(value.key, value.value, value.coded_value) # type: ignore # noqa c[name] = mrsl_val else: c[name] = value # type: ignore self.headers[hdrs.COOKIE] = c.output(header='', sep=';').strip() def update_content_encoding(self, data: Any) -> None: """Set request content encoding.""" if not data: return enc = self.headers.get(hdrs.CONTENT_ENCODING, '').lower() if enc: if self.compress: raise ValueError( 'compress can not be set ' 'if Content-Encoding header is set') elif self.compress: if not isinstance(self.compress, str): self.compress = 'deflate' self.headers[hdrs.CONTENT_ENCODING] = self.compress self.chunked = True # enable chunked, no need to deal with length def update_transfer_encoding(self) -> None: """Analyze transfer-encoding header.""" te = self.headers.get(hdrs.TRANSFER_ENCODING, '').lower() if 'chunked' in te: if self.chunked: raise ValueError( 'chunked can not be set ' 'if "Transfer-Encoding: chunked" header is set') elif self.chunked: if hdrs.CONTENT_LENGTH in self.headers: raise ValueError( 'chunked can not be set ' 'if Content-Length header is set') self.headers[hdrs.TRANSFER_ENCODING] = 'chunked' else: if hdrs.CONTENT_LENGTH not in self.headers: self.headers[hdrs.CONTENT_LENGTH] = str(len(self.body)) def update_auth(self, auth: Optional[BasicAuth]) -> None: """Set basic auth.""" if auth is None: auth = self.auth if auth is None: return if not isinstance(auth, helpers.BasicAuth): raise TypeError('BasicAuth() tuple is required instead') self.headers[hdrs.AUTHORIZATION] = auth.encode() def update_body_from_data(self, body: Any) -> None: if not body: return # FormData if isinstance(body, FormData): body = body() try: body = payload.PAYLOAD_REGISTRY.get(body, disposition=None) except payload.LookupError: body = FormData(body)() self.body = body # enable chunked encoding if needed if not self.chunked: if hdrs.CONTENT_LENGTH not in self.headers: size = body.size if size is None: self.chunked = True else: if hdrs.CONTENT_LENGTH not in self.headers: self.headers[hdrs.CONTENT_LENGTH] = str(size) # set content-type if (hdrs.CONTENT_TYPE not in self.headers and hdrs.CONTENT_TYPE not in self.skip_auto_headers): self.headers[hdrs.CONTENT_TYPE] = body.content_type # copy payload headers if body.headers: for (key, value) in body.headers.items(): if key not in self.headers: self.headers[key] = value def update_expect_continue(self, expect: bool=False) -> None: if expect: self.headers[hdrs.EXPECT] = '100-continue' elif self.headers.get(hdrs.EXPECT, '').lower() == '100-continue': expect = True if expect: self._continue = self.loop.create_future() def update_proxy(self, proxy: Optional[URL], proxy_auth: Optional[BasicAuth], proxy_headers: Optional[LooseHeaders]) -> None: if proxy and not proxy.scheme == 'http': raise ValueError("Only http proxies are supported") if proxy_auth and not isinstance(proxy_auth, helpers.BasicAuth): raise ValueError("proxy_auth must be None or BasicAuth() tuple") self.proxy = proxy self.proxy_auth = proxy_auth self.proxy_headers = proxy_headers def keep_alive(self) -> bool: if self.version < HttpVersion10: # keep alive not supported at all return False if self.version == HttpVersion10: if self.headers.get(hdrs.CONNECTION) == 'keep-alive': return True else: # no headers means we close for Http 1.0 return False elif self.headers.get(hdrs.CONNECTION) == 'close': return False return True async def write_bytes(self, writer: AbstractStreamWriter, conn: 'Connection') -> None: """Support coroutines that yields bytes objects.""" # 100 response if self._continue is not None: await writer.drain() await self._continue protocol = conn.protocol assert protocol is not None try: if isinstance(self.body, payload.Payload): await self.body.write(writer) else: if isinstance(self.body, (bytes, bytearray)): self.body = (self.body,) # type: ignore for chunk in self.body: await writer.write(chunk) # type: ignore await writer.write_eof() except OSError as exc: new_exc = ClientOSError( exc.errno, 'Can not write request body for %s' % self.url) new_exc.__context__ = exc new_exc.__cause__ = exc protocol.set_exception(new_exc) except asyncio.CancelledError as exc: if not conn.closed: protocol.set_exception(exc) except Exception as exc: protocol.set_exception(exc) finally: self._writer = None async def send(self, conn: 'Connection') -> 'ClientResponse': # Specify request target: # - CONNECT request must send authority form URI # - not CONNECT proxy must send absolute form URI # - most common is origin form URI if self.method == hdrs.METH_CONNECT: path = '{}:{}'.format(self.url.raw_host, self.url.port) elif self.proxy and not self.is_ssl(): path = str(self.url) else: path = self.url.raw_path if self.url.raw_query_string: path += '?' + self.url.raw_query_string protocol = conn.protocol assert protocol is not None writer = StreamWriter( protocol, self.loop, on_chunk_sent=self._on_chunk_request_sent ) if self.compress: writer.enable_compression(self.compress) if self.chunked is not None: writer.enable_chunking() # set default content-type if (self.method in self.POST_METHODS and hdrs.CONTENT_TYPE not in self.skip_auto_headers and hdrs.CONTENT_TYPE not in self.headers): self.headers[hdrs.CONTENT_TYPE] = 'application/octet-stream' # set the connection header connection = self.headers.get(hdrs.CONNECTION) if not connection: if self.keep_alive(): if self.version == HttpVersion10: connection = 'keep-alive' else: if self.version == HttpVersion11: connection = 'close' if connection is not None: self.headers[hdrs.CONNECTION] = connection # status + headers status_line = '{0} {1} HTTP/{2[0]}.{2[1]}'.format( self.method, path, self.version) await writer.write_headers(status_line, self.headers) self._writer = self.loop.create_task(self.write_bytes(writer, conn)) response_class = self.response_class assert response_class is not None self.response = response_class( self.method, self.original_url, writer=self._writer, continue100=self._continue, timer=self._timer, request_info=self.request_info, traces=self._traces, loop=self.loop, session=self._session ) return self.response async def close(self) -> None: if self._writer is not None: try: await self._writer finally: self._writer = None def terminate(self) -> None: if self._writer is not None: if not self.loop.is_closed(): self._writer.cancel() self._writer = None async def _on_chunk_request_sent(self, chunk: bytes) -> None: for trace in self._traces: await trace.send_request_chunk_sent(chunk)
class HttpRequest(RequestBase): """An :class:`HttpClient` request for an HTTP resource. This class has a similar interface to :class:`urllib.request.Request`. :param files: optional dictionary of name, file-like-objects. :param allow_redirects: allow the response to follow redirects. .. attribute:: method The request method .. attribute:: version HTTP version for this request, usually ``HTTP/1.1`` .. attribute:: history List of past :class:`.HttpResponse` (collected during redirects). .. attribute:: wait_continue if ``True``, the :class:`HttpRequest` includes the ``Expect: 100-Continue`` header. .. attribute:: stream Allow for streaming body """ _proxy = None _ssl = None _tunnel = None _write_done = False def __init__(self, client, url, method, inp_params=None, headers=None, data=None, files=None, json=None, history=None, auth=None, charset=None, max_redirects=10, source_address=None, allow_redirects=False, decompress=True, version=None, wait_continue=False, websocket_handler=None, cookies=None, params=None, stream=False, proxies=None, verify=True, cert=None, **extra): self.client = client self.method = method.upper() self.inp_params = inp_params or {} self.unredirected_headers = CIMultiDict() self.history = history self.wait_continue = wait_continue self.max_redirects = max_redirects self.allow_redirects = allow_redirects self.charset = charset or 'utf-8' self.version = version self.decompress = decompress self.websocket_handler = websocket_handler self.source_address = source_address self.stream = stream self.verify = verify self.cert = cert if auth and not isinstance(auth, Auth): auth = HTTPBasicAuth(*auth) self.auth = auth self.url = full_url(url, params, method=self.method) self._set_proxy(proxies) self.key = RequestKey.create(self) self.headers = client.get_headers(self, headers) self.body = self._encode_body(data, files, json) self.unredirected_headers['host'] = self.key.netloc cookies = cookiejar_from_dict(client.cookies, cookies) if cookies: cookies.add_cookie_header(self) @property def _loop(self): return self.client._loop @property def ssl(self): """Context for TLS connections. If this is a tunneled request and the tunnel connection is not yet established, it returns ``None``. """ return self._ssl @property def proxy(self): """Proxy server for this request. """ return self._proxy @property def tunnel(self): """Tunnel for this request. """ return self._tunnel def __repr__(self): return self.first_line() __str__ = __repr__ def first_line(self): if self.method == 'CONNECT': url = self.key.netloc elif self._proxy: url = self.url else: p = urlparse(self.url) url = urlunparse(('', '', p.path or '/', p.params, p.query, p.fragment)) return '%s %s %s' % (self.method, url, self.version) def is_chunked(self): return self.body and 'content-length' not in self.headers def encode(self): """The bytes representation of this :class:`HttpRequest`. Called by :class:`HttpResponse` when it needs to encode this :class:`HttpRequest` before sending it to the HTTP resource. """ # Call body before fist_line in case the query is changes. first_line = self.first_line() if self.body and self.wait_continue: self.headers['expect'] = '100-continue' headers = self.headers if self.unredirected_headers: headers = self.unredirected_headers.copy() headers.update(self.headers) buffer = [first_line.encode('ascii'), b'\r\n'] buffer.extend((('%s: %s\r\n' % (name, value)).encode(CHARSET) for name, value in headers.items())) buffer.append(b'\r\n') return b''.join(buffer) def add_header(self, key, value): self.headers[key] = value def has_header(self, header_name): """Check ``header_name`` is in this request headers. """ return (header_name in self.headers or header_name in self.unredirected_headers) def get_header(self, header_name, default=None): """Retrieve ``header_name`` from this request headers. """ return self.headers.get( header_name, self.unredirected_headers.get(header_name, default)) def remove_header(self, header_name): """Remove ``header_name`` from this request. """ val1 = self.headers.pop(header_name, None) val2 = self.unredirected_headers.pop(header_name, None) return val1 or val2 def add_unredirected_header(self, header_name, header_value): self.unredirected_headers[header_name] = header_value def write_body(self, transport): assert not self._write_done, 'Body already sent' self._write_done = True if not self.body: return if is_streamed(self.body): self._loop.create_task(self._write_streamed_data(transport)) else: self._write_body_data(transport, self.body, True) # INTERNAL ENCODING METHODS def _encode_body(self, data, files, json): body = None ct = None if isinstance(data, (str, bytes)): if files: raise ValueError('data cannot be a string or bytes when ' 'files are present') body = to_bytes(data, self.charset) elif data and is_streamed(data): if files: raise ValueError('data cannot be an iterator when ' 'files are present') if 'content-length' not in self.headers: self.headers['transfer-encoding'] = 'chunked' return data elif data or files: if files: body, ct = self._encode_files(data, files) else: body, ct = self._encode_params(data) elif json: body = _json.dumps(json).encode(self.charset) ct = 'application/json' if not self.headers.get('content-type') and ct: self.headers['Content-Type'] = ct if body: self.headers['content-length'] = str(len(body)) return body def _encode_files(self, data, files): fields = [] for field, val in mapping_iterator(data or ()): if (isinstance(val, str) or isinstance(val, bytes) or not hasattr(val, '__iter__')): val = [val] for v in val: if v is not None: if not isinstance(v, bytes): v = str(v) fields.append( (field.decode('utf-8') if isinstance(field, bytes) else field, v.encode('utf-8') if isinstance(v, str) else v)) for (k, v) in mapping_iterator(files): # support for explicit filename ft = None if isinstance(v, (tuple, list)): if len(v) == 2: fn, fp = v else: fn, fp, ft = v else: fn = guess_filename(v) or k fp = v if isinstance(fp, bytes): fp = BytesIO(fp) elif isinstance(fp, str): fp = StringIO(fp) if ft: new_v = (fn, fp.read(), ft) else: new_v = (fn, fp.read()) fields.append((k, new_v)) # return encode_multipart_formdata(fields, charset=self.charset) def _encode_params(self, params): content_type = self.headers.get('content-type') # No content type given, chose one if not content_type: content_type = FORM_URL_ENCODED if hasattr(params, 'read'): params = params.read() if content_type in JSON_CONTENT_TYPES: body = _json.dumps(params) elif content_type == FORM_URL_ENCODED: body = urlencode(tuple(split_url_params(params))) elif content_type == MULTIPART_FORM_DATA: body, content_type = encode_multipart_formdata( params, charset=self.charset) else: body = params return to_bytes(body, self.charset), content_type def _write_body_data(self, transport, data, finish=False): if self.is_chunked(): data = http_chunks(data, finish) elif data: data = (data, ) else: return for chunk in data: transport.write(chunk) async def _write_streamed_data(self, transport): for data in self.body: try: data = await data except TypeError: pass self._write_body_data(transport, data) self._write_body_data(transport, b'', True) # PROXY INTERNALS def _set_proxy(self, proxies): url = urlparse(self.url) request_proxies = self.client.proxies.copy() if proxies: request_proxies.update(proxies) self.proxies = request_proxies # if url.scheme in request_proxies: host, port = get_hostport(url.scheme, url.netloc) no_proxy = [ n for n in request_proxies.get('no', '').split(',') if n ] if not any(map(host.endswith, no_proxy)): proxy_url = request_proxies[url.scheme] if url.scheme in tls_schemes: self._tunnel = proxy_url else: self._proxy = proxy_url
class ClientRequest: GET_METHODS = {hdrs.METH_GET, hdrs.METH_HEAD, hdrs.METH_OPTIONS} POST_METHODS = {hdrs.METH_PATCH, hdrs.METH_POST, hdrs.METH_PUT} ALL_METHODS = GET_METHODS.union(POST_METHODS).union( {hdrs.METH_DELETE, hdrs.METH_TRACE}) DEFAULT_HEADERS = { hdrs.ACCEPT: '*/*', hdrs.ACCEPT_ENCODING: 'gzip, deflate', } body = b'' auth = None response = None response_class = None _writer = None # async task for streaming data _continue = None # waiter future for '100 Continue' response # N.B. # Adding __del__ method with self._writer closing doesn't make sense # because _writer is instance method, thus it keeps a reference to self. # Until writer has finished finalizer will not be called. def __init__(self, method, url, *, params=None, headers=None, skip_auto_headers=frozenset(), data=None, cookies=None, auth=None, version=http.HttpVersion11, compress=None, chunked=None, expect100=False, loop=None, response_class=None, proxy=None, proxy_auth=None, proxy_from_env=False, timer=None, session=None, auto_decompress=True): if loop is None: loop = asyncio.get_event_loop() assert isinstance(url, URL), url assert isinstance(proxy, (URL, type(None))), proxy self._session = session if params: q = MultiDict(url.query) url2 = url.with_query(params) q.extend(url2.query) url = url.with_query(q) self.url = url.with_fragment(None) self.original_url = url self.method = method.upper() self.chunked = chunked self.compress = compress self.loop = loop self.length = None self.response_class = response_class or ClientResponse self._timer = timer if timer is not None else TimerNoop() self._auto_decompress = auto_decompress if loop.get_debug(): self._source_traceback = traceback.extract_stack(sys._getframe(1)) self.update_version(version) self.update_host(url) self.update_headers(headers) self.update_auto_headers(skip_auto_headers) self.update_cookies(cookies) self.update_content_encoding(data) self.update_auth(auth) self.update_proxy(proxy, proxy_auth, proxy_from_env) self.update_body_from_data(data) self.update_transfer_encoding() self.update_expect_continue(expect100) @property def host(self): return self.url.host @property def port(self): return self.url.port @property def request_info(self): return RequestInfo(self.url, self.method, self.headers) def update_host(self, url): """Update destination host, port and connection type (ssl).""" # get host/port if not url.host: raise ValueError( "Could not parse hostname from URL '{}'".format(url)) # basic auth info username, password = url.user, url.password if username: self.auth = helpers.BasicAuth(username, password or '') # Record entire netloc for usage in host header scheme = url.scheme self.ssl = scheme in ('https', 'wss') def update_version(self, version): """Convert request version to two elements tuple. parser HTTP version '1.1' => (1, 1) """ if isinstance(version, str): v = [l.strip() for l in version.split('.', 1)] try: version = int(v[0]), int(v[1]) except ValueError: raise ValueError( 'Can not parse http version number: {}' .format(version)) from None self.version = version def update_headers(self, headers): """Update request headers.""" self.headers = CIMultiDict() if headers: if isinstance(headers, (dict, MultiDictProxy, MultiDict)): headers = headers.items() for key, value in headers: self.headers.add(key, value) def update_auto_headers(self, skip_auto_headers): self.skip_auto_headers = CIMultiDict( (hdr, None) for hdr in sorted(skip_auto_headers)) used_headers = self.headers.copy() used_headers.extend(self.skip_auto_headers) for hdr, val in self.DEFAULT_HEADERS.items(): if hdr not in used_headers: self.headers.add(hdr, val) # add host if hdrs.HOST not in used_headers: netloc = self.url.raw_host if not self.url.is_default_port(): netloc += ':' + str(self.url.port) self.headers[hdrs.HOST] = netloc if hdrs.USER_AGENT not in used_headers: self.headers[hdrs.USER_AGENT] = SERVER_SOFTWARE def update_cookies(self, cookies): """Update request cookies header.""" if not cookies: return c = SimpleCookie() if hdrs.COOKIE in self.headers: c.load(self.headers.get(hdrs.COOKIE, '')) del self.headers[hdrs.COOKIE] for name, value in cookies.items(): if isinstance(value, Morsel): # Preserve coded_value mrsl_val = value.get(value.key, Morsel()) mrsl_val.set(value.key, value.value, value.coded_value) c[name] = mrsl_val else: c[name] = value self.headers[hdrs.COOKIE] = c.output(header='', sep=';').strip() def update_content_encoding(self, data): """Set request content encoding.""" if not data: return enc = self.headers.get(hdrs.CONTENT_ENCODING, '').lower() if enc: if self.compress: raise ValueError( 'compress can not be set ' 'if Content-Encoding header is set') elif self.compress: if not isinstance(self.compress, str): self.compress = 'deflate' self.headers[hdrs.CONTENT_ENCODING] = self.compress self.chunked = True # enable chunked, no need to deal with length def update_transfer_encoding(self): """Analyze transfer-encoding header.""" te = self.headers.get(hdrs.TRANSFER_ENCODING, '').lower() if 'chunked' in te: if self.chunked: raise ValueError( 'chunked can not be set ' 'if "Transfer-Encoding: chunked" header is set') elif self.chunked: if hdrs.CONTENT_LENGTH in self.headers: raise ValueError( 'chunked can not be set ' 'if Content-Length header is set') self.headers[hdrs.TRANSFER_ENCODING] = 'chunked' else: if hdrs.CONTENT_LENGTH not in self.headers: self.headers[hdrs.CONTENT_LENGTH] = str(len(self.body)) def update_auth(self, auth): """Set basic auth.""" if auth is None: auth = self.auth if auth is None: return if not isinstance(auth, helpers.BasicAuth): raise TypeError('BasicAuth() tuple is required instead') self.headers[hdrs.AUTHORIZATION] = auth.encode() def update_body_from_data(self, body): if not body: return # FormData if isinstance(body, FormData): body = body() try: body = payload.PAYLOAD_REGISTRY.get(body, disposition=None) except payload.LookupError: body = FormData(body)() self.body = body # enable chunked encoding if needed if not self.chunked: if hdrs.CONTENT_LENGTH not in self.headers: size = body.size if size is None: self.chunked = True else: if hdrs.CONTENT_LENGTH not in self.headers: self.headers[hdrs.CONTENT_LENGTH] = str(size) # set content-type if (hdrs.CONTENT_TYPE not in self.headers and hdrs.CONTENT_TYPE not in self.skip_auto_headers): self.headers[hdrs.CONTENT_TYPE] = body.content_type # copy payload headers if body.headers: for (key, value) in body.headers.items(): if key not in self.headers: self.headers[key] = value def update_expect_continue(self, expect=False): if expect: self.headers[hdrs.EXPECT] = '100-continue' elif self.headers.get(hdrs.EXPECT, '').lower() == '100-continue': expect = True if expect: self._continue = helpers.create_future(self.loop) def update_proxy(self, proxy, proxy_auth, proxy_from_env): if proxy_from_env and not proxy: proxy_url = getproxies().get(self.original_url.scheme) proxy = URL(proxy_url) if proxy_url else None if proxy and not proxy.scheme == 'http': raise ValueError("Only http proxies are supported") if proxy_auth and not isinstance(proxy_auth, helpers.BasicAuth): raise ValueError("proxy_auth must be None or BasicAuth() tuple") self.proxy = proxy self.proxy_auth = proxy_auth def keep_alive(self): if self.version < HttpVersion10: # keep alive not supported at all return False if self.version == HttpVersion10: if self.headers.get(hdrs.CONNECTION) == 'keep-alive': return True else: # no headers means we close for Http 1.0 return False elif self.headers.get(hdrs.CONNECTION) == 'close': return False return True @asyncio.coroutine def write_bytes(self, writer, conn): """Support coroutines that yields bytes objects.""" # 100 response if self._continue is not None: yield from writer.drain() yield from self._continue try: if isinstance(self.body, payload.Payload): yield from self.body.write(writer) else: if isinstance(self.body, (bytes, bytearray)): self.body = (self.body,) for chunk in self.body: writer.write(chunk) yield from writer.write_eof() except OSError as exc: new_exc = ClientOSError( exc.errno, 'Can not write request body for %s' % self.url) new_exc.__context__ = exc new_exc.__cause__ = exc conn.protocol.set_exception(new_exc) except asyncio.CancelledError as exc: if not conn.closed: conn.protocol.set_exception(exc) except Exception as exc: conn.protocol.set_exception(exc) finally: self._writer = None def send(self, conn): # Specify request target: # - CONNECT request must send authority form URI # - not CONNECT proxy must send absolute form URI # - most common is origin form URI if self.method == hdrs.METH_CONNECT: path = '{}:{}'.format(self.url.raw_host, self.url.port) elif self.proxy and not self.ssl: path = str(self.url) else: path = self.url.raw_path if self.url.raw_query_string: path += '?' + self.url.raw_query_string writer = PayloadWriter(conn.writer, self.loop) if self.compress: writer.enable_compression(self.compress) if self.chunked is not None: writer.enable_chunking() # set default content-type if (self.method in self.POST_METHODS and hdrs.CONTENT_TYPE not in self.skip_auto_headers and hdrs.CONTENT_TYPE not in self.headers): self.headers[hdrs.CONTENT_TYPE] = 'application/octet-stream' # set the connection header connection = self.headers.get(hdrs.CONNECTION) if not connection: if self.keep_alive(): if self.version == HttpVersion10: connection = 'keep-alive' else: if self.version == HttpVersion11: connection = 'close' if connection is not None: self.headers[hdrs.CONNECTION] = connection # status + headers status_line = '{0} {1} HTTP/{2[0]}.{2[1]}\r\n'.format( self.method, path, self.version) writer.write_headers(status_line, self.headers) self._writer = helpers.ensure_future( self.write_bytes(writer, conn), loop=self.loop) self.response = self.response_class( self.method, self.original_url, writer=self._writer, continue100=self._continue, timer=self._timer, request_info=self.request_info, auto_decompress=self._auto_decompress ) self.response._post_init(self.loop, self._session) return self.response @asyncio.coroutine def close(self): if self._writer is not None: try: yield from self._writer finally: self._writer = None def terminate(self): if self._writer is not None: if not self.loop.is_closed(): self._writer.cancel() self._writer = None
class HttpRequest(RequestBase): """An :class:`HttpClient` request for an HTTP resource. This class has a similar interface to :class:`urllib.request.Request`. :param files: optional dictionary of name, file-like-objects. :param allow_redirects: allow the response to follow redirects. .. attribute:: method The request method .. attribute:: version HTTP version for this request, usually ``HTTP/1.1`` .. attribute:: history List of past :class:`.HttpResponse` (collected during redirects). .. attribute:: wait_continue if ``True``, the :class:`HttpRequest` includes the ``Expect: 100-Continue`` header. .. attribute:: stream Allow for streaming body """ _proxy = None _ssl = None _tunnel = None _write_done = False def __init__(self, client, url, method, inp_params=None, headers=None, data=None, files=None, json=None, history=None, auth=None, charset=None, max_redirects=10, source_address=None, allow_redirects=False, decompress=True, version=None, wait_continue=False, websocket_handler=None, cookies=None, params=None, stream=False, proxies=None, verify=True, cert=None, **extra): self.client = client self.method = method.upper() self.inp_params = inp_params or {} self.unredirected_headers = CIMultiDict() self.history = history self.wait_continue = wait_continue self.max_redirects = max_redirects self.allow_redirects = allow_redirects self.charset = charset or 'utf-8' self.version = version self.decompress = decompress self.websocket_handler = websocket_handler self.source_address = source_address self.stream = stream self.verify = verify self.cert = cert if auth and not isinstance(auth, Auth): auth = HTTPBasicAuth(*auth) self.auth = auth self.url = full_url(url, params, method=self.method) self._set_proxy(proxies) self.key = RequestKey.create(self) self.headers = client.get_headers(self, headers) self.body = self._encode_body(data, files, json) self.unredirected_headers['host'] = self.key.netloc cookies = cookiejar_from_dict(client.cookies, cookies) if cookies: cookies.add_cookie_header(self) @property def _loop(self): return self.client._loop @property def ssl(self): """Context for TLS connections. If this is a tunneled request and the tunnel connection is not yet established, it returns ``None``. """ return self._ssl @property def proxy(self): """Proxy server for this request. """ return self._proxy @property def tunnel(self): """Tunnel for this request. """ return self._tunnel def __repr__(self): return self.first_line() __str__ = __repr__ def first_line(self): if self.method == 'CONNECT': url = self.key.netloc elif self._proxy: url = self.url else: p = urlparse(self.url) url = urlunparse(('', '', p.path or '/', p.params, p.query, p.fragment)) return '%s %s %s' % (self.method, url, self.version) def is_chunked(self): return self.body and 'content-length' not in self.headers def encode(self): """The bytes representation of this :class:`HttpRequest`. Called by :class:`HttpResponse` when it needs to encode this :class:`HttpRequest` before sending it to the HTTP resource. """ # Call body before fist_line in case the query is changes. first_line = self.first_line() if self.body and self.wait_continue: self.headers['expect'] = '100-continue' headers = self.headers if self.unredirected_headers: headers = self.unredirected_headers.copy() headers.update(self.headers) buffer = [first_line.encode('ascii'), b'\r\n'] buffer.extend((('%s: %s\r\n' % (name, value)).encode(CHARSET) for name, value in headers.items())) buffer.append(b'\r\n') return b''.join(buffer) def add_header(self, key, value): self.headers[key] = value def has_header(self, header_name): """Check ``header_name`` is in this request headers. """ return (header_name in self.headers or header_name in self.unredirected_headers) def get_header(self, header_name, default=None): """Retrieve ``header_name`` from this request headers. """ return self.headers.get( header_name, self.unredirected_headers.get(header_name, default)) def remove_header(self, header_name): """Remove ``header_name`` from this request. """ val1 = self.headers.pop(header_name, None) val2 = self.unredirected_headers.pop(header_name, None) return val1 or val2 def add_unredirected_header(self, header_name, header_value): self.unredirected_headers[header_name] = header_value def write_body(self, transport): assert not self._write_done, 'Body already sent' self._write_done = True if not self.body: return if is_streamed(self.body): self._loop.create_task(self._write_streamed_data(transport)) else: self._write_body_data(transport, self.body, True) # INTERNAL ENCODING METHODS def _encode_body(self, data, files, json): body = None ct = None if isinstance(data, (str, bytes)): if files: raise ValueError('data cannot be a string or bytes when ' 'files are present') body = to_bytes(data, self.charset) elif data and is_streamed(data): if files: raise ValueError('data cannot be an iterator when ' 'files are present') if 'content-length' not in self.headers: self.headers['transfer-encoding'] = 'chunked' return data elif data or files: if files: body, ct = self._encode_files(data, files) else: body, ct = self._encode_params(data) elif json: body = _json.dumps(json).encode(self.charset) ct = 'application/json' if not self.headers.get('content-type') and ct: self.headers['Content-Type'] = ct if body: self.headers['content-length'] = str(len(body)) return body def _encode_files(self, data, files): fields = [] for field, val in mapping_iterator(data or ()): if (isinstance(val, str) or isinstance(val, bytes) or not hasattr(val, '__iter__')): val = [val] for v in val: if v is not None: if not isinstance(v, bytes): v = str(v) fields.append((field.decode('utf-8') if isinstance(field, bytes) else field, v.encode('utf-8') if isinstance(v, str) else v)) for (k, v) in mapping_iterator(files): # support for explicit filename ft = None if isinstance(v, (tuple, list)): if len(v) == 2: fn, fp = v else: fn, fp, ft = v else: fn = guess_filename(v) or k fp = v if isinstance(fp, bytes): fp = BytesIO(fp) elif isinstance(fp, str): fp = StringIO(fp) if ft: new_v = (fn, fp.read(), ft) else: new_v = (fn, fp.read()) fields.append((k, new_v)) # return encode_multipart_formdata(fields, charset=self.charset) def _encode_params(self, params): content_type = self.headers.get('content-type') # No content type given, chose one if not content_type: content_type = FORM_URL_ENCODED if hasattr(params, 'read'): params = params.read() if content_type in JSON_CONTENT_TYPES: body = _json.dumps(params) elif content_type == FORM_URL_ENCODED: body = urlencode(tuple(split_url_params(params))) elif content_type == MULTIPART_FORM_DATA: body, content_type = encode_multipart_formdata( params, charset=self.charset) else: body = params return to_bytes(body, self.charset), content_type def _write_body_data(self, transport, data, finish=False): if self.is_chunked(): data = http_chunks(data, finish) elif data: data = (data,) else: return for chunk in data: transport.write(chunk) async def _write_streamed_data(self, transport): for data in self.body: try: data = await data except TypeError: pass self._write_body_data(transport, data) self._write_body_data(transport, b'', True) # PROXY INTERNALS def _set_proxy(self, proxies): url = urlparse(self.url) request_proxies = self.client.proxies.copy() if proxies: request_proxies.update(proxies) self.proxies = request_proxies # if url.scheme in request_proxies: host, port = get_hostport(url.scheme, url.netloc) no_proxy = [n for n in request_proxies.get('no', '').split(',') if n] if not any(map(host.endswith, no_proxy)): proxy_url = request_proxies[url.scheme] if url.scheme in tls_schemes: self._tunnel = proxy_url else: self._proxy = proxy_url
class ClientRequest: GET_METHODS = { hdrs.METH_GET, hdrs.METH_HEAD, hdrs.METH_OPTIONS, hdrs.METH_TRACE, } POST_METHODS = {hdrs.METH_PATCH, hdrs.METH_POST, hdrs.METH_PUT} ALL_METHODS = GET_METHODS.union(POST_METHODS).union({hdrs.METH_DELETE}) DEFAULT_HEADERS = { hdrs.ACCEPT: '*/*', hdrs.ACCEPT_ENCODING: 'gzip, deflate', } body = b'' auth = None response = None response_class = None _writer = None # async task for streaming data (except last byte) _writer_last_byte = None # added async task for sending last byte (R. van Emous @ Computest) _continue = None # waiter future for '100 Continue' response # N.B. # Adding __del__ method with self._writer closing doesn't make sense # because _writer is instance method, thus it keeps a reference to self. # Until writer has finished finalizer will not be called. def __init__(self, method, url, *, params=None, headers=None, skip_auto_headers=frozenset(), data=None, cookies=None, auth=None, version=http.HttpVersion11, compress=None, chunked=None, expect100=False, loop=None, response_class=None, proxy=None, proxy_auth=None, timer=None, session=None, ssl=None, proxy_headers=None, traces=None, final_byte_time=None): # argument for when to send final byte (R. van Emous @ Computest) if loop is None: loop = asyncio.get_event_loop() assert isinstance(url, URL), url assert isinstance(proxy, (URL, type(None))), proxy self._session = session if params: q = MultiDict(url.query) url2 = url.with_query(params) q.extend(url2.query) url = url.with_query(q) self.original_url = url self.url = url.with_fragment(None) self.method = method.upper() self.chunked = chunked self.compress = compress self.loop = loop self.length = None self.response_class = response_class or ClientResponse self._timer = timer if timer is not None else TimerNoop() self._ssl = ssl if loop.get_debug(): self._source_traceback = traceback.extract_stack(sys._getframe(1)) self.update_version(version) self.update_host(url) self.update_headers(headers) self.update_auto_headers(skip_auto_headers) self.update_cookies(cookies) self.update_content_encoding(data) self.update_auth(auth) self.update_proxy(proxy, proxy_auth, proxy_headers) self.update_body_from_data(data) if data or self.method not in self.GET_METHODS: self.update_transfer_encoding() self.update_expect_continue(expect100) if traces is None: traces = [] self._traces = traces self.final_byte_time = final_byte_time # when to send final byte (R. van Emous @ Computest) def is_ssl(self): return self.url.scheme in ('https', 'wss') @property def ssl(self): return self._ssl @property def connection_key(self): proxy_headers = self.proxy_headers if proxy_headers: h = hash(tuple((k, v) for k, v in proxy_headers.items())) else: h = None return ConnectionKey(self.host, self.port, self.is_ssl(), self.ssl, self.proxy, self.proxy_auth, h) @property def host(self): return self.url.host @property def port(self): return self.url.port @property def request_info(self): return RequestInfo(self.url, self.method, self.headers, self.original_url) def update_host(self, url): """Update destination host, port and connection type (ssl).""" # get host/port if not url.host: raise InvalidURL(url) # basic auth info username, password = url.user, url.password if username: self.auth = helpers.BasicAuth(username, password or '') def update_version(self, version): """Convert request version to two elements tuple. parser HTTP version '1.1' => (1, 1) """ if isinstance(version, str): v = [l.strip() for l in version.split('.', 1)] try: version = int(v[0]), int(v[1]) except ValueError: raise ValueError( 'Can not parse http version number: {}' .format(version)) from None self.version = version def update_headers(self, headers): """Update request headers.""" self.headers = CIMultiDict() if headers: if isinstance(headers, (dict, MultiDictProxy, MultiDict)): headers = headers.items() for key, value in headers: self.headers.add(key, value) def update_auto_headers(self, skip_auto_headers): self.skip_auto_headers = CIMultiDict( (hdr, None) for hdr in sorted(skip_auto_headers)) used_headers = self.headers.copy() used_headers.extend(self.skip_auto_headers) for hdr, val in self.DEFAULT_HEADERS.items(): if hdr not in used_headers: self.headers.add(hdr, val) # add host if hdrs.HOST not in used_headers: netloc = self.url.raw_host if not self.url.is_default_port(): netloc += ':' + str(self.url.port) self.headers[hdrs.HOST] = netloc if hdrs.USER_AGENT not in used_headers: self.headers[hdrs.USER_AGENT] = SERVER_SOFTWARE def update_cookies(self, cookies): """Update request cookies header.""" if not cookies: return c = SimpleCookie() if hdrs.COOKIE in self.headers: c.load(self.headers.get(hdrs.COOKIE, '')) del self.headers[hdrs.COOKIE] for name, value in cookies.items(): if isinstance(value, Morsel): # Preserve coded_value mrsl_val = value.get(value.key, Morsel()) mrsl_val.set(value.key, value.value, value.coded_value) c[name] = mrsl_val else: c[name] = value self.headers[hdrs.COOKIE] = c.output(header='', sep=';').strip() def update_content_encoding(self, data): """Set request content encoding.""" if not data: return enc = self.headers.get(hdrs.CONTENT_ENCODING, '').lower() if enc: if self.compress: raise ValueError( 'compress can not be set ' 'if Content-Encoding header is set') elif self.compress: if not isinstance(self.compress, str): self.compress = 'deflate' self.headers[hdrs.CONTENT_ENCODING] = self.compress self.chunked = True # enable chunked, no need to deal with length def update_transfer_encoding(self): """Analyze transfer-encoding header.""" te = self.headers.get(hdrs.TRANSFER_ENCODING, '').lower() if 'chunked' in te: if self.chunked: raise ValueError( 'chunked can not be set ' 'if "Transfer-Encoding: chunked" header is set') elif self.chunked: if hdrs.CONTENT_LENGTH in self.headers: raise ValueError( 'chunked can not be set ' 'if Content-Length header is set') self.headers[hdrs.TRANSFER_ENCODING] = 'chunked' else: if hdrs.CONTENT_LENGTH not in self.headers: self.headers[hdrs.CONTENT_LENGTH] = str(len(self.body)) def update_auth(self, auth): """Set basic auth.""" if auth is None: auth = self.auth if auth is None: return if not isinstance(auth, helpers.BasicAuth): raise TypeError('BasicAuth() tuple is required instead') self.headers[hdrs.AUTHORIZATION] = auth.encode() def update_body_from_data(self, body): if not body: return # FormData if isinstance(body, FormData): body = body() try: body = payload.PAYLOAD_REGISTRY.get(body, disposition=None) except payload.LookupError: body = FormData(body)() self.body = body # enable chunked encoding if needed if not self.chunked: if hdrs.CONTENT_LENGTH not in self.headers: size = body.size if size is None: self.chunked = True else: if hdrs.CONTENT_LENGTH not in self.headers: self.headers[hdrs.CONTENT_LENGTH] = str(size) # set content-type if (hdrs.CONTENT_TYPE not in self.headers and hdrs.CONTENT_TYPE not in self.skip_auto_headers): self.headers[hdrs.CONTENT_TYPE] = body.content_type # copy payload headers if body.headers: for (key, value) in body.headers.items(): if key not in self.headers: self.headers[key] = value def update_expect_continue(self, expect=False): if expect: self.headers[hdrs.EXPECT] = '100-continue' elif self.headers.get(hdrs.EXPECT, '').lower() == '100-continue': expect = True if expect: self._continue = self.loop.create_future() def update_proxy(self, proxy, proxy_auth, proxy_headers): if proxy and not proxy.scheme == 'http': raise ValueError("Only http proxies are supported") if proxy_auth and not isinstance(proxy_auth, helpers.BasicAuth): raise ValueError("proxy_auth must be None or BasicAuth() tuple") self.proxy = proxy self.proxy_auth = proxy_auth self.proxy_headers = proxy_headers def keep_alive(self): if self.version < HttpVersion10: # keep alive not supported at all return False if self.version == HttpVersion10: if self.headers.get(hdrs.CONNECTION) == 'keep-alive': return True else: # no headers means we close for Http 1.0 return False elif self.headers.get(hdrs.CONNECTION) == 'close': return False return True def get_time_ns(self): if sys.version_info >= (3, 6): return time.time_ns() else: return time.time() * 1e9 async def __my_own_sleep(self, wait_until): # get sleep time minus 20 ms sleep_time = wait_until - self.get_time_ns() / 1e6 - 20 # wait longest part async if sleep_time > 0: await asyncio.sleep(sleep_time / 1000) # wait last 20 ms or less synchronously for more accuracy while wait_until - self.get_time_ns() / 1e6 > 0: pass async def write_bytes(self, writer, conn, body_part, final_byte_time=None): """Support coroutines that yields bytes objects.""" if final_byte_time: await self.__my_own_sleep(final_byte_time) # 100 response if self._continue is not None: await writer.drain() await self._continue try: if isinstance(body_part, payload.Payload): await body_part.write(writer) else: if isinstance(body_part, (bytes, bytearray)): body_part = (body_part,) for chunk in body_part: await writer.write(chunk) if final_byte_time: # only write eof after last byte (R. van Emous @ Computest) await writer.write_eof() except OSError as exc: new_exc = ClientOSError( exc.errno, 'Can not write request body for %s' % self.url) new_exc.__context__ = exc new_exc.__cause__ = exc conn.protocol.set_exception(new_exc) except asyncio.CancelledError as exc: if not conn.closed: conn.protocol.set_exception(exc) except Exception as exc: conn.protocol.set_exception(exc) finally: self._writer = None async def send(self, conn): # Specify request target: # - CONNECT request must send authority form URI # - not CONNECT proxy must send absolute form URI # - most common is origin form URI if self.method == hdrs.METH_CONNECT: path = '{}:{}'.format(self.url.raw_host, self.url.port) elif self.proxy and not self.is_ssl(): path = str(self.url) else: path = self.url.raw_path if self.url.raw_query_string: path += '?' + self.url.raw_query_string writer = StreamWriter( conn.protocol, self.loop, on_chunk_sent=self._on_chunk_request_sent ) if self.compress: writer.enable_compression(self.compress) if self.chunked is not None: writer.enable_chunking() # set default content-type if (self.method in self.POST_METHODS and hdrs.CONTENT_TYPE not in self.skip_auto_headers and hdrs.CONTENT_TYPE not in self.headers): self.headers[hdrs.CONTENT_TYPE] = 'application/octet-stream' # set the connection header connection = self.headers.get(hdrs.CONNECTION) if not connection: if self.keep_alive(): if self.version == HttpVersion10: connection = 'keep-alive' else: if self.version == HttpVersion11: connection = 'close' if connection is not None: self.headers[hdrs.CONNECTION] = connection # status + headers status_line = '{0} {1} HTTP/{2[0]}.{2[1]}'.format( self.method, path, self.version) await writer.write_headers(status_line, self.headers) # ----------- # if the request has a body of two bytes or more, send the last byte synchronized (R. van Emous @ Computest) if self.body and self.body._size > 1: if self.final_byte_time is not None: self.last_byte = copy.deepcopy(self.body) self.last_byte._size = 1 self.last_byte._value = bytes([self.last_byte._value[-1]]) self.body._size = self.body._size - 1 if self.body._size == 1: self.body._value = bytes([self.body._value[:-1]]) else: self.body._value = self.body._value[:-1] # send all but one byte self._writer = self.loop.create_task(self.write_bytes(writer, conn, self.body)) self._writer_last_byte = self.loop.create_task( self.write_bytes(writer, conn, self.last_byte, self.final_byte_time)) else: self._writer = self.loop.create_task(self.write_bytes(writer, conn, self.body)) # ----------- self.response = self.response_class( self.method, self.original_url, writer=self._writer, writer_last_byte=self._writer_last_byte, # added argument (R. van Emous @ Computest) continue100=self._continue, timer=self._timer, request_info=self.request_info, traces=self._traces, loop=self.loop, session=self._session ) return self.response async def close(self): if self._writer is not None: try: await self._writer finally: self._writer = None def terminate(self): if self._writer is not None: if not self.loop.is_closed(): self._writer.cancel() self._writer = None async def _on_chunk_request_sent(self, chunk): for trace in self._traces: await trace.send_request_chunk_sent(chunk)