class HttpTunnel(RequestBase): first_line = None data = None decompress = False method = 'CONNECT' def __init__(self, client, req): self.client = client self.key = req self.headers = CIMultiDict(client.DEFAULT_TUNNEL_HEADERS) def __repr__(self): return 'Tunnel %s' % self.url __str__ = __repr__ def encode(self): self.headers['host'] = self.key.netloc self.first_line = 'CONNECT http://%s:%s HTTP/1.1' % self.key.address buffer = [self.first_line.encode('ascii'), b'\r\n'] buffer.extend((('%s: %s\r\n' % (name, value)).encode(CHARSET) for name, value in self.headers.items())) buffer.append(b'\r\n') return b''.join(buffer) def has_header(self, header_name): return header_name in self.headers def get_header(self, header_name, default=None): return self.headers.get(header_name, default) def remove_header(self, header_name): self.headers.pop(header_name, None)
async def start(self, connection, read_until_eof=False): # vk.com return url like this: http://REDIRECT_URI#access_token=... # but aiohttp by default removes all parameters after '#' await super().start(connection, read_until_eof) headers = CIMultiDict(self.headers) location = headers.get(hdrs.LOCATION, None) if location: headers[hdrs.LOCATION] = location.replace('#', '?') self.headers = CIMultiDictProxy(headers) self.raw_headers = tuple(headers.items()) return self
class ClientRequest: GET_METHODS = { hdrs.METH_GET, hdrs.METH_HEAD, hdrs.METH_OPTIONS, hdrs.METH_TRACE, } POST_METHODS = {hdrs.METH_PATCH, hdrs.METH_POST, hdrs.METH_PUT} ALL_METHODS = GET_METHODS.union(POST_METHODS).union({hdrs.METH_DELETE}) DEFAULT_HEADERS = { hdrs.ACCEPT: '*/*', hdrs.ACCEPT_ENCODING: 'gzip, deflate', } body = b'' auth = None response = None response_class = None _writer = None # async task for streaming data _continue = None # waiter future for '100 Continue' response # N.B. # Adding __del__ method with self._writer closing doesn't make sense # because _writer is instance method, thus it keeps a reference to self. # Until writer has finished finalizer will not be called. def __init__(self, method: str, url: URL, *, params: Optional[Mapping[str, str]]=None, headers: Optional[LooseHeaders]=None, skip_auto_headers: Iterable[str]=frozenset(), data: Any=None, cookies: Optional[LooseCookies]=None, auth: Optional[BasicAuth]=None, version: http.HttpVersion=http.HttpVersion11, compress: Optional[str]=None, chunked: Optional[bool]=None, expect100: bool=False, loop: Optional[asyncio.AbstractEventLoop]=None, response_class: Optional[Type['ClientResponse']]=None, proxy: Optional[URL]=None, proxy_auth: Optional[BasicAuth]=None, timer: Optional[BaseTimerContext]=None, session: Optional['ClientSession']=None, ssl: Union[SSLContext, bool, Fingerprint, None]=None, proxy_headers: Optional[LooseHeaders]=None, traces: Optional[List['Trace']]=None): if loop is None: loop = asyncio.get_event_loop() assert isinstance(url, URL), url assert isinstance(proxy, (URL, type(None))), proxy # FIXME: session is None in tests only, need to fix tests # assert session is not None self._session = cast('ClientSession', session) if params: q = MultiDict(url.query) url2 = url.with_query(params) q.extend(url2.query) url = url.with_query(q) self.original_url = url self.url = url.with_fragment(None) self.method = method.upper() self.chunked = chunked self.compress = compress self.loop = loop self.length = None if response_class is None: real_response_class = ClientResponse else: real_response_class = response_class self.response_class = real_response_class # type: Type[ClientResponse] self._timer = timer if timer is not None else TimerNoop() self._ssl = ssl if loop.get_debug(): self._source_traceback = traceback.extract_stack(sys._getframe(1)) self.update_version(version) self.update_host(url) self.update_headers(headers) self.update_auto_headers(skip_auto_headers) self.update_cookies(cookies) self.update_content_encoding(data) self.update_auth(auth) self.update_proxy(proxy, proxy_auth, proxy_headers) self.update_body_from_data(data) if data or self.method not in self.GET_METHODS: self.update_transfer_encoding() self.update_expect_continue(expect100) if traces is None: traces = [] self._traces = traces def is_ssl(self) -> bool: return self.url.scheme in ('https', 'wss') @property def ssl(self) -> Union['SSLContext', None, bool, Fingerprint]: return self._ssl @property def connection_key(self) -> ConnectionKey: proxy_headers = self.proxy_headers if proxy_headers: h = hash(tuple((k, v) for k, v in proxy_headers.items())) # type: Optional[int] # noqa else: h = None return ConnectionKey(self.host, self.port, self.is_ssl(), self.ssl, self.proxy, self.proxy_auth, h) @property def host(self) -> str: ret = self.url.host assert ret is not None return ret @property def port(self) -> Optional[int]: return self.url.port @property def request_info(self) -> RequestInfo: headers = CIMultiDictProxy(self.headers) # type: CIMultiDictProxy[str] return RequestInfo(self.url, self.method, headers, self.original_url) def update_host(self, url: URL) -> None: """Update destination host, port and connection type (ssl).""" # get host/port if not url.host: raise InvalidURL(url) # basic auth info username, password = url.user, url.password if username: self.auth = helpers.BasicAuth(username, password or '') def update_version(self, version: Union[http.HttpVersion, str]) -> None: """Convert request version to two elements tuple. parser HTTP version '1.1' => (1, 1) """ if isinstance(version, str): v = [l.strip() for l in version.split('.', 1)] try: version = http.HttpVersion(int(v[0]), int(v[1])) except ValueError: raise ValueError( 'Can not parse http version number: {}' .format(version)) from None self.version = version def update_headers(self, headers: Optional[LooseHeaders]) -> None: """Update request headers.""" self.headers = CIMultiDict() # type: CIMultiDict[str] # add host netloc = cast(str, self.url.raw_host) if helpers.is_ipv6_address(netloc): netloc = '[{}]'.format(netloc) if not self.url.is_default_port(): netloc += ':' + str(self.url.port) self.headers[hdrs.HOST] = netloc if headers: if isinstance(headers, (dict, MultiDictProxy, MultiDict)): headers = headers.items() # type: ignore for key, value in headers: # A special case for Host header if key.lower() == 'host': self.headers[key] = value else: self.headers.add(key, value) def update_auto_headers(self, skip_auto_headers: Iterable[str]) -> None: self.skip_auto_headers = CIMultiDict( (hdr, None) for hdr in sorted(skip_auto_headers)) used_headers = self.headers.copy() used_headers.extend(self.skip_auto_headers) # type: ignore for hdr, val in self.DEFAULT_HEADERS.items(): if hdr not in used_headers: self.headers.add(hdr, val) if hdrs.USER_AGENT not in used_headers: self.headers[hdrs.USER_AGENT] = SERVER_SOFTWARE def update_cookies(self, cookies: Optional[LooseCookies]) -> None: """Update request cookies header.""" if not cookies: return c = SimpleCookie() if hdrs.COOKIE in self.headers: c.load(self.headers.get(hdrs.COOKIE, '')) del self.headers[hdrs.COOKIE] if isinstance(cookies, Mapping): iter_cookies = cookies.items() else: iter_cookies = cookies # type: ignore for name, value in iter_cookies: if isinstance(value, Morsel): # Preserve coded_value mrsl_val = value.get(value.key, Morsel()) mrsl_val.set(value.key, value.value, value.coded_value) # type: ignore # noqa c[name] = mrsl_val else: c[name] = value # type: ignore self.headers[hdrs.COOKIE] = c.output(header='', sep=';').strip() def update_content_encoding(self, data: Any) -> None: """Set request content encoding.""" if not data: return enc = self.headers.get(hdrs.CONTENT_ENCODING, '').lower() if enc: if self.compress: raise ValueError( 'compress can not be set ' 'if Content-Encoding header is set') elif self.compress: if not isinstance(self.compress, str): self.compress = 'deflate' self.headers[hdrs.CONTENT_ENCODING] = self.compress self.chunked = True # enable chunked, no need to deal with length def update_transfer_encoding(self) -> None: """Analyze transfer-encoding header.""" te = self.headers.get(hdrs.TRANSFER_ENCODING, '').lower() if 'chunked' in te: if self.chunked: raise ValueError( 'chunked can not be set ' 'if "Transfer-Encoding: chunked" header is set') elif self.chunked: if hdrs.CONTENT_LENGTH in self.headers: raise ValueError( 'chunked can not be set ' 'if Content-Length header is set') self.headers[hdrs.TRANSFER_ENCODING] = 'chunked' else: if hdrs.CONTENT_LENGTH not in self.headers: self.headers[hdrs.CONTENT_LENGTH] = str(len(self.body)) def update_auth(self, auth: Optional[BasicAuth]) -> None: """Set basic auth.""" if auth is None: auth = self.auth if auth is None: return if not isinstance(auth, helpers.BasicAuth): raise TypeError('BasicAuth() tuple is required instead') self.headers[hdrs.AUTHORIZATION] = auth.encode() def update_body_from_data(self, body: Any) -> None: if not body: return # FormData if isinstance(body, FormData): body = body() try: body = payload.PAYLOAD_REGISTRY.get(body, disposition=None) except payload.LookupError: body = FormData(body)() self.body = body # enable chunked encoding if needed if not self.chunked: if hdrs.CONTENT_LENGTH not in self.headers: size = body.size if size is None: self.chunked = True else: if hdrs.CONTENT_LENGTH not in self.headers: self.headers[hdrs.CONTENT_LENGTH] = str(size) # set content-type if (hdrs.CONTENT_TYPE not in self.headers and hdrs.CONTENT_TYPE not in self.skip_auto_headers): self.headers[hdrs.CONTENT_TYPE] = body.content_type # copy payload headers if body.headers: for (key, value) in body.headers.items(): if key not in self.headers: self.headers[key] = value def update_expect_continue(self, expect: bool=False) -> None: if expect: self.headers[hdrs.EXPECT] = '100-continue' elif self.headers.get(hdrs.EXPECT, '').lower() == '100-continue': expect = True if expect: self._continue = self.loop.create_future() def update_proxy(self, proxy: Optional[URL], proxy_auth: Optional[BasicAuth], proxy_headers: Optional[LooseHeaders]) -> None: if proxy and not proxy.scheme == 'http': raise ValueError("Only http proxies are supported") if proxy_auth and not isinstance(proxy_auth, helpers.BasicAuth): raise ValueError("proxy_auth must be None or BasicAuth() tuple") self.proxy = proxy self.proxy_auth = proxy_auth self.proxy_headers = proxy_headers def keep_alive(self) -> bool: if self.version < HttpVersion10: # keep alive not supported at all return False if self.version == HttpVersion10: if self.headers.get(hdrs.CONNECTION) == 'keep-alive': return True else: # no headers means we close for Http 1.0 return False elif self.headers.get(hdrs.CONNECTION) == 'close': return False return True async def write_bytes(self, writer: AbstractStreamWriter, conn: 'Connection') -> None: """Support coroutines that yields bytes objects.""" # 100 response if self._continue is not None: await writer.drain() await self._continue protocol = conn.protocol assert protocol is not None try: if isinstance(self.body, payload.Payload): await self.body.write(writer) else: if isinstance(self.body, (bytes, bytearray)): self.body = (self.body,) # type: ignore for chunk in self.body: await writer.write(chunk) # type: ignore await writer.write_eof() except OSError as exc: new_exc = ClientOSError( exc.errno, 'Can not write request body for %s' % self.url) new_exc.__context__ = exc new_exc.__cause__ = exc protocol.set_exception(new_exc) except asyncio.CancelledError as exc: if not conn.closed: protocol.set_exception(exc) except Exception as exc: protocol.set_exception(exc) finally: self._writer = None async def send(self, conn: 'Connection') -> 'ClientResponse': # Specify request target: # - CONNECT request must send authority form URI # - not CONNECT proxy must send absolute form URI # - most common is origin form URI if self.method == hdrs.METH_CONNECT: path = '{}:{}'.format(self.url.raw_host, self.url.port) elif self.proxy and not self.is_ssl(): path = str(self.url) else: path = self.url.raw_path if self.url.raw_query_string: path += '?' + self.url.raw_query_string protocol = conn.protocol assert protocol is not None writer = StreamWriter( protocol, self.loop, on_chunk_sent=self._on_chunk_request_sent ) if self.compress: writer.enable_compression(self.compress) if self.chunked is not None: writer.enable_chunking() # set default content-type if (self.method in self.POST_METHODS and hdrs.CONTENT_TYPE not in self.skip_auto_headers and hdrs.CONTENT_TYPE not in self.headers): self.headers[hdrs.CONTENT_TYPE] = 'application/octet-stream' # set the connection header connection = self.headers.get(hdrs.CONNECTION) if not connection: if self.keep_alive(): if self.version == HttpVersion10: connection = 'keep-alive' else: if self.version == HttpVersion11: connection = 'close' if connection is not None: self.headers[hdrs.CONNECTION] = connection # status + headers status_line = '{0} {1} HTTP/{2[0]}.{2[1]}'.format( self.method, path, self.version) await writer.write_headers(status_line, self.headers) self._writer = self.loop.create_task(self.write_bytes(writer, conn)) response_class = self.response_class assert response_class is not None self.response = response_class( self.method, self.original_url, writer=self._writer, continue100=self._continue, timer=self._timer, request_info=self.request_info, traces=self._traces, loop=self.loop, session=self._session ) return self.response async def close(self) -> None: if self._writer is not None: try: await self._writer finally: self._writer = None def terminate(self) -> None: if self._writer is not None: if not self.loop.is_closed(): self._writer.cancel() self._writer = None async def _on_chunk_request_sent(self, chunk: bytes) -> None: for trace in self._traces: await trace.send_request_chunk_sent(chunk)
def parse_headers(self, lines): """Parses RFC 5322 headers from a stream. Line continuations are supported. Returns list of header name and value pairs. Header name is in upper case. """ headers = CIMultiDict() raw_headers = [] lines_idx = 1 line = lines[1] line_count = len(lines) while line: header_length = len(line) # Parse initial header name : value pair. try: bname, bvalue = line.split(b':', 1) except ValueError: raise InvalidHeader(line) from None bname = bname.strip(b' \t') if HDRRE.search(bname): raise InvalidHeader(bname) # next line lines_idx += 1 line = lines[lines_idx] # consume continuation lines continuation = line and line[0] in (32, 9) # (' ', '\t') if continuation: bvalue = [bvalue] while continuation: header_length += len(line) if header_length > self.max_field_size: raise LineTooLong( 'request header field {}'.format( bname.decode("utf8", "xmlcharrefreplace")), self.max_field_size) bvalue.append(line) # next line lines_idx += 1 if lines_idx < line_count: line = lines[lines_idx] if line: continuation = line[0] in (32, 9) # (' ', '\t') else: line = b'' break bvalue = b''.join(bvalue) else: if header_length > self.max_field_size: raise LineTooLong( 'request header field {}'.format( bname.decode("utf8", "xmlcharrefreplace")), self.max_field_size) bvalue = bvalue.strip() name = istr(bname.decode('utf-8', 'surrogateescape')) value = bvalue.decode('utf-8', 'surrogateescape') headers.add(name, value) raw_headers.append((bname, bvalue)) close_conn = None encoding = None upgrade = False chunked = False raw_headers = tuple(raw_headers) # keep-alive conn = headers.get(hdrs.CONNECTION) if conn: v = conn.lower() if v == 'close': close_conn = True elif v == 'keep-alive': close_conn = False elif v == 'upgrade': upgrade = True # encoding enc = headers.get(hdrs.CONTENT_ENCODING) if enc: enc = enc.lower() if enc in ('gzip', 'deflate'): encoding = enc # chunking te = headers.get(hdrs.TRANSFER_ENCODING) if te and 'chunked' in te.lower(): chunked = True return headers, raw_headers, close_conn, encoding, upgrade, chunked
class AiohttpClientMockResponse: """Mock Aiohttp client response.""" def __init__( self, method, url, status=200, response=None, json=None, text=None, cookies=None, exc=None, headers=None, side_effect=None, ): """Initialize a fake response.""" if json is not None: text = _json.dumps(json) if text is not None: response = text.encode("utf-8") if response is None: response = b"" self.method = method self._url = url self.status = status self.response = response self.exc = exc self.side_effect = side_effect self._headers = CIMultiDict(headers or {}) self._cookies = {} if cookies: for name, data in cookies.items(): cookie = mock.MagicMock() cookie.value = data self._cookies[name] = cookie def match_request(self, method, url, params=None): """Test if response answers request.""" if method.lower() != self.method.lower(): return False # regular expression matching if isinstance(self._url, RETYPE): return self._url.search(str(url)) is not None if (self._url.scheme != url.scheme or self._url.host != url.host or self._url.path != url.path): return False # Ensure all query components in matcher are present in the request request_qs = parse_qs(url.query_string) matcher_qs = parse_qs(self._url.query_string) for key, vals in matcher_qs.items(): for val in vals: try: request_qs.get(key, []).remove(val) except ValueError: return False return True @property def headers(self): """Return content_type.""" return self._headers @property def cookies(self): """Return dict of cookies.""" return self._cookies @property def url(self): """Return yarl of URL.""" return self._url @property def content_type(self): """Return yarl of URL.""" return self._headers.get("content-type") @property def content(self): """Return content.""" return mock_stream(self.response) async def read(self): """Return mock response.""" return self.response async def text(self, encoding="utf-8", errors="strict"): """Return mock response as a string.""" return self.response.decode(encoding, errors=errors) async def json(self, encoding="utf-8", content_type=None): """Return mock response as a json.""" return _json.loads(self.response.decode(encoding)) def release(self): """Mock release.""" def raise_for_status(self): """Raise error if status is 400 or higher.""" if self.status >= 400: request_info = mock.Mock(real_url="http://example.com") raise ClientResponseError( request_info=request_info, history=None, code=self.status, headers=self.headers, ) def close(self): """Mock close."""
class AioHttpTransportResponse(AsyncHttpResponse): """Methods for accessing response body data. :param request: The HttpRequest object :type request: ~azure.core.pipeline.transport.HttpRequest :param aiohttp_response: Returned from ClientSession.request(). :type aiohttp_response: aiohttp.ClientResponse object :param block_size: block size of data sent over connection. :type block_size: int :param bool decompress: If True which is default, will attempt to decode the body based on the *content-encoding* header. """ def __init__(self, request: HttpRequest, aiohttp_response: aiohttp.ClientResponse, block_size=None, *, decompress=True) -> None: super(AioHttpTransportResponse, self).__init__(request, aiohttp_response, block_size=block_size) # https://aiohttp.readthedocs.io/en/stable/client_reference.html#aiohttp.ClientResponse self.status_code = aiohttp_response.status self.headers = CIMultiDict(aiohttp_response.headers) self.reason = aiohttp_response.reason self.content_type = aiohttp_response.headers.get('content-type') self._content = None self._decompressed_content = False self._decompress = decompress def body(self) -> bytes: """Return the whole body as bytes in memory. """ return _aiohttp_body_helper(self) def text(self, encoding: Optional[str] = None) -> str: """Return the whole body as a string. If encoding is not provided, rely on aiohttp auto-detection. :param str encoding: The encoding to apply. """ # super().text detects charset based on self._content() which is compressed # implement the decoding explicitly here body = self.body() ctype = self.headers.get(aiohttp.hdrs.CONTENT_TYPE, "").lower() mimetype = aiohttp.helpers.parse_mimetype(ctype) if not encoding: # extract encoding from mimetype, if caller does not specify encoding = mimetype.parameters.get("charset") if encoding: try: codecs.lookup(encoding) except LookupError: encoding = None if not encoding: if mimetype.type == "application" and ( mimetype.subtype == "json" or mimetype.subtype == "rdap" ): # RFC 7159 states that the default encoding is UTF-8. # RFC 7483 defines application/rdap+json encoding = "utf-8" elif body is None: raise RuntimeError( "Cannot guess the encoding of a not yet read body" ) else: try: import cchardet as chardet except ImportError: # pragma: no cover try: import chardet # type: ignore except ImportError: # pragma: no cover import charset_normalizer as chardet # type: ignore[no-redef] encoding = chardet.detect(body)["encoding"] if encoding == "utf-8" or encoding is None: encoding = "utf-8-sig" return body.decode(encoding) async def load_body(self) -> None: """Load in memory the body, so it could be accessible from sync methods.""" try: self._content = await self.internal_response.read() except aiohttp.client_exceptions.ClientPayloadError as err: # This is the case that server closes connection before we finish the reading. aiohttp library # raises ClientPayloadError. raise IncompleteReadError(err, error=err) def stream_download(self, pipeline, **kwargs) -> AsyncIteratorType[bytes]: """Generator for streaming response body data. :param pipeline: The pipeline object :type pipeline: azure.core.pipeline.Pipeline :keyword bool decompress: If True which is default, will attempt to decode the body based on the *content-encoding* header. """ return AioHttpStreamDownloadGenerator(pipeline, self, **kwargs) def __getstate__(self): # Be sure body is loaded in memory, otherwise not pickable and let it throw self.body() state = self.__dict__.copy() # Remove the unpicklable entries. state['internal_response'] = None # aiohttp response are not pickable (see headers comments) state['headers'] = CIMultiDict(self.headers) # MultiDictProxy is not pickable return state
class ClientRequest: GET_METHODS = {hdrs.METH_GET, hdrs.METH_HEAD, hdrs.METH_OPTIONS} POST_METHODS = {hdrs.METH_PATCH, hdrs.METH_POST, hdrs.METH_PUT} ALL_METHODS = GET_METHODS.union(POST_METHODS).union( {hdrs.METH_DELETE, hdrs.METH_TRACE}) DEFAULT_HEADERS = { hdrs.ACCEPT: '*/*', hdrs.ACCEPT_ENCODING: 'gzip, deflate', } SERVER_SOFTWARE = HttpMessage.SERVER_SOFTWARE body = b'' auth = None response = None response_class = None _writer = None # async task for streaming data _continue = None # waiter future for '100 Continue' response # N.B. # Adding __del__ method with self._writer closing doesn't make sense # because _writer is instance method, thus it keeps a reference to self. # Until writer has finished finalizer will not be called. def __init__(self, method, url, *, params=None, headers=None, skip_auto_headers=frozenset(), data=None, cookies=None, auth=None, encoding='utf-8', version=aiohttp.HttpVersion11, compress=None, chunked=None, expect100=False, loop=None, response_class=None, proxy=None, proxy_auth=None, timeout=5*60): if loop is None: loop = asyncio.get_event_loop() assert isinstance(url, URL), url assert isinstance(proxy, (URL, type(None))), proxy if params: q = MultiDict(url.query) url2 = url.with_query(params) q.extend(url2.query) url = url.with_query(q) self.url = url.with_fragment(None) self.method = method.upper() self.encoding = encoding self.chunked = chunked self.compress = compress self.loop = loop self.response_class = response_class or ClientResponse self._timeout = timeout if loop.get_debug(): self._source_traceback = traceback.extract_stack(sys._getframe(1)) self.update_version(version) self.update_host(url) self.update_headers(headers) self.update_auto_headers(skip_auto_headers) self.update_cookies(cookies) self.update_content_encoding(data) self.update_auth(auth) self.update_proxy(proxy, proxy_auth) self.update_body_from_data(data, skip_auto_headers) self.update_transfer_encoding() self.update_expect_continue(expect100) @property def host(self): return self.url.host @property def port(self): return self.url.port def update_host(self, url): """Update destination host, port and connection type (ssl).""" # get host/port if not url.host: raise ValueError('Host could not be detected.') # basic auth info username, password = url.user, url.password if username: self.auth = helpers.BasicAuth(username, password or '') # Record entire netloc for usage in host header scheme = url.scheme self.ssl = scheme in ('https', 'wss') def update_version(self, version): """Convert request version to two elements tuple. parser HTTP version '1.1' => (1, 1) """ if isinstance(version, str): v = [l.strip() for l in version.split('.', 1)] try: version = int(v[0]), int(v[1]) except ValueError: raise ValueError( 'Can not parse http version number: {}' .format(version)) from None self.version = version def update_headers(self, headers): """Update request headers.""" self.headers = CIMultiDict() if headers: if isinstance(headers, dict): headers = headers.items() elif isinstance(headers, (MultiDictProxy, MultiDict)): headers = headers.items() for key, value in headers: self.headers.add(key, value) def update_auto_headers(self, skip_auto_headers): self.skip_auto_headers = skip_auto_headers used_headers = set(self.headers) | skip_auto_headers for hdr, val in self.DEFAULT_HEADERS.items(): if hdr not in used_headers: self.headers.add(hdr, val) # add host if hdrs.HOST not in used_headers: netloc = self.url.host if not self.url.is_default_port(): netloc += ':' + str(self.url.port) self.headers[hdrs.HOST] = netloc if hdrs.USER_AGENT not in used_headers: self.headers[hdrs.USER_AGENT] = self.SERVER_SOFTWARE def update_cookies(self, cookies): """Update request cookies header.""" if not cookies: return c = http.cookies.SimpleCookie() if hdrs.COOKIE in self.headers: c.load(self.headers.get(hdrs.COOKIE, '')) del self.headers[hdrs.COOKIE] for name, value in cookies.items(): if isinstance(value, http.cookies.Morsel): c[value.key] = value.value else: c[name] = value self.headers[hdrs.COOKIE] = c.output(header='', sep=';').strip() def update_content_encoding(self, data): """Set request content encoding.""" if not data: return enc = self.headers.get(hdrs.CONTENT_ENCODING, '').lower() if enc: if self.compress is not False: self.compress = enc # enable chunked, no need to deal with length self.chunked = True elif self.compress: if not isinstance(self.compress, str): self.compress = 'deflate' self.headers[hdrs.CONTENT_ENCODING] = self.compress self.chunked = True # enable chunked, no need to deal with length def update_auth(self, auth): """Set basic auth.""" if auth is None: auth = self.auth if auth is None: return if not isinstance(auth, helpers.BasicAuth): raise TypeError('BasicAuth() tuple is required instead') self.headers[hdrs.AUTHORIZATION] = auth.encode() def update_body_from_data(self, data, skip_auto_headers): if not data: return if isinstance(data, str): data = data.encode(self.encoding) if isinstance(data, (bytes, bytearray)): self.body = data if (hdrs.CONTENT_TYPE not in self.headers and hdrs.CONTENT_TYPE not in skip_auto_headers): self.headers[hdrs.CONTENT_TYPE] = 'application/octet-stream' if hdrs.CONTENT_LENGTH not in self.headers and not self.chunked: self.headers[hdrs.CONTENT_LENGTH] = str(len(self.body)) elif isinstance(data, (asyncio.StreamReader, streams.StreamReader, streams.DataQueue)): self.body = data elif asyncio.iscoroutine(data): self.body = data if (hdrs.CONTENT_LENGTH not in self.headers and self.chunked is None): self.chunked = True elif isinstance(data, io.IOBase): assert not isinstance(data, io.StringIO), \ 'attempt to send text data instead of binary' self.body = data if not self.chunked and isinstance(data, io.BytesIO): # Not chunking if content-length can be determined size = len(data.getbuffer()) self.headers[hdrs.CONTENT_LENGTH] = str(size) self.chunked = False elif (not self.chunked and isinstance(data, (io.BufferedReader, io.BufferedRandom))): # Not chunking if content-length can be determined try: size = os.fstat(data.fileno()).st_size - data.tell() self.headers[hdrs.CONTENT_LENGTH] = str(size) self.chunked = False except OSError: # data.fileno() is not supported, e.g. # io.BufferedReader(io.BytesIO(b'data')) self.chunked = True else: self.chunked = True if hasattr(data, 'mode'): if data.mode == 'r': raise ValueError('file {!r} should be open in binary mode' ''.format(data)) if (hdrs.CONTENT_TYPE not in self.headers and hdrs.CONTENT_TYPE not in skip_auto_headers and hasattr(data, 'name')): mime = mimetypes.guess_type(data.name)[0] mime = 'application/octet-stream' if mime is None else mime self.headers[hdrs.CONTENT_TYPE] = mime elif isinstance(data, MultipartWriter): self.body = data.serialize() self.headers.update(data.headers) self.chunked = self.chunked or 8192 else: if not isinstance(data, helpers.FormData): data = helpers.FormData(data) self.body = data(self.encoding) if (hdrs.CONTENT_TYPE not in self.headers and hdrs.CONTENT_TYPE not in skip_auto_headers): self.headers[hdrs.CONTENT_TYPE] = data.content_type if data.is_multipart: self.chunked = self.chunked or 8192 else: if (hdrs.CONTENT_LENGTH not in self.headers and not self.chunked): self.headers[hdrs.CONTENT_LENGTH] = str(len(self.body)) def update_transfer_encoding(self): """Analyze transfer-encoding header.""" te = self.headers.get(hdrs.TRANSFER_ENCODING, '').lower() if self.chunked: if hdrs.CONTENT_LENGTH in self.headers: del self.headers[hdrs.CONTENT_LENGTH] if 'chunked' not in te: self.headers[hdrs.TRANSFER_ENCODING] = 'chunked' self.chunked = self.chunked if type(self.chunked) is int else 8192 else: if 'chunked' in te: self.chunked = 8192 else: self.chunked = None if hdrs.CONTENT_LENGTH not in self.headers: self.headers[hdrs.CONTENT_LENGTH] = str(len(self.body)) def update_expect_continue(self, expect=False): if expect: self.headers[hdrs.EXPECT] = '100-continue' elif self.headers.get(hdrs.EXPECT, '').lower() == '100-continue': expect = True if expect: self._continue = helpers.create_future(self.loop) def update_proxy(self, proxy, proxy_auth): if proxy and not proxy.scheme == 'http': raise ValueError("Only http proxies are supported") if proxy_auth and not isinstance(proxy_auth, helpers.BasicAuth): raise ValueError("proxy_auth must be None or BasicAuth() tuple") self.proxy = proxy self.proxy_auth = proxy_auth @asyncio.coroutine def write_bytes(self, request, reader): """Support coroutines that yields bytes objects.""" # 100 response if self._continue is not None: yield from self._continue try: if asyncio.iscoroutine(self.body): request.transport.set_tcp_nodelay(True) exc = None value = None stream = self.body while True: try: if exc is not None: result = stream.throw(exc) else: result = stream.send(value) except StopIteration as exc: if isinstance(exc.value, bytes): yield from request.write(exc.value, drain=True) break except: self.response.close() raise if isinstance(result, asyncio.Future): exc = None value = None try: value = yield result except Exception as err: exc = err elif isinstance(result, (bytes, bytearray)): yield from request.write(result, drain=True) value = None else: raise ValueError( 'Bytes object is expected, got: %s.' % type(result)) elif isinstance(self.body, (asyncio.StreamReader, streams.StreamReader)): request.transport.set_tcp_nodelay(True) chunk = yield from self.body.read(streams.DEFAULT_LIMIT) while chunk: yield from request.write(chunk, drain=True) chunk = yield from self.body.read(streams.DEFAULT_LIMIT) elif isinstance(self.body, streams.DataQueue): request.transport.set_tcp_nodelay(True) while True: try: chunk = yield from self.body.read() if chunk is EOF_MARKER: break yield from request.write(chunk, drain=True) except streams.EofStream: break elif isinstance(self.body, io.IOBase): chunk = self.body.read(self.chunked) while chunk: request.write(chunk) chunk = self.body.read(self.chunked) request.transport.set_tcp_nodelay(True) else: if isinstance(self.body, (bytes, bytearray)): self.body = (self.body,) for chunk in self.body: request.write(chunk) request.transport.set_tcp_nodelay(True) except Exception as exc: new_exc = aiohttp.ClientRequestError( 'Can not write request body for %s' % self.url) new_exc.__context__ = exc new_exc.__cause__ = exc reader.set_exception(new_exc) else: assert request.transport.tcp_nodelay try: ret = request.write_eof() # NB: in asyncio 3.4.1+ StreamWriter.drain() is coroutine # see bug #170 if (asyncio.iscoroutine(ret) or isinstance(ret, asyncio.Future)): yield from ret except Exception as exc: new_exc = aiohttp.ClientRequestError( 'Can not write request body for %s' % self.url) new_exc.__context__ = exc new_exc.__cause__ = exc reader.set_exception(new_exc) self._writer = None def send(self, writer, reader): writer.set_tcp_cork(True) path = self.url.raw_path if self.url.raw_query_string: path += '?' + self.url.raw_query_string request = aiohttp.Request(writer, self.method, path, self.version) if self.compress: request.add_compression_filter(self.compress) if self.chunked is not None: request.enable_chunked_encoding() request.add_chunking_filter(self.chunked) # set default content-type if (self.method in self.POST_METHODS and hdrs.CONTENT_TYPE not in self.skip_auto_headers and hdrs.CONTENT_TYPE not in self.headers): self.headers[hdrs.CONTENT_TYPE] = 'application/octet-stream' for k, value in self.headers.items(): request.add_header(k, value) request.send_headers() self._writer = helpers.ensure_future( self.write_bytes(request, reader), loop=self.loop) self.response = self.response_class( self.method, self.url, writer=self._writer, continue100=self._continue, timeout=self._timeout) self.response._post_init(self.loop) return self.response @asyncio.coroutine def close(self): if self._writer is not None: try: yield from self._writer finally: self._writer = None def terminate(self): if self._writer is not None: if not self.loop.is_closed(): self._writer.cancel() self._writer = None
def get_deployments(): ''' API endpoint to fetch deployment info --- parameters: - in: query name: completed required: false schema: type: boolean description: > Filter datasets by the completed attribute - in: query name: delayed_mode required: false schema: type: boolean description: > Filter datasets by the delayed_mode attribute - in: query name: minTime required: false schema: type: string example: now-12hr description: > Filter datasets with by last file's modtime being newer than minTime. Enter a datetime string (yyyy-MM-ddTHH:mm:ssZ) Or specify 'now-nUnits' for example now-12hr (integers only!) responses: 200: description: Success 400: description: Bad Request 500: description: Internal Server Error 501: description: Not Implemented ''' # Parse case insensitive query parameters request_query = CIMultiDict(request.args) query = {} # Set up the mongo query def parse_date(datestr): ''' Parse the time query param ''' try: if datestr.startswith('now-'): p = re.compile(r'^now-(?P<val>\d+)\s*(?P<units>\w+)$') match = p.search(datestr) val = int(match.group('val')) units = match.group('units') # If not valid units, exception will throw unknown_unit = Unit(units) hrs = Unit('hours') # convert to hours num_hrs = unknown_unit.convert(val, hrs) dt_now = datetime.now(tz=timezone.utc) return dt_now - timedelta(hours=num_hrs) return dateparse(datestr) except Exception: return None # Get the query values completed = request_query.get('completed', None) if completed and completed.lower() in ['true', 'false']: query['completed'] = completed.lower() == 'true' delayed_mode = request_query.get('delayed_mode', None) if delayed_mode and delayed_mode.lower() in ['true', 'false']: query['delayed_mode'] = delayed_mode.lower() == 'true' min_time = request_query.get('minTime', None) if min_time: min_time_dt = parse_date(min_time) if min_time_dt is not None: query['latest_file_mtime'] = {'$gte': min_time_dt} deployments = db.Deployment.find(query) results = [] for deployment in deployments: d = json.loads(deployment.to_json()) d['id'] = d['_id']['$oid'] del d['_id'] del d['user_id'] d.pop('compliance_check_report', None) d['sos'] = deployment.sos d['iso'] = deployment.iso d['dap'] = deployment.dap d['erddap'] = deployment.erddap d['thredds'] = deployment.thredds d['attribution'] = deployment.attribution results.append(d) return jsonify(results=results, num_results=len(results))
def make_mocked_request(method, path, headers=None, *, version=HttpVersion(1, 1), closing=False, app=None, writer=sentinel, payload_writer=sentinel, protocol=sentinel, transport=sentinel, payload=sentinel, sslcontext=None, secure_proxy_ssl_header=None, client_max_size=1024**2, loop=...): """Creates mocked web.Request testing purposes. Useful in unit tests, when spinning full web server is overkill or specific conditions and errors are hard to trigger. """ task = mock.Mock() if loop is ...: loop = mock.Mock() loop.create_future.return_value = () if version < HttpVersion(1, 1): closing = True if headers: headers = CIMultiDict(headers) raw_hdrs = tuple( (k.encode('utf-8'), v.encode('utf-8')) for k, v in headers.items()) else: headers = CIMultiDict() raw_hdrs = () chunked = 'chunked' in headers.get(hdrs.TRANSFER_ENCODING, '').lower() message = RawRequestMessage( method, path, version, headers, raw_hdrs, closing, False, False, chunked, URL(path)) if app is None: app = _create_app_mock() if protocol is sentinel: protocol = mock.Mock() if transport is sentinel: transport = _create_transport(sslcontext) if writer is sentinel: writer = mock.Mock() writer.transport = transport if payload_writer is sentinel: payload_writer = mock.Mock() payload_writer.write_eof.side_effect = noop payload_writer.drain.side_effect = noop protocol.transport = transport protocol.writer = writer if payload is sentinel: payload = mock.Mock() time_service = mock.Mock() time_service.time.return_value = 12345 time_service.strtime.return_value = "Tue, 15 Nov 1994 08:12:31 GMT" @contextmanager def timeout(*args, **kw): yield time_service.timeout = mock.Mock() time_service.timeout.side_effect = timeout req = Request(message, payload, protocol, payload_writer, time_service, task, loop, secure_proxy_ssl_header=secure_proxy_ssl_header, client_max_size=client_max_size) match_info = UrlMappingMatchInfo({}, mock.Mock()) match_info.add_app(app) req._match_info = match_info return req
def produce_plot(self, query, mode): """ Handler for a GetMap and GetVSec requests. Produces a plot with the parameters specified in the URL. # TODO: Handle multiple layers. (mr, 2010-06-09) # TODO: Cache the produced images: Check whether an image with the given # parameters has already been produced. (mr, 2010-08-18) """ logging.debug("GetMap/GetVSec request. Interpreting parameters..") # 1) Make query parameters Case Insensitive # ========================================= query = CIMultiDict(query) # 2) Evaluate query parameters: # ============================= # Image size. figsize = float(query.get('WIDTH', 900)), float(query.get('HEIGHT', 600)) logging.debug(" requested image size = %sx%s", figsize[0], figsize[1]) # Requested layers. layers = [ layer for layer in query.get('LAYERS', '').strip().split(',') if layer ] layer = layers[0] if len(layers) > 0 else '' if layer.find(".") > 0: dataset, layer = layer.split(".") else: dataset = None logging.debug(" requested dataset = '%s', layer = '%s'", dataset, layer) # Requested style(s). styles = [ style for style in query.get('STYLES', 'default').strip().split(',') if style ] style = styles[0] if len(styles) > 0 else None logging.debug(" requested style = '%s'", style) # Forecast initialisation time. init_time = query.get('DIM_INIT_TIME') if init_time is not None: try: init_time = parse_iso_datetime(init_time) except ValueError: return self.create_service_exception( code="InvalidDimensionValue", text= "DIM_INIT_TIME has wrong format (needs to be 2005-08-29T13:00:00Z)" ) logging.debug(" requested initialisation time = '%s'", init_time) # Forecast valid time. valid_time = query.get('TIME') if valid_time is not None: try: valid_time = parse_iso_datetime(valid_time) except ValueError: return self.create_service_exception( code="InvalidDimensionValue", text= "TIME has wrong format (needs to be 2005-08-29T13:00:00Z)") logging.debug(" requested (valid) time = '%s'", valid_time) # Coordinate reference system. crs = query.get('SRS', 'EPSG:4326').lower() # Allow to request vertical sections via GetMap, if the specified CRS is of type "VERT:??". msg = None if crs.startswith('vert:logp'): mode = "getvsec" else: try: get_projection_params(crs) except ValueError: return self.create_service_exception( code="InvalidSRS", text="The requested CRS '{}' is not supported.".format( crs)) logging.debug(" requested coordinate reference system = '%s'", crs) # Create a frameless figure (WMS) or one with title and legend # (MSS specific)? Default is WMS mode (frameless). noframe = query.get('FRAME', 'off').lower() == 'off' # Transparency. transparent = query.get('TRANSPARENT', 'false').lower() == 'true' if transparent: logging.debug(" requested transparent image") # Return format (image/png, text/xml, etc.). return_format = query.get('FORMAT', 'image/png').lower() logging.debug(" requested return format = '%s'", return_format) if return_format not in ["image/png", "text/xml"]: return self.create_service_exception( code="InvalidFORMAT", text="unsupported FORMAT: '{}'".format(return_format)) # 3) Check GetMap/GetVSec-specific parameters and produce # the image with the corresponding section driver. # ======================================================= if mode == "getmap": # Check requested layer. if (dataset not in self.hsec_layer_registry) or ( layer not in self.hsec_layer_registry[dataset]): return self.create_service_exception( code="LayerNotDefined", text="Invalid LAYER '{}.{}' requested".format( dataset, layer)) # Check if the layer requires time information and if they are given. if self.hsec_layer_registry[dataset][ layer].uses_inittime_dimension() and init_time is None: return self.create_service_exception( code="MissingDimensionValue", text= "INIT_TIME not specified (use the DIM_INIT_TIME keyword)") if self.hsec_layer_registry[dataset][ layer].uses_validtime_dimension() and valid_time is None: return self.create_service_exception( code="MissingDimensionValue", text="TIME not specified") # Check if the requested coordinate system is supported. if not self.hsec_layer_registry[dataset][layer].support_epsg_code( crs): return self.create_service_exception( code="InvalidSRS", text="The requested CRS '{}' is not supported.".format( crs)) # Bounding box. try: bbox = [ float(v) for v in query.get('BBOX', '-180,-90,180,90').split(',') ] except ValueError: return self.create_service_exception( text="Invalid BBOX: {}".format(query.get("BBOX"))) # Vertical level, if applicable. level = query.get('ELEVATION') level = float(level) if level is not None else None layer_datatypes = self.hsec_layer_registry[dataset][ layer].required_datatypes() if any(_x in layer_datatypes for _x in ["pl", "al", "ml", "tl", "pv"]) and level is None: # Use the default value. level = -1 elif ("sfc" in layer_datatypes) and \ all(_x not in layer_datatypes for _x in ["pl", "al", "ml", "tl", "pv"]) and \ level is not None: return self.create_service_exception( text= "ELEVATION argument not applicable for layer '{}'. Please omit this argument." .format(layer)) plot_driver = self.hsec_drivers[dataset] try: plot_driver.set_plot_parameters( self.hsec_layer_registry[dataset][layer], bbox=bbox, level=level, crs=crs, init_time=init_time, valid_time=valid_time, style=style, figsize=figsize, noframe=noframe, transparent=transparent, return_format=return_format) image = plot_driver.plot() except (IOError, ValueError) as ex: logging.error("ERROR: %s %s", type(ex), ex) logging.debug("%s", traceback.format_exc()) msg = "The data corresponding to your request is not available. Please check the " \ "times and/or levels you have specified.\n\n" \ "Error message: '{}'".format(ex) return self.create_service_exception(text=msg) elif mode == "getvsec": # Vertical secton path. path = query.get("PATH") if path is None: return self.create_service_exception(text="PATH not specified") try: path = [float(v) for v in path.split(',')] path = [[lat, lon] for lat, lon in zip(path[0::2], path[1::2])] except ValueError: return self.create_service_exception( text="Invalid PATH: {}".format(path)) logging.debug("VSEC PATH: %s", path) # Check requested layers. if (dataset not in self.vsec_layer_registry) or ( layer not in self.vsec_layer_registry[dataset]): return self.create_service_exception( code="LayerNotDefined", text="Invalid LAYER '{}.{}' requested".format( dataset, layer)) # Check if the layer requires time information and if they are given. if self.vsec_layer_registry[dataset][ layer].uses_inittime_dimension(): if init_time is None: return self.create_service_exception( code="MissingDimensionValue", text= "INIT_TIME not specified (use the DIM_INIT_TIME keyword)" ) if valid_time is None: return self.create_service_exception( code="MissingDimensionValue", text="TIME not specified") # Bounding box (num interp. points, p_bot, num labels, p_top). try: bbox = [ float(v) for v in query.get("BBOX", "101,1050,10,180").split(",") ] except ValueError: return self.create_service_exception( text="Invalid BBOX: {}".format(query.get("BBOX"))) plot_driver = self.vsec_drivers[dataset] try: plot_driver.set_plot_parameters( plot_object=self.vsec_layer_registry[dataset][layer], vsec_path=path, vsec_numpoints=bbox[0], vsec_path_connection="greatcircle", vsec_numlabels=bbox[2], init_time=init_time, valid_time=valid_time, style=style, bbox=bbox, figsize=figsize, noframe=noframe, transparent=transparent, return_format=return_format) image = plot_driver.plot() except (IOError, ValueError) as ex: logging.error("ERROR: %s %s", type(ex), ex) msg = "The data corresponding to your request is not available. Please check the " \ "times and/or path you have specified.\n\n" \ "Error message: {}".format(ex) return self.create_service_exception(text=msg) # 4) Return the produced image. # ============================= return image, return_format
class ClientRequest: GET_METHODS = { hdrs.METH_GET, hdrs.METH_HEAD, hdrs.METH_OPTIONS, hdrs.METH_TRACE, } POST_METHODS = {hdrs.METH_PATCH, hdrs.METH_POST, hdrs.METH_PUT} ALL_METHODS = GET_METHODS.union(POST_METHODS).union({hdrs.METH_DELETE}) DEFAULT_HEADERS = { hdrs.ACCEPT: '*/*', hdrs.ACCEPT_ENCODING: 'gzip, deflate', } body = b'' auth = None response = None response_class = None _writer = None # async task for streaming data (except last byte) _writer_last_byte = None # added async task for sending last byte (R. van Emous @ Computest) _continue = None # waiter future for '100 Continue' response # N.B. # Adding __del__ method with self._writer closing doesn't make sense # because _writer is instance method, thus it keeps a reference to self. # Until writer has finished finalizer will not be called. def __init__(self, method, url, *, params=None, headers=None, skip_auto_headers=frozenset(), data=None, cookies=None, auth=None, version=http.HttpVersion11, compress=None, chunked=None, expect100=False, loop=None, response_class=None, proxy=None, proxy_auth=None, timer=None, session=None, ssl=None, proxy_headers=None, traces=None, final_byte_time=None): # argument for when to send final byte (R. van Emous @ Computest) if loop is None: loop = asyncio.get_event_loop() assert isinstance(url, URL), url assert isinstance(proxy, (URL, type(None))), proxy self._session = session if params: q = MultiDict(url.query) url2 = url.with_query(params) q.extend(url2.query) url = url.with_query(q) self.original_url = url self.url = url.with_fragment(None) self.method = method.upper() self.chunked = chunked self.compress = compress self.loop = loop self.length = None self.response_class = response_class or ClientResponse self._timer = timer if timer is not None else TimerNoop() self._ssl = ssl if loop.get_debug(): self._source_traceback = traceback.extract_stack(sys._getframe(1)) self.update_version(version) self.update_host(url) self.update_headers(headers) self.update_auto_headers(skip_auto_headers) self.update_cookies(cookies) self.update_content_encoding(data) self.update_auth(auth) self.update_proxy(proxy, proxy_auth, proxy_headers) self.update_body_from_data(data) if data or self.method not in self.GET_METHODS: self.update_transfer_encoding() self.update_expect_continue(expect100) if traces is None: traces = [] self._traces = traces self.final_byte_time = final_byte_time # when to send final byte (R. van Emous @ Computest) def is_ssl(self): return self.url.scheme in ('https', 'wss') @property def ssl(self): return self._ssl @property def connection_key(self): proxy_headers = self.proxy_headers if proxy_headers: h = hash(tuple((k, v) for k, v in proxy_headers.items())) else: h = None return ConnectionKey(self.host, self.port, self.is_ssl(), self.ssl, self.proxy, self.proxy_auth, h) @property def host(self): return self.url.host @property def port(self): return self.url.port @property def request_info(self): return RequestInfo(self.url, self.method, self.headers, self.original_url) def update_host(self, url): """Update destination host, port and connection type (ssl).""" # get host/port if not url.host: raise InvalidURL(url) # basic auth info username, password = url.user, url.password if username: self.auth = helpers.BasicAuth(username, password or '') def update_version(self, version): """Convert request version to two elements tuple. parser HTTP version '1.1' => (1, 1) """ if isinstance(version, str): v = [l.strip() for l in version.split('.', 1)] try: version = int(v[0]), int(v[1]) except ValueError: raise ValueError( 'Can not parse http version number: {}' .format(version)) from None self.version = version def update_headers(self, headers): """Update request headers.""" self.headers = CIMultiDict() if headers: if isinstance(headers, (dict, MultiDictProxy, MultiDict)): headers = headers.items() for key, value in headers: self.headers.add(key, value) def update_auto_headers(self, skip_auto_headers): self.skip_auto_headers = CIMultiDict( (hdr, None) for hdr in sorted(skip_auto_headers)) used_headers = self.headers.copy() used_headers.extend(self.skip_auto_headers) for hdr, val in self.DEFAULT_HEADERS.items(): if hdr not in used_headers: self.headers.add(hdr, val) # add host if hdrs.HOST not in used_headers: netloc = self.url.raw_host if not self.url.is_default_port(): netloc += ':' + str(self.url.port) self.headers[hdrs.HOST] = netloc if hdrs.USER_AGENT not in used_headers: self.headers[hdrs.USER_AGENT] = SERVER_SOFTWARE def update_cookies(self, cookies): """Update request cookies header.""" if not cookies: return c = SimpleCookie() if hdrs.COOKIE in self.headers: c.load(self.headers.get(hdrs.COOKIE, '')) del self.headers[hdrs.COOKIE] for name, value in cookies.items(): if isinstance(value, Morsel): # Preserve coded_value mrsl_val = value.get(value.key, Morsel()) mrsl_val.set(value.key, value.value, value.coded_value) c[name] = mrsl_val else: c[name] = value self.headers[hdrs.COOKIE] = c.output(header='', sep=';').strip() def update_content_encoding(self, data): """Set request content encoding.""" if not data: return enc = self.headers.get(hdrs.CONTENT_ENCODING, '').lower() if enc: if self.compress: raise ValueError( 'compress can not be set ' 'if Content-Encoding header is set') elif self.compress: if not isinstance(self.compress, str): self.compress = 'deflate' self.headers[hdrs.CONTENT_ENCODING] = self.compress self.chunked = True # enable chunked, no need to deal with length def update_transfer_encoding(self): """Analyze transfer-encoding header.""" te = self.headers.get(hdrs.TRANSFER_ENCODING, '').lower() if 'chunked' in te: if self.chunked: raise ValueError( 'chunked can not be set ' 'if "Transfer-Encoding: chunked" header is set') elif self.chunked: if hdrs.CONTENT_LENGTH in self.headers: raise ValueError( 'chunked can not be set ' 'if Content-Length header is set') self.headers[hdrs.TRANSFER_ENCODING] = 'chunked' else: if hdrs.CONTENT_LENGTH not in self.headers: self.headers[hdrs.CONTENT_LENGTH] = str(len(self.body)) def update_auth(self, auth): """Set basic auth.""" if auth is None: auth = self.auth if auth is None: return if not isinstance(auth, helpers.BasicAuth): raise TypeError('BasicAuth() tuple is required instead') self.headers[hdrs.AUTHORIZATION] = auth.encode() def update_body_from_data(self, body): if not body: return # FormData if isinstance(body, FormData): body = body() try: body = payload.PAYLOAD_REGISTRY.get(body, disposition=None) except payload.LookupError: body = FormData(body)() self.body = body # enable chunked encoding if needed if not self.chunked: if hdrs.CONTENT_LENGTH not in self.headers: size = body.size if size is None: self.chunked = True else: if hdrs.CONTENT_LENGTH not in self.headers: self.headers[hdrs.CONTENT_LENGTH] = str(size) # set content-type if (hdrs.CONTENT_TYPE not in self.headers and hdrs.CONTENT_TYPE not in self.skip_auto_headers): self.headers[hdrs.CONTENT_TYPE] = body.content_type # copy payload headers if body.headers: for (key, value) in body.headers.items(): if key not in self.headers: self.headers[key] = value def update_expect_continue(self, expect=False): if expect: self.headers[hdrs.EXPECT] = '100-continue' elif self.headers.get(hdrs.EXPECT, '').lower() == '100-continue': expect = True if expect: self._continue = self.loop.create_future() def update_proxy(self, proxy, proxy_auth, proxy_headers): if proxy and not proxy.scheme == 'http': raise ValueError("Only http proxies are supported") if proxy_auth and not isinstance(proxy_auth, helpers.BasicAuth): raise ValueError("proxy_auth must be None or BasicAuth() tuple") self.proxy = proxy self.proxy_auth = proxy_auth self.proxy_headers = proxy_headers def keep_alive(self): if self.version < HttpVersion10: # keep alive not supported at all return False if self.version == HttpVersion10: if self.headers.get(hdrs.CONNECTION) == 'keep-alive': return True else: # no headers means we close for Http 1.0 return False elif self.headers.get(hdrs.CONNECTION) == 'close': return False return True def get_time_ns(self): if sys.version_info >= (3, 6): return time.time_ns() else: return time.time() * 1e9 async def __my_own_sleep(self, wait_until): # get sleep time minus 20 ms sleep_time = wait_until - self.get_time_ns() / 1e6 - 20 # wait longest part async if sleep_time > 0: await asyncio.sleep(sleep_time / 1000) # wait last 20 ms or less synchronously for more accuracy while wait_until - self.get_time_ns() / 1e6 > 0: pass async def write_bytes(self, writer, conn, body_part, final_byte_time=None): """Support coroutines that yields bytes objects.""" if final_byte_time: await self.__my_own_sleep(final_byte_time) # 100 response if self._continue is not None: await writer.drain() await self._continue try: if isinstance(body_part, payload.Payload): await body_part.write(writer) else: if isinstance(body_part, (bytes, bytearray)): body_part = (body_part,) for chunk in body_part: await writer.write(chunk) if final_byte_time: # only write eof after last byte (R. van Emous @ Computest) await writer.write_eof() except OSError as exc: new_exc = ClientOSError( exc.errno, 'Can not write request body for %s' % self.url) new_exc.__context__ = exc new_exc.__cause__ = exc conn.protocol.set_exception(new_exc) except asyncio.CancelledError as exc: if not conn.closed: conn.protocol.set_exception(exc) except Exception as exc: conn.protocol.set_exception(exc) finally: self._writer = None async def send(self, conn): # Specify request target: # - CONNECT request must send authority form URI # - not CONNECT proxy must send absolute form URI # - most common is origin form URI if self.method == hdrs.METH_CONNECT: path = '{}:{}'.format(self.url.raw_host, self.url.port) elif self.proxy and not self.is_ssl(): path = str(self.url) else: path = self.url.raw_path if self.url.raw_query_string: path += '?' + self.url.raw_query_string writer = StreamWriter( conn.protocol, self.loop, on_chunk_sent=self._on_chunk_request_sent ) if self.compress: writer.enable_compression(self.compress) if self.chunked is not None: writer.enable_chunking() # set default content-type if (self.method in self.POST_METHODS and hdrs.CONTENT_TYPE not in self.skip_auto_headers and hdrs.CONTENT_TYPE not in self.headers): self.headers[hdrs.CONTENT_TYPE] = 'application/octet-stream' # set the connection header connection = self.headers.get(hdrs.CONNECTION) if not connection: if self.keep_alive(): if self.version == HttpVersion10: connection = 'keep-alive' else: if self.version == HttpVersion11: connection = 'close' if connection is not None: self.headers[hdrs.CONNECTION] = connection # status + headers status_line = '{0} {1} HTTP/{2[0]}.{2[1]}'.format( self.method, path, self.version) await writer.write_headers(status_line, self.headers) # ----------- # if the request has a body of two bytes or more, send the last byte synchronized (R. van Emous @ Computest) if self.body and self.body._size > 1: if self.final_byte_time is not None: self.last_byte = copy.deepcopy(self.body) self.last_byte._size = 1 self.last_byte._value = bytes([self.last_byte._value[-1]]) self.body._size = self.body._size - 1 if self.body._size == 1: self.body._value = bytes([self.body._value[:-1]]) else: self.body._value = self.body._value[:-1] # send all but one byte self._writer = self.loop.create_task(self.write_bytes(writer, conn, self.body)) self._writer_last_byte = self.loop.create_task( self.write_bytes(writer, conn, self.last_byte, self.final_byte_time)) else: self._writer = self.loop.create_task(self.write_bytes(writer, conn, self.body)) # ----------- self.response = self.response_class( self.method, self.original_url, writer=self._writer, writer_last_byte=self._writer_last_byte, # added argument (R. van Emous @ Computest) continue100=self._continue, timer=self._timer, request_info=self.request_info, traces=self._traces, loop=self.loop, session=self._session ) return self.response async def close(self): if self._writer is not None: try: await self._writer finally: self._writer = None def terminate(self): if self._writer is not None: if not self.loop.is_closed(): self._writer.cancel() self._writer = None async def _on_chunk_request_sent(self, chunk): for trace in self._traces: await trace.send_request_chunk_sent(chunk)
def make_mocked_request(method, path, headers=None, *, match_info=sentinel, version=HttpVersion(1, 1), closing=False, app=None, writer=sentinel, protocol=sentinel, transport=sentinel, payload=sentinel, sslcontext=None, client_max_size=1024**2, loop=...): """Creates mocked web.Request testing purposes. Useful in unit tests, when spinning full web server is overkill or specific conditions and errors are hard to trigger. """ task = mock.Mock() if loop is ...: loop = mock.Mock() loop.create_future.return_value = () if version < HttpVersion(1, 1): closing = True if headers: headers = CIMultiDict(headers) raw_hdrs = tuple( (k.encode('utf-8'), v.encode('utf-8')) for k, v in headers.items()) else: headers = CIMultiDict() raw_hdrs = () chunked = 'chunked' in headers.get(hdrs.TRANSFER_ENCODING, '').lower() message = RawRequestMessage( method, path, version, headers, raw_hdrs, closing, False, False, chunked, URL(path)) if app is None: app = _create_app_mock() if protocol is sentinel: protocol = mock.Mock() if transport is sentinel: transport = _create_transport(sslcontext) if writer is sentinel: writer = mock.Mock() writer.write_headers = make_mocked_coro(None) writer.write = make_mocked_coro(None) writer.write_eof = make_mocked_coro(None) writer.drain = make_mocked_coro(None) writer.transport = transport protocol.transport = transport protocol.writer = writer if payload is sentinel: payload = mock.Mock() req = Request(message, payload, protocol, writer, task, loop, client_max_size=client_max_size) match_info = UrlMappingMatchInfo( {} if match_info is sentinel else match_info, mock.Mock()) match_info.add_app(app) req._match_info = match_info return req
class HTTPMessage: def __init__(self): self.version = "HTTP/1.1" self.headers = CIMultiDict() self.body = b"" self.rawform = None self.form = None self.json = None self.xml = None self.files = None self.text = None self.boundary = "--------BOUNDARY--------" def check_version(self): if not self.version.startswith("HTTP/"): raise HTTPError("HTTP version must start with HTTP/") if self.version not in ["HTTP/1.0", "HTTP/1.1"]: raise HTTPError("HTTP version not supported") def transfer_encodings(self): encoding = self.headers.get("Transfer-Encoding", "identity") return [enc.strip() for enc in encoding.split(",")] def is_chunked(self): return "chunked" in self.transfer_encodings() def parse_body(self): type, param = parseheader(self.headers.get("Content-Type", "")) is_json = type == "application/json" or type.endswith("+json") is_xml = type in XML_TYPES or type.endswith("+xml") is_text = type in TEXT_TYPES or type.startswith("text/") or is_json or is_xml if is_text: try: self.text = self.body.decode(param.get("charset", "UTF-8")) except UnicodeDecodeError: raise HTTPError("Failed to decode HTTP body") if type == "application/x-www-form-urlencoded": self.form = formdecode(self.text) self.rawform = formdecode(self.text, False) if is_json: try: self.json = json.loads(self.text) except json.JSONDecodeError: raise HTTPError("Failed to decode JSON body") if is_xml: try: self.xml = xml.parse(self.text) except ValueError as e: raise HTTPError("Failed to decode XML body: %s" %e) if type.startswith("multipart/form-data"): if "boundary" not in param: raise HTTPError("multipart/form-data required boundary parameter") self.boundary = param["boundary"] self.files = self.parse_files(self.body) def parse_files(self, data): split = b"--%s" %self.boundary.encode() parts = data.split(split) if parts[-1] != b"--\r\n" or parts[0] != b"": raise HTTPError("Failed to decode multipart body") files = MultiDict() for part in parts[1:-1]: if part[:2] != b"\r\n" or part[-2:] != b"\r\n": raise HTTPError("Failed to decode multipart body") part = part[2:-2] if not b"\r\n\r\n" in part: raise HTTPError("Failed to decode multipart body") head, body = part.split(b"\r\n\r\n", 1) try: lines = head.decode().split("\r\n") except UnicodeDecodeError: raise HTTPError("Failed to decode multipart body") headers = {} for header in lines: if not ": " in header: raise HTTPError("Invalid line in multipart headers") key, value = header.split(": ", 1) headers[key] = value if "Content-Disposition" not in headers: raise HTTPError("Expected Content-Disposition header in multipart data") type, param = parseheader(headers["Content-Disposition"]) if type != "form-data": raise HTTPError("Expected form-data header in multipart data") if "name" not in param: raise HTTPError("Expected name parameter in Content-Disposition header") files[param["name"]] = body return files def encode_body(self): text = self.text body = self.body if self.rawform is not None: if "Content-Type" not in self.headers: self.headers["Content-Type"] = "application/x-www-form-urlencoded" text = formencode(self.rawform, False) elif self.form is not None: if "Content-Type" not in self.headers: self.headers["Content-Type"] = "application/x-www-form-urlencoded" text = formencode(self.form) elif self.json is not None: if "Content-Type" not in self.headers: self.headers["Content-Type"] = "application/json" text = json.dumps(self.json) elif self.xml is not None: if "Content-Type" not in self.headers: self.headers["Content-Type"] = "application/xml" text = self.xml.encode() elif self.files is not None: if "Content-Type" not in self.headers: self.headers["Content-Type"] = "multipart/form-data" self.headers["Content-Type"] += "; boundary=%s" %self.boundary text = None body = b"" for name, data in self.files.items(): name = name.replace('"', '\\"') body += b"--%s\r\n" %self.boundary.encode() body += b"Content-Disposition: form-data; name=\"%s\"\r\n\r\n" %name.encode() body += data + b"\r\n" body += b"--%s--\r\n" %self.boundary.encode() if text is not None: if "Content-Type" not in self.headers: self.headers["Content-Type"] = "text/plain" body = text.encode() if body and "Content-Type" not in self.headers: self.headers["Content-Type"] = "application/octet-stream" if self.is_chunked(): if not body: return b"0\r\n\r\n" return b"%x\r\n" %len(body) + body + b"\r\n0\r\n\r\n" else: if body: self.headers["Content-Length"] = len(body) return body def encode_start_line(self): raise NotImplementedError("%s.encode_start_line" %self.__class__.__name__) def encode_headers(self): self.encode_body() lines = [self.encode_start_line()] for key, value in self.headers.items(): lines.append("%s: %s" %(key, value)) text = "\r\n".join(lines) + "\r\n\r\n" return text.encode() def encode(self): return self.encode_headers() + self.encode_body() @classmethod def parse(cls, data, head=False): parser = HTTPParser(cls, head) parser.update(data) parser.eof() if parser.buffer: raise HTTPError("Got more data than expected") return parser.message
class StreamResponse(BaseClass, HeadersMixin): _length_check = True def __init__( self, *, status: int = 200, reason: Optional[str] = None, headers: Optional[LooseHeaders] = None, ) -> None: self._body = None self._keep_alive = None # type: Optional[bool] self._chunked = False self._compression = False self._compression_force = None # type: Optional[ContentCoding] self._cookies = SimpleCookie() # type: SimpleCookie[str] self._req = None # type: Optional[BaseRequest] self._payload_writer = None # type: Optional[AbstractStreamWriter] self._eof_sent = False self._body_length = 0 self._state = {} # type: Dict[str, Any] if headers is not None: self._headers = CIMultiDict(headers) # type: CIMultiDict[str] else: self._headers = CIMultiDict() self.set_status(status, reason) @property def prepared(self) -> bool: return self._payload_writer is not None @property def task(self) -> "Optional[asyncio.Task[None]]": if self._req: return self._req.task else: return None @property def status(self) -> int: return self._status @property def chunked(self) -> bool: return self._chunked @property def compression(self) -> bool: return self._compression @property def reason(self) -> str: return self._reason def set_status( self, status: int, reason: Optional[str] = None, _RESPONSES: Mapping[int, Tuple[str, str]] = RESPONSES, ) -> None: assert not self.prepared, ( "Cannot change the response status code after " "the headers have been sent") self._status = int(status) if reason is None: try: reason = _RESPONSES[self._status][0] except Exception: reason = "" self._reason = reason @property def keep_alive(self) -> Optional[bool]: return self._keep_alive def force_close(self) -> None: self._keep_alive = False @property def body_length(self) -> int: return self._body_length @property def output_length(self) -> int: warnings.warn("output_length is deprecated", DeprecationWarning) assert self._payload_writer return self._payload_writer.buffer_size def enable_chunked_encoding(self, chunk_size: Optional[int] = None) -> None: """Enables automatic chunked transfer encoding.""" self._chunked = True if hdrs.CONTENT_LENGTH in self._headers: raise RuntimeError("You can't enable chunked encoding when " "a content length is set") if chunk_size is not None: warnings.warn("Chunk size is deprecated #1615", DeprecationWarning) def enable_compression(self, force: Optional[Union[bool, ContentCoding]] = None ) -> None: """Enables response compression encoding.""" # Backwards compatibility for when force was a bool <0.17. if type(force) == bool: force = ContentCoding.deflate if force else ContentCoding.identity warnings.warn("Using boolean for force is deprecated #3318", DeprecationWarning) elif force is not None: assert isinstance(force, ContentCoding), ("force should one of " "None, bool or " "ContentEncoding") self._compression = True self._compression_force = force @property def headers(self) -> "CIMultiDict[str]": return self._headers @property def cookies(self) -> "SimpleCookie[str]": return self._cookies def set_cookie( self, name: str, value: str, *, expires: Optional[str] = None, domain: Optional[str] = None, max_age: Optional[Union[int, str]] = None, path: str = "/", secure: Optional[bool] = None, httponly: Optional[bool] = None, version: Optional[str] = None, samesite: Optional[str] = None, ) -> None: """Set or update response cookie. Sets new cookie or updates existent with new value. Also updates only those params which are not None. """ old = self._cookies.get(name) if old is not None and old.coded_value == "": # deleted cookie self._cookies.pop(name, None) self._cookies[name] = value c = self._cookies[name] if expires is not None: c["expires"] = expires elif c.get("expires") == "Thu, 01 Jan 1970 00:00:00 GMT": del c["expires"] if domain is not None: c["domain"] = domain if max_age is not None: c["max-age"] = str(max_age) elif "max-age" in c: del c["max-age"] c["path"] = path if secure is not None: c["secure"] = secure if httponly is not None: c["httponly"] = httponly if version is not None: c["version"] = version if samesite is not None: c["samesite"] = samesite def del_cookie(self, name: str, *, domain: Optional[str] = None, path: str = "/") -> None: """Delete cookie. Creates new empty expired cookie. """ # TODO: do we need domain/path here? self._cookies.pop(name, None) self.set_cookie( name, "", max_age=0, expires="Thu, 01 Jan 1970 00:00:00 GMT", domain=domain, path=path, ) @property def content_length(self) -> Optional[int]: # Just a placeholder for adding setter return super().content_length @content_length.setter def content_length(self, value: Optional[int]) -> None: if value is not None: value = int(value) if self._chunked: raise RuntimeError("You can't set content length when " "chunked encoding is enable") self._headers[hdrs.CONTENT_LENGTH] = str(value) else: self._headers.pop(hdrs.CONTENT_LENGTH, None) @property def content_type(self) -> str: # Just a placeholder for adding setter return super().content_type @content_type.setter def content_type(self, value: str) -> None: self.content_type # read header values if needed self._content_type = str(value) self._generate_content_type_header() @property def charset(self) -> Optional[str]: # Just a placeholder for adding setter return super().charset @charset.setter def charset(self, value: Optional[str]) -> None: ctype = self.content_type # read header values if needed if ctype == "application/octet-stream": raise RuntimeError("Setting charset for application/octet-stream " "doesn't make sense, setup content_type first") assert self._content_dict is not None if value is None: self._content_dict.pop("charset", None) else: self._content_dict["charset"] = str(value).lower() self._generate_content_type_header() @property def last_modified(self) -> Optional[datetime.datetime]: """The value of Last-Modified HTTP header, or None. This header is represented as a `datetime` object. """ return parse_http_date(self._headers.get(hdrs.LAST_MODIFIED)) @last_modified.setter def last_modified( self, value: Optional[Union[int, float, datetime.datetime, str]]) -> None: if value is None: self._headers.pop(hdrs.LAST_MODIFIED, None) elif isinstance(value, (int, float)): self._headers[hdrs.LAST_MODIFIED] = time.strftime( "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(math.ceil(value))) elif isinstance(value, datetime.datetime): self._headers[hdrs.LAST_MODIFIED] = time.strftime( "%a, %d %b %Y %H:%M:%S GMT", value.utctimetuple()) elif isinstance(value, str): self._headers[hdrs.LAST_MODIFIED] = value @property def etag(self) -> Optional[ETag]: quoted_value = self._headers.get(hdrs.ETAG) if not quoted_value: return None elif quoted_value == ETAG_ANY: return ETag(value=ETAG_ANY) match = QUOTED_ETAG_RE.fullmatch(quoted_value) if not match: return None is_weak, value = match.group(1, 2) return ETag( is_weak=bool(is_weak), value=value, ) @etag.setter def etag(self, value: Optional[Union[ETag, str]]) -> None: if value is None: self._headers.pop(hdrs.ETAG, None) elif (isinstance(value, str) and value == ETAG_ANY) or (isinstance(value, ETag) and value.value == ETAG_ANY): self._headers[hdrs.ETAG] = ETAG_ANY elif isinstance(value, str): validate_etag_value(value) self._headers[hdrs.ETAG] = f'"{value}"' elif isinstance(value, ETag) and isinstance(value.value, str): validate_etag_value(value.value) hdr_value = f'W/"{value.value}"' if value.is_weak else f'"{value.value}"' self._headers[hdrs.ETAG] = hdr_value else: raise ValueError(f"Unsupported etag type: {type(value)}. " f"etag must be str, ETag or None") def _generate_content_type_header(self, CONTENT_TYPE: istr = hdrs.CONTENT_TYPE ) -> None: assert self._content_dict is not None assert self._content_type is not None params = "; ".join(f"{k}={v}" for k, v in self._content_dict.items()) if params: ctype = self._content_type + "; " + params else: ctype = self._content_type self._headers[CONTENT_TYPE] = ctype async def _do_start_compression(self, coding: ContentCoding) -> None: if coding != ContentCoding.identity: assert self._payload_writer is not None self._headers[hdrs.CONTENT_ENCODING] = coding.value self._payload_writer.enable_compression(coding.value) # Compressed payload may have different content length, # remove the header self._headers.popall(hdrs.CONTENT_LENGTH, None) async def _start_compression(self, request: "BaseRequest") -> None: if self._compression_force: await self._do_start_compression(self._compression_force) else: accept_encoding = request.headers.get(hdrs.ACCEPT_ENCODING, "").lower() for coding in ContentCoding: if coding.value in accept_encoding: await self._do_start_compression(coding) return async def prepare( self, request: "BaseRequest") -> Optional[AbstractStreamWriter]: if self._eof_sent: return None if self._payload_writer is not None: return self._payload_writer return await self._start(request) async def _start(self, request: "BaseRequest") -> AbstractStreamWriter: self._req = request writer = self._payload_writer = request._payload_writer await self._prepare_headers() await request._prepare_hook(self) await self._write_headers() return writer async def _prepare_headers(self) -> None: request = self._req assert request is not None writer = self._payload_writer assert writer is not None keep_alive = self._keep_alive if keep_alive is None: keep_alive = request.keep_alive self._keep_alive = keep_alive version = request.version headers = self._headers for cookie in self._cookies.values(): value = cookie.output(header="")[1:] headers.add(hdrs.SET_COOKIE, value) if self._compression: await self._start_compression(request) if self._chunked: if version != HttpVersion11: raise RuntimeError("Using chunked encoding is forbidden " "for HTTP/{0.major}.{0.minor}".format( request.version)) writer.enable_chunking() headers[hdrs.TRANSFER_ENCODING] = "chunked" if hdrs.CONTENT_LENGTH in headers: del headers[hdrs.CONTENT_LENGTH] elif self._length_check: writer.length = self.content_length if writer.length is None: if version >= HttpVersion11 and self.status != 204: writer.enable_chunking() headers[hdrs.TRANSFER_ENCODING] = "chunked" if hdrs.CONTENT_LENGTH in headers: del headers[hdrs.CONTENT_LENGTH] else: keep_alive = False # HTTP 1.1: https://tools.ietf.org/html/rfc7230#section-3.3.2 # HTTP 1.0: https://tools.ietf.org/html/rfc1945#section-10.4 elif version >= HttpVersion11 and self.status in (100, 101, 102, 103, 204): del headers[hdrs.CONTENT_LENGTH] if self.status not in (204, 304): headers.setdefault(hdrs.CONTENT_TYPE, "application/octet-stream") headers.setdefault(hdrs.DATE, rfc822_formatted_time()) headers.setdefault(hdrs.SERVER, SERVER_SOFTWARE) # connection header if hdrs.CONNECTION not in headers: if keep_alive: if version == HttpVersion10: headers[hdrs.CONNECTION] = "keep-alive" else: if version == HttpVersion11: headers[hdrs.CONNECTION] = "close" async def _write_headers(self) -> None: request = self._req assert request is not None writer = self._payload_writer assert writer is not None # status line version = request.version status_line = "HTTP/{}.{} {} {}".format(version[0], version[1], self._status, self._reason) await writer.write_headers(status_line, self._headers) async def write(self, data: bytes) -> None: assert isinstance( data, (bytes, bytearray, memoryview)), "data argument must be byte-ish (%r)" % type(data) if self._eof_sent: raise RuntimeError("Cannot call write() after write_eof()") if self._payload_writer is None: raise RuntimeError("Cannot call write() before prepare()") await self._payload_writer.write(data) async def drain(self) -> None: assert not self._eof_sent, "EOF has already been sent" assert self._payload_writer is not None, "Response has not been started" warnings.warn( "drain method is deprecated, use await resp.write()", DeprecationWarning, stacklevel=2, ) await self._payload_writer.drain() async def write_eof(self, data: bytes = b"") -> None: assert isinstance( data, (bytes, bytearray, memoryview)), "data argument must be byte-ish (%r)" % type(data) if self._eof_sent: return assert self._payload_writer is not None, "Response has not been started" await self._payload_writer.write_eof(data) self._eof_sent = True self._req = None self._body_length = self._payload_writer.output_size self._payload_writer = None def __repr__(self) -> str: if self._eof_sent: info = "eof" elif self.prepared: assert self._req is not None info = f"{self._req.method} {self._req.path} " else: info = "not prepared" return f"<{self.__class__.__name__} {self.reason} {info}>" def __getitem__(self, key: str) -> Any: return self._state[key] def __setitem__(self, key: str, value: Any) -> None: self._state[key] = value def __delitem__(self, key: str) -> None: del self._state[key] def __len__(self) -> int: return len(self._state) def __iter__(self) -> Iterator[str]: return iter(self._state) def __hash__(self) -> int: return hash(id(self)) def __eq__(self, other: object) -> bool: return self is other
class AioHttpTransportResponse(AsyncHttpResponse): """Methods for accessing response body data. :param request: The HttpRequest object :type request: ~azure.core.pipeline.transport.HttpRequest :param aiohttp_response: Returned from ClientSession.request(). :type aiohttp_response: aiohttp.ClientResponse object :param block_size: block size of data sent over connection. :type block_size: int :param bool decompress: If True which is default, will attempt to decode the body based on the *content-encoding* header. """ def __init__(self, request: HttpRequest, aiohttp_response: aiohttp.ClientResponse, block_size=None, *, decompress=True) -> None: super(AioHttpTransportResponse, self).__init__(request, aiohttp_response, block_size=block_size) # https://aiohttp.readthedocs.io/en/stable/client_reference.html#aiohttp.ClientResponse self.status_code = aiohttp_response.status self.headers = CIMultiDict(aiohttp_response.headers) self.reason = aiohttp_response.reason self.content_type = aiohttp_response.headers.get('content-type') self._body = None self._decompressed_body = None self._decompress = decompress def body(self) -> bytes: """Return the whole body as bytes in memory. """ if self._body is None: raise ValueError( "Body is not available. Call async method load_body, or do your call with stream=False." ) if not self._decompress: return self._body enc = self.headers.get('Content-Encoding') if not enc: return self._body enc = enc.lower() if enc in ("gzip", "deflate"): if self._decompressed_body: return self._decompressed_body import zlib zlib_mode = 16 + zlib.MAX_WBITS if enc == "gzip" else zlib.MAX_WBITS decompressor = zlib.decompressobj(wbits=zlib_mode) self._decompressed_body = decompressor.decompress(self._body) return self._decompressed_body return self._body def text(self, encoding: Optional[str] = None) -> str: """Return the whole body as a string. If encoding is not provided, rely on aiohttp auto-detection. :param str encoding: The encoding to apply. """ # super().text detects charset based on self._body() which is compressed # implement the decoding explicitly here body = self.body() ctype = self.headers.get(aiohttp.hdrs.CONTENT_TYPE, "").lower() mimetype = aiohttp.helpers.parse_mimetype(ctype) encoding = mimetype.parameters.get("charset") if encoding: try: codecs.lookup(encoding) except LookupError: encoding = None if not encoding: if mimetype.type == "application" and ( mimetype.subtype == "json" or mimetype.subtype == "rdap"): # RFC 7159 states that the default encoding is UTF-8. # RFC 7483 defines application/rdap+json encoding = "utf-8" elif body is None: raise RuntimeError( "Cannot guess the encoding of a not yet read body") else: encoding = chardet.detect(body)["encoding"] if not encoding: encoding = "utf-8-sig" return body.decode(encoding) async def load_body(self) -> None: """Load in memory the body, so it could be accessible from sync methods.""" self._body = await self.internal_response.read() def stream_download(self, pipeline, **kwargs) -> AsyncIteratorType[bytes]: """Generator for streaming response body data. :param pipeline: The pipeline object :type pipeline: azure.core.pipeline.Pipeline :keyword bool decompress: If True which is default, will attempt to decode the body based on the *content-encoding* header. """ return AioHttpStreamDownloadGenerator(pipeline, self, **kwargs) def __getstate__(self): # Be sure body is loaded in memory, otherwise not pickable and let it throw self.body() state = self.__dict__.copy() # Remove the unpicklable entries. state[ 'internal_response'] = None # aiohttp response are not pickable (see headers comments) state['headers'] = CIMultiDict( self.headers) # MultiDictProxy is not pickable return state
class StreamResponse(HeadersMixin): def __init__(self, *, status=200, reason=None, headers=None): self._body = None self._keep_alive = None self._chunked = False self._chunk_size = None self._compression = False self._compression_force = False self._headers = CIMultiDict() self._cookies = http.cookies.SimpleCookie() self.set_status(status, reason) self._req = None self._resp_impl = None self._eof_sent = False self._tcp_nodelay = True self._tcp_cork = False if headers is not None: self._headers.extend(headers) self._parse_content_type(self._headers.get(hdrs.CONTENT_TYPE)) self._generate_content_type_header() def _copy_cookies(self): for cookie in self._cookies.values(): value = cookie.output(header='')[1:] self.headers.add(hdrs.SET_COOKIE, value) @property def prepared(self): return self._resp_impl is not None @property def started(self): warnings.warn('use Response.prepared instead', DeprecationWarning) return self.prepared @property def status(self): return self._status @property def chunked(self): return self._chunked @property def compression(self): return self._compression @property def reason(self): return self._reason def set_status(self, status, reason=None): self._status = int(status) if reason is None: reason = ResponseImpl.calc_reason(status) self._reason = reason @property def keep_alive(self): return self._keep_alive def force_close(self): self._keep_alive = False def enable_chunked_encoding(self, chunk_size=None): """Enables automatic chunked transfer encoding.""" self._chunked = True self._chunk_size = chunk_size def enable_compression(self, force=None): """Enables response compression encoding.""" # Backwards compatibility for when force was a bool <0.17. if type(force) == bool: force = ContentCoding.deflate if force else ContentCoding.identity elif force is not None: assert isinstance(force, ContentCoding), ("force should one of " "None, bool or " "ContentEncoding") self._compression = True self._compression_force = force @property def headers(self): return self._headers @property def cookies(self): return self._cookies def set_cookie(self, name, value, *, expires=None, domain=None, max_age=None, path='/', secure=None, httponly=None, version=None): """Set or update response cookie. Sets new cookie or updates existent with new value. Also updates only those params which are not None. """ old = self._cookies.get(name) if old is not None and old.coded_value == '': # deleted cookie self._cookies.pop(name, None) self._cookies[name] = value c = self._cookies[name] if expires is not None: c['expires'] = expires elif c.get('expires') == 'Thu, 01 Jan 1970 00:00:00 GMT': del c['expires'] if domain is not None: c['domain'] = domain if max_age is not None: c['max-age'] = max_age elif 'max-age' in c: del c['max-age'] c['path'] = path if secure is not None: c['secure'] = secure if httponly is not None: c['httponly'] = httponly if version is not None: c['version'] = version def del_cookie(self, name, *, domain=None, path='/'): """Delete cookie. Creates new empty expired cookie. """ # TODO: do we need domain/path here? self._cookies.pop(name, None) self.set_cookie(name, '', max_age=0, expires="Thu, 01 Jan 1970 00:00:00 GMT", domain=domain, path=path) @property def content_length(self): # Just a placeholder for adding setter return super().content_length @content_length.setter def content_length(self, value): if value is not None: value = int(value) # TODO: raise error if chunked enabled self.headers[hdrs.CONTENT_LENGTH] = str(value) else: self.headers.pop(hdrs.CONTENT_LENGTH, None) @property def content_type(self): # Just a placeholder for adding setter return super().content_type @content_type.setter def content_type(self, value): self.content_type # read header values if needed self._content_type = str(value) self._generate_content_type_header() @property def charset(self): # Just a placeholder for adding setter return super().charset @charset.setter def charset(self, value): ctype = self.content_type # read header values if needed if ctype == 'application/octet-stream': raise RuntimeError("Setting charset for application/octet-stream " "doesn't make sense, setup content_type first") if value is None: self._content_dict.pop('charset', None) else: self._content_dict['charset'] = str(value).lower() self._generate_content_type_header() @property def last_modified(self, _LAST_MODIFIED=hdrs.LAST_MODIFIED): """The value of Last-Modified HTTP header, or None. This header is represented as a `datetime` object. """ httpdate = self.headers.get(_LAST_MODIFIED) if httpdate is not None: timetuple = parsedate(httpdate) if timetuple is not None: return datetime.datetime(*timetuple[:6], tzinfo=datetime.timezone.utc) return None @last_modified.setter def last_modified(self, value): if value is None: self.headers.pop(hdrs.LAST_MODIFIED, None) elif isinstance(value, (int, float)): self.headers[hdrs.LAST_MODIFIED] = time.strftime( "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(math.ceil(value))) elif isinstance(value, datetime.datetime): self.headers[hdrs.LAST_MODIFIED] = time.strftime( "%a, %d %b %Y %H:%M:%S GMT", value.utctimetuple()) elif isinstance(value, str): self.headers[hdrs.LAST_MODIFIED] = value @property def tcp_nodelay(self): return self._tcp_nodelay def set_tcp_nodelay(self, value): value = bool(value) self._tcp_nodelay = value if value: self._tcp_cork = False if self._resp_impl is None: return if value: self._resp_impl.transport.set_tcp_cork(False) self._resp_impl.transport.set_tcp_nodelay(value) @property def tcp_cork(self): return self._tcp_cork def set_tcp_cork(self, value): value = bool(value) self._tcp_cork = value if value: self._tcp_nodelay = False if self._resp_impl is None: return if value: self._resp_impl.transport.set_tcp_nodelay(False) self._resp_impl.transport.set_tcp_cork(value) def _generate_content_type_header(self, CONTENT_TYPE=hdrs.CONTENT_TYPE): params = '; '.join("%s=%s" % i for i in self._content_dict.items()) if params: ctype = self._content_type + '; ' + params else: ctype = self._content_type self.headers[CONTENT_TYPE] = ctype def _start_pre_check(self, request): if self._resp_impl is not None: if self._req is not request: raise RuntimeError( "Response has been started with different request.") else: return self._resp_impl else: return None def _do_start_compression(self, coding): if coding != ContentCoding.identity: self.headers[hdrs.CONTENT_ENCODING] = coding.value self._resp_impl.add_compression_filter(coding.value) self.content_length = None def _start_compression(self, request): if self._compression_force: self._do_start_compression(self._compression_force) else: accept_encoding = request.headers.get(hdrs.ACCEPT_ENCODING, '').lower() for coding in ContentCoding: if coding.value in accept_encoding: self._do_start_compression(coding) return def start(self, request): warnings.warn('use .prepare(request) instead', DeprecationWarning) resp_impl = self._start_pre_check(request) if resp_impl is not None: return resp_impl return self._start(request) @asyncio.coroutine def prepare(self, request): resp_impl = self._start_pre_check(request) if resp_impl is not None: return resp_impl yield from request.app.on_response_prepare.send(request, self) return self._start(request) def _start(self, request): self._req = request keep_alive = self._keep_alive if keep_alive is None: keep_alive = request.keep_alive self._keep_alive = keep_alive resp_impl = self._resp_impl = ResponseImpl(request._writer, self._status, request.version, not keep_alive, self._reason) self._copy_cookies() if self._compression: self._start_compression(request) if self._chunked: if request.version != HttpVersion11: raise RuntimeError("Using chunked encoding is forbidden " "for HTTP/{0.major}.{0.minor}".format( request.version)) resp_impl.enable_chunked_encoding() if self._chunk_size: resp_impl.add_chunking_filter(self._chunk_size) headers = self.headers.items() for key, val in headers: resp_impl.add_header(key, val) resp_impl.transport.set_tcp_nodelay(self._tcp_nodelay) resp_impl.transport.set_tcp_cork(self._tcp_cork) self._send_headers(resp_impl) return resp_impl def _send_headers(self, resp_impl): # Durty hack required for # https://github.com/KeepSafe/aiohttp/issues/1093 # File sender may override it resp_impl.send_headers() def write(self, data): assert isinstance(data, (bytes, bytearray, memoryview)), \ "data argument must be byte-ish (%r)" % type(data) if self._eof_sent: raise RuntimeError("Cannot call write() after write_eof()") if self._resp_impl is None: raise RuntimeError("Cannot call write() before start()") if data: return self._resp_impl.write(data) else: return () @asyncio.coroutine def drain(self): if self._resp_impl is None: raise RuntimeError("Response has not been started") yield from self._resp_impl.transport.drain() @asyncio.coroutine def write_eof(self): if self._eof_sent: return if self._resp_impl is None: raise RuntimeError("Response has not been started") yield from self._resp_impl.write_eof() self._eof_sent = True def __repr__(self): if self.started: info = "{} {} ".format(self._req.method, self._req.path) else: info = "not started" return "<{} {} {}>".format(self.__class__.__name__, self.reason, info)
class ClientRequest: GET_METHODS = {hdrs.METH_GET, hdrs.METH_HEAD, hdrs.METH_OPTIONS} POST_METHODS = {hdrs.METH_PATCH, hdrs.METH_POST, hdrs.METH_PUT} ALL_METHODS = GET_METHODS.union(POST_METHODS).union( {hdrs.METH_DELETE, hdrs.METH_TRACE}) DEFAULT_HEADERS = { hdrs.ACCEPT: '*/*', hdrs.ACCEPT_ENCODING: 'gzip, deflate', } SERVER_SOFTWARE = HttpMessage.SERVER_SOFTWARE body = b'' auth = None response = None response_class = None _writer = None # async task for streaming data _continue = None # waiter future for '100 Continue' response # N.B. # Adding __del__ method with self._writer closing doesn't make sense # because _writer is instance method, thus it keeps a reference to self. # Until writer has finished finalizer will not be called. def __init__(self, method, url, *, params=None, headers=None, skip_auto_headers=frozenset(), data=None, cookies=None, auth=None, encoding='utf-8', version=aiohttp.HttpVersion11, compress=None, chunked=None, expect100=False, loop=None, response_class=None, proxy=None, proxy_auth=None, timer=None): if loop is None: loop = asyncio.get_event_loop() assert isinstance(url, URL), url assert isinstance(proxy, (URL, type(None))), proxy if params: q = MultiDict(url.query) url2 = url.with_query(params) q.extend(url2.query) url = url.with_query(q) self.url = url.with_fragment(None) self.original_url = url self.method = method.upper() self.encoding = encoding self.chunked = chunked self.compress = compress self.loop = loop self.response_class = response_class or ClientResponse self._timer = timer if timer is not None else TimerNoop() if loop.get_debug(): self._source_traceback = traceback.extract_stack(sys._getframe(1)) self.update_version(version) self.update_host(url) self.update_headers(headers) self.update_auto_headers(skip_auto_headers) self.update_cookies(cookies) self.update_content_encoding(data) self.update_auth(auth) self.update_proxy(proxy, proxy_auth) self.update_body_from_data(data, skip_auto_headers) self.update_transfer_encoding() self.update_expect_continue(expect100) @property def host(self): return self.url.host @property def port(self): return self.url.port def update_host(self, url): """Update destination host, port and connection type (ssl).""" # get host/port if not url.host: raise ValueError( "Could not parse hostname from URL '{}'".format(url)) # basic auth info username, password = url.user, url.password if username: self.auth = helpers.BasicAuth(username, password or '') # Record entire netloc for usage in host header scheme = url.scheme self.ssl = scheme in ('https', 'wss') def update_version(self, version): """Convert request version to two elements tuple. parser HTTP version '1.1' => (1, 1) """ if isinstance(version, str): v = [l.strip() for l in version.split('.', 1)] try: version = int(v[0]), int(v[1]) except ValueError: raise ValueError( 'Can not parse http version number: {}' .format(version)) from None self.version = version def update_headers(self, headers): """Update request headers.""" self.headers = CIMultiDict() if headers: if isinstance(headers, dict): headers = headers.items() elif isinstance(headers, (MultiDictProxy, MultiDict)): headers = headers.items() for key, value in headers: self.headers.add(key, value) def update_auto_headers(self, skip_auto_headers): self.skip_auto_headers = skip_auto_headers used_headers = set(self.headers) | skip_auto_headers for hdr, val in self.DEFAULT_HEADERS.items(): if hdr not in used_headers: self.headers.add(hdr, val) # add host if hdrs.HOST not in used_headers: netloc = self.url.raw_host if not self.url.is_default_port(): netloc += ':' + str(self.url.port) self.headers[hdrs.HOST] = netloc if hdrs.USER_AGENT not in used_headers: self.headers[hdrs.USER_AGENT] = self.SERVER_SOFTWARE def update_cookies(self, cookies): """Update request cookies header.""" if not cookies: return c = SimpleCookie() if hdrs.COOKIE in self.headers: c.load(self.headers.get(hdrs.COOKIE, '')) del self.headers[hdrs.COOKIE] for name, value in cookies.items(): if isinstance(value, Morsel): # Preserve coded_value mrsl_val = value.get(value.key, Morsel()) mrsl_val.set(value.key, value.value, value.coded_value) c[name] = mrsl_val else: c[name] = value self.headers[hdrs.COOKIE] = c.output(header='', sep=';').strip() def update_content_encoding(self, data): """Set request content encoding.""" if not data: return enc = self.headers.get(hdrs.CONTENT_ENCODING, '').lower() if enc: if self.compress is not False: self.compress = enc # enable chunked, no need to deal with length self.chunked = True elif self.compress: if not isinstance(self.compress, str): self.compress = 'deflate' self.headers[hdrs.CONTENT_ENCODING] = self.compress self.chunked = True # enable chunked, no need to deal with length def update_auth(self, auth): """Set basic auth.""" if auth is None: auth = self.auth if auth is None: return if not isinstance(auth, helpers.BasicAuth): raise TypeError('BasicAuth() tuple is required instead') self.headers[hdrs.AUTHORIZATION] = auth.encode() def update_body_from_data(self, data, skip_auto_headers): if not data: return if isinstance(data, str): data = data.encode(self.encoding) if isinstance(data, (bytes, bytearray)): self.body = data if (hdrs.CONTENT_TYPE not in self.headers and hdrs.CONTENT_TYPE not in skip_auto_headers): self.headers[hdrs.CONTENT_TYPE] = 'application/octet-stream' if hdrs.CONTENT_LENGTH not in self.headers and not self.chunked: self.headers[hdrs.CONTENT_LENGTH] = str(len(self.body)) elif isinstance(data, (asyncio.StreamReader, streams.StreamReader, streams.DataQueue)): self.body = data elif asyncio.iscoroutine(data): self.body = data if (hdrs.CONTENT_LENGTH not in self.headers and self.chunked is None): self.chunked = True elif isinstance(data, io.IOBase): assert not isinstance(data, io.StringIO), \ 'attempt to send text data instead of binary' self.body = data if not self.chunked and isinstance(data, io.BytesIO): # Not chunking if content-length can be determined size = len(data.getbuffer()) self.headers[hdrs.CONTENT_LENGTH] = str(size) self.chunked = False elif (not self.chunked and isinstance(data, (io.BufferedReader, io.BufferedRandom))): # Not chunking if content-length can be determined try: size = os.fstat(data.fileno()).st_size - data.tell() self.headers[hdrs.CONTENT_LENGTH] = str(size) self.chunked = False except OSError: # data.fileno() is not supported, e.g. # io.BufferedReader(io.BytesIO(b'data')) self.chunked = True else: self.chunked = True if hasattr(data, 'mode'): if data.mode == 'r': raise ValueError('file {!r} should be open in binary mode' ''.format(data)) if (hdrs.CONTENT_TYPE not in self.headers and hdrs.CONTENT_TYPE not in skip_auto_headers and hasattr(data, 'name')): mime = mimetypes.guess_type(data.name)[0] mime = 'application/octet-stream' if mime is None else mime self.headers[hdrs.CONTENT_TYPE] = mime elif isinstance(data, MultipartWriter): self.body = data.serialize() self.headers.update(data.headers) self.chunked = self.chunked or 8192 else: if not isinstance(data, helpers.FormData): data = helpers.FormData(data) self.body = data(self.encoding) if (hdrs.CONTENT_TYPE not in self.headers and hdrs.CONTENT_TYPE not in skip_auto_headers): self.headers[hdrs.CONTENT_TYPE] = data.content_type if data.is_multipart: self.chunked = self.chunked or 8192 else: if (hdrs.CONTENT_LENGTH not in self.headers and not self.chunked): self.headers[hdrs.CONTENT_LENGTH] = str(len(self.body)) def update_transfer_encoding(self): """Analyze transfer-encoding header.""" te = self.headers.get(hdrs.TRANSFER_ENCODING, '').lower() if self.chunked: if hdrs.CONTENT_LENGTH in self.headers: del self.headers[hdrs.CONTENT_LENGTH] if 'chunked' not in te: self.headers[hdrs.TRANSFER_ENCODING] = 'chunked' self.chunked = self.chunked if type(self.chunked) is int else 8192 else: if 'chunked' in te: self.chunked = 8192 else: self.chunked = None if hdrs.CONTENT_LENGTH not in self.headers: self.headers[hdrs.CONTENT_LENGTH] = str(len(self.body)) def update_expect_continue(self, expect=False): if expect: self.headers[hdrs.EXPECT] = '100-continue' elif self.headers.get(hdrs.EXPECT, '').lower() == '100-continue': expect = True if expect: self._continue = helpers.create_future(self.loop) def update_proxy(self, proxy, proxy_auth): if proxy and not proxy.scheme == 'http': raise ValueError("Only http proxies are supported") if proxy_auth and not isinstance(proxy_auth, helpers.BasicAuth): raise ValueError("proxy_auth must be None or BasicAuth() tuple") self.proxy = proxy self.proxy_auth = proxy_auth @asyncio.coroutine def write_bytes(self, request, reader): """Support coroutines that yields bytes objects.""" # 100 response if self._continue is not None: yield from self._continue try: if asyncio.iscoroutine(self.body): exc = None value = None stream = self.body while True: try: if exc is not None: result = stream.throw(exc) else: result = stream.send(value) except StopIteration as exc: if isinstance(exc.value, bytes): yield from request.write(exc.value, drain=True) break except: self.response.close() raise if isinstance(result, asyncio.Future): exc = None value = None try: value = yield result except Exception as err: exc = err elif isinstance(result, (bytes, bytearray)): yield from request.write(result, drain=True) value = None else: raise ValueError( 'Bytes object is expected, got: %s.' % type(result)) elif isinstance(self.body, (asyncio.StreamReader, streams.StreamReader)): chunk = yield from self.body.read(streams.DEFAULT_LIMIT) while chunk: yield from request.write(chunk, drain=True) chunk = yield from self.body.read(streams.DEFAULT_LIMIT) elif isinstance(self.body, streams.DataQueue): while True: try: chunk = yield from self.body.read() if not chunk: break yield from request.write(chunk, drain=True) except streams.EofStream: break elif isinstance(self.body, io.IOBase): chunk = self.body.read(self.chunked) while chunk: request.write(chunk) chunk = self.body.read(self.chunked) else: if isinstance(self.body, (bytes, bytearray)): self.body = (self.body,) for chunk in self.body: request.write(chunk) except Exception as exc: new_exc = aiohttp.ClientRequestError( 'Can not write request body for %s' % self.url) new_exc.__context__ = exc new_exc.__cause__ = exc reader.set_exception(new_exc) else: try: ret = request.write_eof() # NB: in asyncio 3.4.1+ StreamWriter.drain() is coroutine # see bug #170 if (asyncio.iscoroutine(ret) or isinstance(ret, asyncio.Future)): yield from ret except Exception as exc: new_exc = aiohttp.ClientRequestError( 'Can not write request body for %s' % self.url) new_exc.__context__ = exc new_exc.__cause__ = exc reader.set_exception(new_exc) self._writer = None def send(self, writer, reader): # Specify request target: # - CONNECT request must send authority form URI # - not CONNECT proxy must send absolute form URI # - most common is origin form URI if self.method == hdrs.METH_CONNECT: path = '{}:{}'.format(self.url.raw_host, self.url.port) elif self.proxy and not self.ssl: path = str(self.url) else: path = self.url.raw_path if self.url.raw_query_string: path += '?' + self.url.raw_query_string request = aiohttp.Request(writer, self.method, path, self.version) if self.compress: request.add_compression_filter(self.compress) if self.chunked is not None: request.enable_chunked_encoding() request.add_chunking_filter(self.chunked) # set default content-type if (self.method in self.POST_METHODS and hdrs.CONTENT_TYPE not in self.skip_auto_headers and hdrs.CONTENT_TYPE not in self.headers): self.headers[hdrs.CONTENT_TYPE] = 'application/octet-stream' for k, value in self.headers.items(): request.add_header(k, value) request.send_headers() self._writer = helpers.ensure_future( self.write_bytes(request, reader), loop=self.loop) self.response = self.response_class( self.method, self.original_url, writer=self._writer, continue100=self._continue, timer=self._timer) self.response._post_init(self.loop) return self.response @asyncio.coroutine def close(self): if self._writer is not None: try: yield from self._writer finally: self._writer = None def terminate(self): if self._writer is not None: if not self.loop.is_closed(): self._writer.cancel() self._writer = None
class HttpRequest(RequestBase): """An :class:`HttpClient` request for an HTTP resource. This class has a similar interface to :class:`urllib.request.Request`. :param files: optional dictionary of name, file-like-objects. :param allow_redirects: allow the response to follow redirects. .. attribute:: method The request method .. attribute:: version HTTP version for this request, usually ``HTTP/1.1`` .. attribute:: history List of past :class:`.HttpResponse` (collected during redirects). .. attribute:: wait_continue if ``True``, the :class:`HttpRequest` includes the ``Expect: 100-Continue`` header. .. attribute:: stream Allow for streaming body """ _proxy = None _ssl = None _tunnel = None _write_done = False def __init__(self, client, url, method, inp_params=None, headers=None, data=None, files=None, json=None, history=None, auth=None, charset=None, max_redirects=10, source_address=None, allow_redirects=False, decompress=True, version=None, wait_continue=False, websocket_handler=None, cookies=None, params=None, stream=False, proxies=None, verify=True, cert=None, **extra): self.client = client self.method = method.upper() self.inp_params = inp_params or {} self.unredirected_headers = CIMultiDict() self.history = history self.wait_continue = wait_continue self.max_redirects = max_redirects self.allow_redirects = allow_redirects self.charset = charset or 'utf-8' self.version = version self.decompress = decompress self.websocket_handler = websocket_handler self.source_address = source_address self.stream = stream self.verify = verify self.cert = cert if auth and not isinstance(auth, Auth): auth = HTTPBasicAuth(*auth) self.auth = auth self.url = full_url(url, params, method=self.method) self._set_proxy(proxies) self.key = RequestKey.create(self) self.headers = client.get_headers(self, headers) self.body = self._encode_body(data, files, json) self.unredirected_headers['host'] = self.key.netloc cookies = cookiejar_from_dict(client.cookies, cookies) if cookies: cookies.add_cookie_header(self) @property def _loop(self): return self.client._loop @property def ssl(self): """Context for TLS connections. If this is a tunneled request and the tunnel connection is not yet established, it returns ``None``. """ return self._ssl @property def proxy(self): """Proxy server for this request. """ return self._proxy @property def tunnel(self): """Tunnel for this request. """ return self._tunnel def __repr__(self): return self.first_line() __str__ = __repr__ def first_line(self): if self.method == 'CONNECT': url = self.key.netloc elif self._proxy: url = self.url else: p = urlparse(self.url) url = urlunparse(('', '', p.path or '/', p.params, p.query, p.fragment)) return '%s %s %s' % (self.method, url, self.version) def is_chunked(self): return self.body and 'content-length' not in self.headers def encode(self): """The bytes representation of this :class:`HttpRequest`. Called by :class:`HttpResponse` when it needs to encode this :class:`HttpRequest` before sending it to the HTTP resource. """ # Call body before fist_line in case the query is changes. first_line = self.first_line() if self.body and self.wait_continue: self.headers['expect'] = '100-continue' headers = self.headers if self.unredirected_headers: headers = self.unredirected_headers.copy() headers.update(self.headers) buffer = [first_line.encode('ascii'), b'\r\n'] buffer.extend((('%s: %s\r\n' % (name, value)).encode(CHARSET) for name, value in headers.items())) buffer.append(b'\r\n') return b''.join(buffer) def add_header(self, key, value): self.headers[key] = value def has_header(self, header_name): """Check ``header_name`` is in this request headers. """ return (header_name in self.headers or header_name in self.unredirected_headers) def get_header(self, header_name, default=None): """Retrieve ``header_name`` from this request headers. """ return self.headers.get( header_name, self.unredirected_headers.get(header_name, default)) def remove_header(self, header_name): """Remove ``header_name`` from this request. """ val1 = self.headers.pop(header_name, None) val2 = self.unredirected_headers.pop(header_name, None) return val1 or val2 def add_unredirected_header(self, header_name, header_value): self.unredirected_headers[header_name] = header_value def write_body(self, transport): assert not self._write_done, 'Body already sent' self._write_done = True if not self.body: return if is_streamed(self.body): self._loop.create_task(self._write_streamed_data(transport)) else: self._write_body_data(transport, self.body, True) # INTERNAL ENCODING METHODS def _encode_body(self, data, files, json): body = None ct = None if isinstance(data, (str, bytes)): if files: raise ValueError('data cannot be a string or bytes when ' 'files are present') body = to_bytes(data, self.charset) elif data and is_streamed(data): if files: raise ValueError('data cannot be an iterator when ' 'files are present') if 'content-length' not in self.headers: self.headers['transfer-encoding'] = 'chunked' return data elif data or files: if files: body, ct = self._encode_files(data, files) else: body, ct = self._encode_params(data) elif json: body = _json.dumps(json).encode(self.charset) ct = 'application/json' if not self.headers.get('content-type') and ct: self.headers['Content-Type'] = ct if body: self.headers['content-length'] = str(len(body)) return body def _encode_files(self, data, files): fields = [] for field, val in mapping_iterator(data or ()): if (isinstance(val, str) or isinstance(val, bytes) or not hasattr(val, '__iter__')): val = [val] for v in val: if v is not None: if not isinstance(v, bytes): v = str(v) fields.append((field.decode('utf-8') if isinstance(field, bytes) else field, v.encode('utf-8') if isinstance(v, str) else v)) for (k, v) in mapping_iterator(files): # support for explicit filename ft = None if isinstance(v, (tuple, list)): if len(v) == 2: fn, fp = v else: fn, fp, ft = v else: fn = guess_filename(v) or k fp = v if isinstance(fp, bytes): fp = BytesIO(fp) elif isinstance(fp, str): fp = StringIO(fp) if ft: new_v = (fn, fp.read(), ft) else: new_v = (fn, fp.read()) fields.append((k, new_v)) # return encode_multipart_formdata(fields, charset=self.charset) def _encode_params(self, params): content_type = self.headers.get('content-type') # No content type given, chose one if not content_type: content_type = FORM_URL_ENCODED if hasattr(params, 'read'): params = params.read() if content_type in JSON_CONTENT_TYPES: body = _json.dumps(params) elif content_type == FORM_URL_ENCODED: body = urlencode(tuple(split_url_params(params))) elif content_type == MULTIPART_FORM_DATA: body, content_type = encode_multipart_formdata( params, charset=self.charset) else: body = params return to_bytes(body, self.charset), content_type def _write_body_data(self, transport, data, finish=False): if self.is_chunked(): data = http_chunks(data, finish) elif data: data = (data,) else: return for chunk in data: transport.write(chunk) async def _write_streamed_data(self, transport): for data in self.body: try: data = await data except TypeError: pass self._write_body_data(transport, data) self._write_body_data(transport, b'', True) # PROXY INTERNALS def _set_proxy(self, proxies): url = urlparse(self.url) request_proxies = self.client.proxies.copy() if proxies: request_proxies.update(proxies) self.proxies = request_proxies # if url.scheme in request_proxies: host, port = get_hostport(url.scheme, url.netloc) no_proxy = [n for n in request_proxies.get('no', '').split(',') if n] if not any(map(host.endswith, no_proxy)): proxy_url = request_proxies[url.scheme] if url.scheme in tls_schemes: self._tunnel = proxy_url else: self._proxy = proxy_url
class StreamingHTTPResponse(BaseHTTPResponse): __slots__ = ( "protocol", "streaming_fn", "status", "content_type", "headers", "chunked", "_cookies", ) def __init__( self, streaming_fn, status=200, headers=None, content_type="text/plain", chunked=True, ): self.content_type = content_type self.streaming_fn = streaming_fn self.status = status self.headers = CIMultiDict(headers or {}) self.chunked = chunked self._cookies = None async def write(self, data): """Writes a chunk of data to the streaming response. :param data: bytes-ish data to be written. """ if type(data) != bytes: data = self._encode_body(data) if self.chunked: await self.protocol.push_data(b"%x\r\n%b\r\n" % (len(data), data)) else: await self.protocol.push_data(data) await self.protocol.drain() async def stream( self, version="1.1", keep_alive=False, keep_alive_timeout=None ): """Streams headers, runs the `streaming_fn` callback that writes content to the response body, then finalizes the response body. """ if version != "1.1": self.chunked = False headers = self.get_headers( version, keep_alive=keep_alive, keep_alive_timeout=keep_alive_timeout, ) await self.protocol.push_data(headers) await self.protocol.drain() await self.streaming_fn(self) if self.chunked: await self.protocol.push_data(b"0\r\n\r\n") # no need to await drain here after this write, because it is the # very last thing we write and nothing needs to wait for it. def get_headers( self, version="1.1", keep_alive=False, keep_alive_timeout=None ): # This is all returned in a kind-of funky way # We tried to make this as fast as possible in pure python timeout_header = b"" if keep_alive and keep_alive_timeout is not None: timeout_header = b"Keep-Alive: %d\r\n" % keep_alive_timeout if self.chunked and version == "1.1": self.headers["Transfer-Encoding"] = "chunked" self.headers.pop("Content-Length", None) self.headers["Content-Type"] = self.headers.get( "Content-Type", self.content_type ) headers = self._parse_headers() if self.status == 200: status = b"OK" else: status = STATUS_CODES.get(self.status) return (b"HTTP/%b %d %b\r\n" b"%b" b"%b\r\n") % ( version.encode(), self.status, status, timeout_header, headers, )
class ClientRequest: GET_METHODS = {hdrs.METH_GET, hdrs.METH_HEAD, hdrs.METH_OPTIONS} POST_METHODS = {hdrs.METH_PATCH, hdrs.METH_POST, hdrs.METH_PUT} ALL_METHODS = GET_METHODS.union(POST_METHODS).union( {hdrs.METH_DELETE, hdrs.METH_TRACE}) DEFAULT_HEADERS = { hdrs.ACCEPT: '*/*', hdrs.ACCEPT_ENCODING: 'gzip, deflate', } body = b'' auth = None response = None response_class = None _writer = None # async task for streaming data _continue = None # waiter future for '100 Continue' response # N.B. # Adding __del__ method with self._writer closing doesn't make sense # because _writer is instance method, thus it keeps a reference to self. # Until writer has finished finalizer will not be called. def __init__(self, method, url, *, params=None, headers=None, skip_auto_headers=frozenset(), data=None, cookies=None, auth=None, version=http.HttpVersion11, compress=None, chunked=None, expect100=False, loop=None, response_class=None, proxy=None, proxy_auth=None, proxy_from_env=False, timer=None, session=None, auto_decompress=True): if loop is None: loop = asyncio.get_event_loop() assert isinstance(url, URL), url assert isinstance(proxy, (URL, type(None))), proxy self._session = session if params: q = MultiDict(url.query) url2 = url.with_query(params) q.extend(url2.query) url = url.with_query(q) self.url = url.with_fragment(None) self.original_url = url self.method = method.upper() self.chunked = chunked self.compress = compress self.loop = loop self.length = None self.response_class = response_class or ClientResponse self._timer = timer if timer is not None else TimerNoop() self._auto_decompress = auto_decompress if loop.get_debug(): self._source_traceback = traceback.extract_stack(sys._getframe(1)) self.update_version(version) self.update_host(url) self.update_headers(headers) self.update_auto_headers(skip_auto_headers) self.update_cookies(cookies) self.update_content_encoding(data) self.update_auth(auth) self.update_proxy(proxy, proxy_auth, proxy_from_env) self.update_body_from_data(data) self.update_transfer_encoding() self.update_expect_continue(expect100) @property def host(self): return self.url.host @property def port(self): return self.url.port @property def request_info(self): return RequestInfo(self.url, self.method, self.headers) def update_host(self, url): """Update destination host, port and connection type (ssl).""" # get host/port if not url.host: raise ValueError( "Could not parse hostname from URL '{}'".format(url)) # basic auth info username, password = url.user, url.password if username: self.auth = helpers.BasicAuth(username, password or '') # Record entire netloc for usage in host header scheme = url.scheme self.ssl = scheme in ('https', 'wss') def update_version(self, version): """Convert request version to two elements tuple. parser HTTP version '1.1' => (1, 1) """ if isinstance(version, str): v = [l.strip() for l in version.split('.', 1)] try: version = int(v[0]), int(v[1]) except ValueError: raise ValueError( 'Can not parse http version number: {}' .format(version)) from None self.version = version def update_headers(self, headers): """Update request headers.""" self.headers = CIMultiDict() if headers: if isinstance(headers, (dict, MultiDictProxy, MultiDict)): headers = headers.items() for key, value in headers: self.headers.add(key, value) def update_auto_headers(self, skip_auto_headers): self.skip_auto_headers = CIMultiDict( (hdr, None) for hdr in sorted(skip_auto_headers)) used_headers = self.headers.copy() used_headers.extend(self.skip_auto_headers) for hdr, val in self.DEFAULT_HEADERS.items(): if hdr not in used_headers: self.headers.add(hdr, val) # add host if hdrs.HOST not in used_headers: netloc = self.url.raw_host if not self.url.is_default_port(): netloc += ':' + str(self.url.port) self.headers[hdrs.HOST] = netloc if hdrs.USER_AGENT not in used_headers: self.headers[hdrs.USER_AGENT] = SERVER_SOFTWARE def update_cookies(self, cookies): """Update request cookies header.""" if not cookies: return c = SimpleCookie() if hdrs.COOKIE in self.headers: c.load(self.headers.get(hdrs.COOKIE, '')) del self.headers[hdrs.COOKIE] for name, value in cookies.items(): if isinstance(value, Morsel): # Preserve coded_value mrsl_val = value.get(value.key, Morsel()) mrsl_val.set(value.key, value.value, value.coded_value) c[name] = mrsl_val else: c[name] = value self.headers[hdrs.COOKIE] = c.output(header='', sep=';').strip() def update_content_encoding(self, data): """Set request content encoding.""" if not data: return enc = self.headers.get(hdrs.CONTENT_ENCODING, '').lower() if enc: if self.compress: raise ValueError( 'compress can not be set ' 'if Content-Encoding header is set') elif self.compress: if not isinstance(self.compress, str): self.compress = 'deflate' self.headers[hdrs.CONTENT_ENCODING] = self.compress self.chunked = True # enable chunked, no need to deal with length def update_transfer_encoding(self): """Analyze transfer-encoding header.""" te = self.headers.get(hdrs.TRANSFER_ENCODING, '').lower() if 'chunked' in te: if self.chunked: raise ValueError( 'chunked can not be set ' 'if "Transfer-Encoding: chunked" header is set') elif self.chunked: if hdrs.CONTENT_LENGTH in self.headers: raise ValueError( 'chunked can not be set ' 'if Content-Length header is set') self.headers[hdrs.TRANSFER_ENCODING] = 'chunked' else: if hdrs.CONTENT_LENGTH not in self.headers: self.headers[hdrs.CONTENT_LENGTH] = str(len(self.body)) def update_auth(self, auth): """Set basic auth.""" if auth is None: auth = self.auth if auth is None: return if not isinstance(auth, helpers.BasicAuth): raise TypeError('BasicAuth() tuple is required instead') self.headers[hdrs.AUTHORIZATION] = auth.encode() def update_body_from_data(self, body): if not body: return # FormData if isinstance(body, FormData): body = body() try: body = payload.PAYLOAD_REGISTRY.get(body, disposition=None) except payload.LookupError: body = FormData(body)() self.body = body # enable chunked encoding if needed if not self.chunked: if hdrs.CONTENT_LENGTH not in self.headers: size = body.size if size is None: self.chunked = True else: if hdrs.CONTENT_LENGTH not in self.headers: self.headers[hdrs.CONTENT_LENGTH] = str(size) # set content-type if (hdrs.CONTENT_TYPE not in self.headers and hdrs.CONTENT_TYPE not in self.skip_auto_headers): self.headers[hdrs.CONTENT_TYPE] = body.content_type # copy payload headers if body.headers: for (key, value) in body.headers.items(): if key not in self.headers: self.headers[key] = value def update_expect_continue(self, expect=False): if expect: self.headers[hdrs.EXPECT] = '100-continue' elif self.headers.get(hdrs.EXPECT, '').lower() == '100-continue': expect = True if expect: self._continue = helpers.create_future(self.loop) def update_proxy(self, proxy, proxy_auth, proxy_from_env): if proxy_from_env and not proxy: proxy_url = getproxies().get(self.original_url.scheme) proxy = URL(proxy_url) if proxy_url else None if proxy and not proxy.scheme == 'http': raise ValueError("Only http proxies are supported") if proxy_auth and not isinstance(proxy_auth, helpers.BasicAuth): raise ValueError("proxy_auth must be None or BasicAuth() tuple") self.proxy = proxy self.proxy_auth = proxy_auth def keep_alive(self): if self.version < HttpVersion10: # keep alive not supported at all return False if self.version == HttpVersion10: if self.headers.get(hdrs.CONNECTION) == 'keep-alive': return True else: # no headers means we close for Http 1.0 return False elif self.headers.get(hdrs.CONNECTION) == 'close': return False return True @asyncio.coroutine def write_bytes(self, writer, conn): """Support coroutines that yields bytes objects.""" # 100 response if self._continue is not None: yield from writer.drain() yield from self._continue try: if isinstance(self.body, payload.Payload): yield from self.body.write(writer) else: if isinstance(self.body, (bytes, bytearray)): self.body = (self.body,) for chunk in self.body: writer.write(chunk) yield from writer.write_eof() except OSError as exc: new_exc = ClientOSError( exc.errno, 'Can not write request body for %s' % self.url) new_exc.__context__ = exc new_exc.__cause__ = exc conn.protocol.set_exception(new_exc) except asyncio.CancelledError as exc: if not conn.closed: conn.protocol.set_exception(exc) except Exception as exc: conn.protocol.set_exception(exc) finally: self._writer = None def send(self, conn): # Specify request target: # - CONNECT request must send authority form URI # - not CONNECT proxy must send absolute form URI # - most common is origin form URI if self.method == hdrs.METH_CONNECT: path = '{}:{}'.format(self.url.raw_host, self.url.port) elif self.proxy and not self.ssl: path = str(self.url) else: path = self.url.raw_path if self.url.raw_query_string: path += '?' + self.url.raw_query_string writer = PayloadWriter(conn.writer, self.loop) if self.compress: writer.enable_compression(self.compress) if self.chunked is not None: writer.enable_chunking() # set default content-type if (self.method in self.POST_METHODS and hdrs.CONTENT_TYPE not in self.skip_auto_headers and hdrs.CONTENT_TYPE not in self.headers): self.headers[hdrs.CONTENT_TYPE] = 'application/octet-stream' # set the connection header connection = self.headers.get(hdrs.CONNECTION) if not connection: if self.keep_alive(): if self.version == HttpVersion10: connection = 'keep-alive' else: if self.version == HttpVersion11: connection = 'close' if connection is not None: self.headers[hdrs.CONNECTION] = connection # status + headers status_line = '{0} {1} HTTP/{2[0]}.{2[1]}\r\n'.format( self.method, path, self.version) writer.write_headers(status_line, self.headers) self._writer = helpers.ensure_future( self.write_bytes(writer, conn), loop=self.loop) self.response = self.response_class( self.method, self.original_url, writer=self._writer, continue100=self._continue, timer=self._timer, request_info=self.request_info, auto_decompress=self._auto_decompress ) self.response._post_init(self.loop, self._session) return self.response @asyncio.coroutine def close(self): if self._writer is not None: try: yield from self._writer finally: self._writer = None def terminate(self): if self._writer is not None: if not self.loop.is_closed(): self._writer.cancel() self._writer = None
class ClientRequest: GET_METHODS = { hdrs.METH_GET, hdrs.METH_HEAD, hdrs.METH_OPTIONS, hdrs.METH_TRACE, } POST_METHODS = {hdrs.METH_PATCH, hdrs.METH_POST, hdrs.METH_PUT} ALL_METHODS = GET_METHODS.union(POST_METHODS).union({hdrs.METH_DELETE}) DEFAULT_HEADERS = { hdrs.ACCEPT: '*/*', hdrs.ACCEPT_ENCODING: 'gzip, deflate', } body = b'' auth = None response = None response_class = None _writer = None # async task for streaming data _continue = None # waiter future for '100 Continue' response # N.B. # Adding __del__ method with self._writer closing doesn't make sense # because _writer is instance method, thus it keeps a reference to self. # Until writer has finished finalizer will not be called. def __init__(self, method, url, *, params=None, headers=None, skip_auto_headers=frozenset(), data=None, cookies=None, auth=None, version=http.HttpVersion11, compress=None, chunked=None, expect100=False, loop=None, response_class=None, proxy=None, proxy_auth=None, timer=None, session=None, auto_decompress=True, verify_ssl=None, fingerprint=None, ssl_context=None, proxy_headers=None): if verify_ssl is False and ssl_context is not None: raise ValueError( "Either disable ssl certificate validation by " "verify_ssl=False or specify ssl_context, not both.") if loop is None: loop = asyncio.get_event_loop() assert isinstance(url, URL), url assert isinstance(proxy, (URL, type(None))), proxy self._session = session if params: q = MultiDict(url.query) url2 = url.with_query(params) q.extend(url2.query) url = url.with_query(q) self.url = url.with_fragment(None) self.original_url = url self.method = method.upper() self.chunked = chunked self.compress = compress self.loop = loop self.length = None self.response_class = response_class or ClientResponse self._timer = timer if timer is not None else TimerNoop() self._auto_decompress = auto_decompress self._verify_ssl = verify_ssl self._ssl_context = ssl_context if loop.get_debug(): self._source_traceback = traceback.extract_stack(sys._getframe(1)) self.update_version(version) self.update_host(url) self.update_headers(headers) self.update_auto_headers(skip_auto_headers) self.update_cookies(cookies) self.update_content_encoding(data) self.update_auth(auth) self.update_proxy(proxy, proxy_auth, proxy_headers) self.update_fingerprint(fingerprint) self.update_body_from_data(data) if data or self.method not in self.GET_METHODS: self.update_transfer_encoding() self.update_expect_continue(expect100) @property def connection_key(self): return ConnectionKey(self.host, self.port, self.ssl) @property def host(self): return self.url.host @property def port(self): return self.url.port @property def request_info(self): return RequestInfo(self.url, self.method, self.headers) def update_host(self, url): """Update destination host, port and connection type (ssl).""" # get host/port if not url.host: raise InvalidURL(url) # basic auth info username, password = url.user, url.password if username: self.auth = helpers.BasicAuth(username, password or '') # Record entire netloc for usage in host header scheme = url.scheme self.ssl = scheme in ('https', 'wss') def update_version(self, version): """Convert request version to two elements tuple. parser HTTP version '1.1' => (1, 1) """ if isinstance(version, str): v = [l.strip() for l in version.split('.', 1)] try: version = int(v[0]), int(v[1]) except ValueError: raise ValueError( 'Can not parse http version number: {}'.format( version)) from None self.version = version def update_headers(self, headers): """Update request headers.""" self.headers = CIMultiDict() if headers: if isinstance(headers, (dict, MultiDictProxy, MultiDict)): headers = headers.items() for key, value in headers: self.headers.add(key, value) def update_auto_headers(self, skip_auto_headers): self.skip_auto_headers = CIMultiDict( (hdr, None) for hdr in sorted(skip_auto_headers)) used_headers = self.headers.copy() used_headers.extend(self.skip_auto_headers) for hdr, val in self.DEFAULT_HEADERS.items(): if hdr not in used_headers: self.headers.add(hdr, val) # add host if hdrs.HOST not in used_headers: netloc = self.url.raw_host if not self.url.is_default_port(): netloc += ':' + str(self.url.port) self.headers[hdrs.HOST] = netloc if hdrs.USER_AGENT not in used_headers: self.headers[hdrs.USER_AGENT] = SERVER_SOFTWARE def update_cookies(self, cookies): """Update request cookies header.""" if not cookies: return c = SimpleCookie() if hdrs.COOKIE in self.headers: c.load(self.headers.get(hdrs.COOKIE, '')) del self.headers[hdrs.COOKIE] for name, value in cookies.items(): if isinstance(value, Morsel): # Preserve coded_value mrsl_val = value.get(value.key, Morsel()) mrsl_val.set(value.key, value.value, value.coded_value) c[name] = mrsl_val else: c[name] = value self.headers[hdrs.COOKIE] = c.output(header='', sep=';').strip() def update_content_encoding(self, data): """Set request content encoding.""" if not data: return enc = self.headers.get(hdrs.CONTENT_ENCODING, '').lower() if enc: if self.compress: raise ValueError('compress can not be set ' 'if Content-Encoding header is set') elif self.compress: if not isinstance(self.compress, str): self.compress = 'deflate' self.headers[hdrs.CONTENT_ENCODING] = self.compress self.chunked = True # enable chunked, no need to deal with length def update_transfer_encoding(self): """Analyze transfer-encoding header.""" te = self.headers.get(hdrs.TRANSFER_ENCODING, '').lower() if 'chunked' in te: if self.chunked: raise ValueError( 'chunked can not be set ' 'if "Transfer-Encoding: chunked" header is set') elif self.chunked: if hdrs.CONTENT_LENGTH in self.headers: raise ValueError('chunked can not be set ' 'if Content-Length header is set') self.headers[hdrs.TRANSFER_ENCODING] = 'chunked' else: if hdrs.CONTENT_LENGTH not in self.headers: self.headers[hdrs.CONTENT_LENGTH] = str(len(self.body)) def update_auth(self, auth): """Set basic auth.""" if auth is None: auth = self.auth if auth is None: return if not isinstance(auth, helpers.BasicAuth): raise TypeError('BasicAuth() tuple is required instead') self.headers[hdrs.AUTHORIZATION] = auth.encode() def update_body_from_data(self, body): if not body: return # FormData if isinstance(body, FormData): body = body() try: body = payload.PAYLOAD_REGISTRY.get(body, disposition=None) except payload.LookupError: body = FormData(body)() self.body = body # enable chunked encoding if needed if not self.chunked: if hdrs.CONTENT_LENGTH not in self.headers: size = body.size if size is None: self.chunked = True else: if hdrs.CONTENT_LENGTH not in self.headers: self.headers[hdrs.CONTENT_LENGTH] = str(size) # set content-type if (hdrs.CONTENT_TYPE not in self.headers and hdrs.CONTENT_TYPE not in self.skip_auto_headers): self.headers[hdrs.CONTENT_TYPE] = body.content_type # copy payload headers if body.headers: for (key, value) in body.headers.items(): if key not in self.headers: self.headers[key] = value def update_expect_continue(self, expect=False): if expect: self.headers[hdrs.EXPECT] = '100-continue' elif self.headers.get(hdrs.EXPECT, '').lower() == '100-continue': expect = True if expect: self._continue = helpers.create_future(self.loop) def update_proxy(self, proxy, proxy_auth, proxy_headers): if proxy and not proxy.scheme == 'http': raise ValueError("Only http proxies are supported") if proxy_auth and not isinstance(proxy_auth, helpers.BasicAuth): raise ValueError("proxy_auth must be None or BasicAuth() tuple") self.proxy = proxy self.proxy_auth = proxy_auth self.proxy_headers = proxy_headers def update_fingerprint(self, fingerprint): if fingerprint: digestlen = len(fingerprint) hashfunc = HASHFUNC_BY_DIGESTLEN.get(digestlen) if not hashfunc: raise ValueError('fingerprint has invalid length') elif hashfunc is md5 or hashfunc is sha1: warnings.warn( 'md5 and sha1 are insecure and deprecated. ' 'Use sha256.', DeprecationWarning, stacklevel=2) client_logger.warn('md5 and sha1 are insecure and deprecated. ' 'Use sha256.') self._hashfunc = hashfunc self._fingerprint = fingerprint @property def verify_ssl(self): """Do check for ssl certifications?""" return self._verify_ssl @property def fingerprint(self): """Expected ssl certificate fingerprint.""" return self._fingerprint @property def ssl_context(self): """SSLContext instance for https requests.""" return self._ssl_context def keep_alive(self): if self.version < HttpVersion10: # keep alive not supported at all return False if self.version == HttpVersion10: if self.headers.get(hdrs.CONNECTION) == 'keep-alive': return True else: # no headers means we close for Http 1.0 return False elif self.headers.get(hdrs.CONNECTION) == 'close': return False return True @asyncio.coroutine def write_bytes(self, writer, conn): """Support coroutines that yields bytes objects.""" # 100 response if self._continue is not None: yield from writer.drain() yield from self._continue try: if isinstance(self.body, payload.Payload): yield from self.body.write(writer) else: if isinstance(self.body, (bytes, bytearray)): self.body = (self.body, ) for chunk in self.body: writer.write(chunk) yield from writer.write_eof() except OSError as exc: new_exc = ClientOSError( exc.errno, 'Can not write request body for %s' % self.url) new_exc.__context__ = exc new_exc.__cause__ = exc conn.protocol.set_exception(new_exc) except asyncio.CancelledError as exc: if not conn.closed: conn.protocol.set_exception(exc) except Exception as exc: conn.protocol.set_exception(exc) finally: self._writer = None def send(self, conn): # Specify request target: # - CONNECT request must send authority form URI # - not CONNECT proxy must send absolute form URI # - most common is origin form URI if self.method == hdrs.METH_CONNECT: path = '{}:{}'.format(self.url.raw_host, self.url.port) elif self.proxy and not self.ssl: path = str(self.url) else: path = self.url.raw_path if self.url.raw_query_string: path += '?' + self.url.raw_query_string writer = PayloadWriter(conn.writer, self.loop) if self.compress: writer.enable_compression(self.compress) if self.chunked is not None: writer.enable_chunking() # set default content-type if (self.method in self.POST_METHODS and hdrs.CONTENT_TYPE not in self.skip_auto_headers and hdrs.CONTENT_TYPE not in self.headers): self.headers[hdrs.CONTENT_TYPE] = 'application/octet-stream' # set the connection header connection = self.headers.get(hdrs.CONNECTION) if not connection: if self.keep_alive(): if self.version == HttpVersion10: connection = 'keep-alive' else: if self.version == HttpVersion11: connection = 'close' if connection is not None: self.headers[hdrs.CONNECTION] = connection # status + headers status_line = '{0} {1} HTTP/{2[0]}.{2[1]}\r\n'.format( self.method, path, self.version) writer.write_headers(status_line, self.headers) self._writer = helpers.ensure_future(self.write_bytes(writer, conn), loop=self.loop) self.response = self.response_class( self.method, self.original_url, writer=self._writer, continue100=self._continue, timer=self._timer, request_info=self.request_info, auto_decompress=self._auto_decompress) self.response._post_init(self.loop, self._session) return self.response @asyncio.coroutine def close(self): if self._writer is not None: try: yield from self._writer finally: self._writer = None def terminate(self): if self._writer is not None: if not self.loop.is_closed(): self._writer.cancel() self._writer = None
class StreamResponse(BaseClass, HeadersMixin): _length_check = True def __init__(self, *, status: int=200, reason: Optional[str]=None, headers: Optional[LooseHeaders]=None) -> None: self._body = None self._keep_alive = None # type: Optional[bool] self._chunked = False self._compression = False self._compression_force = None # type: Optional[ContentCoding] self._cookies = SimpleCookie() self._req = None # type: Optional[BaseRequest] self._payload_writer = None # type: Optional[AbstractStreamWriter] self._eof_sent = False self._body_length = 0 self._state = {} # type: Dict[str, Any] if headers is not None: self._headers = CIMultiDict(headers) # type: CIMultiDict[str] else: self._headers = CIMultiDict() # type: CIMultiDict[str] self.set_status(status, reason) @property def prepared(self) -> bool: return self._payload_writer is not None @property def task(self) -> 'asyncio.Task[None]': return getattr(self._req, 'task', None) @property def status(self) -> int: return self._status @property def chunked(self) -> bool: return self._chunked @property def compression(self) -> bool: return self._compression @property def reason(self) -> str: return self._reason def set_status(self, status: int, reason: Optional[str]=None, _RESPONSES: Mapping[int, Tuple[str, str]]=RESPONSES) -> None: assert not self.prepared, \ 'Cannot change the response status code after ' \ 'the headers have been sent' self._status = int(status) if reason is None: try: reason = _RESPONSES[self._status][0] except Exception: reason = '' self._reason = reason @property def keep_alive(self) -> Optional[bool]: return self._keep_alive def force_close(self) -> None: self._keep_alive = False @property def body_length(self) -> int: return self._body_length @property def output_length(self) -> int: warnings.warn('output_length is deprecated', DeprecationWarning) assert self._payload_writer return self._payload_writer.buffer_size def enable_chunked_encoding(self, chunk_size: Optional[int]=None) -> None: """Enables automatic chunked transfer encoding.""" self._chunked = True if hdrs.CONTENT_LENGTH in self._headers: raise RuntimeError("You can't enable chunked encoding when " "a content length is set") if chunk_size is not None: warnings.warn('Chunk size is deprecated #1615', DeprecationWarning) def enable_compression(self, force: Optional[Union[bool, ContentCoding]]=None ) -> None: """Enables response compression encoding.""" # Backwards compatibility for when force was a bool <0.17. if type(force) == bool: force = ContentCoding.deflate if force else ContentCoding.identity warnings.warn("Using boolean for force is deprecated #3318", DeprecationWarning) elif force is not None: assert isinstance(force, ContentCoding), ("force should one of " "None, bool or " "ContentEncoding") self._compression = True self._compression_force = force @property def headers(self) -> 'CIMultiDict[str]': return self._headers @property def cookies(self) -> SimpleCookie: return self._cookies def set_cookie(self, name: str, value: str, *, expires: Optional[str]=None, domain: Optional[str]=None, max_age: Optional[Union[int, str]]=None, path: str='/', secure: Optional[str]=None, httponly: Optional[str]=None, version: Optional[str]=None) -> None: """Set or update response cookie. Sets new cookie or updates existent with new value. Also updates only those params which are not None. """ old = self._cookies.get(name) if old is not None and old.coded_value == '': # deleted cookie self._cookies.pop(name, None) self._cookies[name] = value c = self._cookies[name] if expires is not None: c['expires'] = expires elif c.get('expires') == 'Thu, 01 Jan 1970 00:00:00 GMT': del c['expires'] if domain is not None: c['domain'] = domain if max_age is not None: c['max-age'] = str(max_age) elif 'max-age' in c: del c['max-age'] c['path'] = path if secure is not None: c['secure'] = secure if httponly is not None: c['httponly'] = httponly if version is not None: c['version'] = version def del_cookie(self, name: str, *, domain: Optional[str]=None, path: str='/') -> None: """Delete cookie. Creates new empty expired cookie. """ # TODO: do we need domain/path here? self._cookies.pop(name, None) self.set_cookie(name, '', max_age=0, expires="Thu, 01 Jan 1970 00:00:00 GMT", domain=domain, path=path) @property def content_length(self) -> Optional[int]: # Just a placeholder for adding setter return super().content_length @content_length.setter def content_length(self, value: Optional[int]) -> None: if value is not None: value = int(value) if self._chunked: raise RuntimeError("You can't set content length when " "chunked encoding is enable") self._headers[hdrs.CONTENT_LENGTH] = str(value) else: self._headers.pop(hdrs.CONTENT_LENGTH, None) @property def content_type(self) -> str: # Just a placeholder for adding setter return super().content_type @content_type.setter def content_type(self, value: str) -> None: self.content_type # read header values if needed self._content_type = str(value) self._generate_content_type_header() @property def charset(self) -> Optional[str]: # Just a placeholder for adding setter return super().charset @charset.setter def charset(self, value: Optional[str]) -> None: ctype = self.content_type # read header values if needed if ctype == 'application/octet-stream': raise RuntimeError("Setting charset for application/octet-stream " "doesn't make sense, setup content_type first") assert self._content_dict is not None if value is None: self._content_dict.pop('charset', None) else: self._content_dict['charset'] = str(value).lower() self._generate_content_type_header() @property def last_modified(self) -> Optional[datetime.datetime]: """The value of Last-Modified HTTP header, or None. This header is represented as a `datetime` object. """ httpdate = self._headers.get(hdrs.LAST_MODIFIED) if httpdate is not None: timetuple = parsedate(httpdate) if timetuple is not None: return datetime.datetime(*timetuple[:6], tzinfo=datetime.timezone.utc) return None @last_modified.setter def last_modified(self, value: Optional[ Union[int, float, datetime.datetime, str]]) -> None: if value is None: self._headers.pop(hdrs.LAST_MODIFIED, None) elif isinstance(value, (int, float)): self._headers[hdrs.LAST_MODIFIED] = time.strftime( "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(math.ceil(value))) elif isinstance(value, datetime.datetime): self._headers[hdrs.LAST_MODIFIED] = time.strftime( "%a, %d %b %Y %H:%M:%S GMT", value.utctimetuple()) elif isinstance(value, str): self._headers[hdrs.LAST_MODIFIED] = value def _generate_content_type_header( self, CONTENT_TYPE: istr=hdrs.CONTENT_TYPE) -> None: assert self._content_dict is not None assert self._content_type is not None params = '; '.join("{}={}".format(k, v) for k, v in self._content_dict.items()) if params: ctype = self._content_type + '; ' + params else: ctype = self._content_type self._headers[CONTENT_TYPE] = ctype async def _do_start_compression(self, coding: ContentCoding) -> None: if coding != ContentCoding.identity: assert self._payload_writer is not None self._headers[hdrs.CONTENT_ENCODING] = coding.value self._payload_writer.enable_compression(coding.value) # Compressed payload may have different content length, # remove the header self._headers.popall(hdrs.CONTENT_LENGTH, None) async def _start_compression(self, request: 'BaseRequest') -> None: if self._compression_force: await self._do_start_compression(self._compression_force) else: accept_encoding = request.headers.get( hdrs.ACCEPT_ENCODING, '').lower() for coding in ContentCoding: if coding.value in accept_encoding: await self._do_start_compression(coding) return async def prepare( self, request: 'BaseRequest' ) -> Optional[AbstractStreamWriter]: if self._eof_sent: return None if self._payload_writer is not None: return self._payload_writer await request._prepare_hook(self) return await self._start(request) async def _start(self, request: 'BaseRequest') -> AbstractStreamWriter: self._req = request keep_alive = self._keep_alive if keep_alive is None: keep_alive = request.keep_alive self._keep_alive = keep_alive version = request.version writer = self._payload_writer = request._payload_writer headers = self._headers for cookie in self._cookies.values(): value = cookie.output(header='')[1:] headers.add(hdrs.SET_COOKIE, value) if self._compression: await self._start_compression(request) if self._chunked: if version != HttpVersion11: raise RuntimeError( "Using chunked encoding is forbidden " "for HTTP/{0.major}.{0.minor}".format(request.version)) writer.enable_chunking() headers[hdrs.TRANSFER_ENCODING] = 'chunked' if hdrs.CONTENT_LENGTH in headers: del headers[hdrs.CONTENT_LENGTH] elif self._length_check: writer.length = self.content_length if writer.length is None: if version >= HttpVersion11: writer.enable_chunking() headers[hdrs.TRANSFER_ENCODING] = 'chunked' if hdrs.CONTENT_LENGTH in headers: del headers[hdrs.CONTENT_LENGTH] else: keep_alive = False headers.setdefault(hdrs.CONTENT_TYPE, 'application/octet-stream') headers.setdefault(hdrs.DATE, rfc822_formatted_time()) headers.setdefault(hdrs.SERVER, SERVER_SOFTWARE) # connection header if hdrs.CONNECTION not in headers: if keep_alive: if version == HttpVersion10: headers[hdrs.CONNECTION] = 'keep-alive' else: if version == HttpVersion11: headers[hdrs.CONNECTION] = 'close' # status line status_line = 'HTTP/{}.{} {} {}'.format( version[0], version[1], self._status, self._reason) await writer.write_headers(status_line, headers) return writer async def write(self, data: bytes) -> None: assert isinstance(data, (bytes, bytearray, memoryview)), \ "data argument must be byte-ish (%r)" % type(data) if self._eof_sent: raise RuntimeError("Cannot call write() after write_eof()") if self._payload_writer is None: raise RuntimeError("Cannot call write() before prepare()") await self._payload_writer.write(data) async def drain(self) -> None: assert not self._eof_sent, "EOF has already been sent" assert self._payload_writer is not None, \ "Response has not been started" warnings.warn("drain method is deprecated, use await resp.write()", DeprecationWarning, stacklevel=2) await self._payload_writer.drain() async def write_eof(self, data: bytes=b'') -> None: assert isinstance(data, (bytes, bytearray, memoryview)), \ "data argument must be byte-ish (%r)" % type(data) if self._eof_sent: return assert self._payload_writer is not None, \ "Response has not been started" await self._payload_writer.write_eof(data) self._eof_sent = True self._req = None self._body_length = self._payload_writer.output_size self._payload_writer = None def __repr__(self) -> str: if self._eof_sent: info = "eof" elif self.prepared: assert self._req is not None info = "{} {} ".format(self._req.method, self._req.path) else: info = "not prepared" return "<{} {} {}>".format(self.__class__.__name__, self.reason, info) def __getitem__(self, key: str) -> Any: return self._state[key] def __setitem__(self, key: str, value: Any) -> None: self._state[key] = value def __delitem__(self, key: str) -> None: del self._state[key] def __len__(self) -> int: return len(self._state) def __iter__(self) -> Iterator[str]: return iter(self._state) def __hash__(self) -> int: return hash(id(self)) def __eq__(self, other: object) -> bool: return self is other
def make_mocked_request(method, path, headers=None, *, version=HttpVersion(1, 1), closing=False, app=None, writer=sentinel, protocol=sentinel, transport=sentinel, payload=sentinel, sslcontext=None, secure_proxy_ssl_header=None): """Creates mocked web.Request testing purposes. Useful in unit tests, when spinning full web server is overkill or specific conditions and errors are hard to trigger. """ if version < HttpVersion(1, 1): closing = True if headers: headers = CIMultiDict(headers) raw_hdrs = [ (k.encode('utf-8'), v.encode('utf-8')) for k, v in headers.items()] else: headers = CIMultiDict() raw_hdrs = [] chunked = 'chunked' in headers.get(hdrs.TRANSFER_ENCODING, '').lower() message = RawRequestMessage(method, path, version, headers, raw_hdrs, closing, False, False, chunked) if app is None: app = _create_app_mock() if protocol is sentinel: protocol = mock.Mock() if transport is sentinel: transport = _create_transport(sslcontext) if writer is sentinel: writer = mock.Mock() writer.transport = transport protocol.transport = transport protocol.writer = writer if payload is sentinel: payload = mock.Mock() time_service = mock.Mock() time_service.time.return_value = 12345 time_service.strtime.return_value = "Tue, 15 Nov 1994 08:12:31 GMT" @contextmanager def timeout(*args, **kw): yield time_service.timeout = mock.Mock() time_service.timeout.side_effect = timeout task = mock.Mock() loop = mock.Mock() loop.create_future.return_value = () req = Request(message, payload, protocol, time_service, task, loop=loop, secure_proxy_ssl_header=secure_proxy_ssl_header) match_info = UrlMappingMatchInfo({}, mock.Mock()) match_info.add_app(app) req._match_info = match_info return req
class HttpMessage(PayloadWriter): """HttpMessage allows to write headers and payload to a stream.""" HOP_HEADERS = None # Must be set by subclass. SERVER_SOFTWARE = 'Python/{0[0]}.{0[1]} aiohttp/{1}'.format( sys.version_info, aiohttp.__version__) upgrade = False # Connection: UPGRADE websocket = False # Upgrade: WEBSOCKET has_chunked_hdr = False # Transfer-encoding: chunked def __init__(self, transport, version, close, loop=None): super().__init__(transport, loop) self.version = version self.closing = close self.keepalive = None self.length = None self.headers = CIMultiDict() self.headers_sent = False @property def body_length(self): return self.output_length def force_close(self): self.closing = True self.keepalive = False def keep_alive(self): if self.keepalive is None: if self.version < HttpVersion10: # keep alive not supported at all return False if self.version == HttpVersion10: if self.headers.get(hdrs.CONNECTION) == 'keep-alive': return True else: # no headers means we close for Http 1.0 return False else: return not self.closing else: return self.keepalive def is_headers_sent(self): return self.headers_sent def add_header(self, name, value): """Analyze headers. Calculate content length, removes hop headers, etc.""" assert not self.headers_sent, 'headers have been sent already' assert isinstance(name, str), \ 'Header name should be a string, got {!r}'.format(name) assert set(name).issubset(ASCIISET), \ 'Header name should contain ASCII chars, got {!r}'.format(name) assert isinstance(value, str), \ 'Header {!r} should have string value, got {!r}'.format( name, value) name = istr(name) value = value.strip() if name == hdrs.CONTENT_LENGTH: self.length = int(value) if name == hdrs.TRANSFER_ENCODING: self.has_chunked_hdr = value.lower() == 'chunked' if name == hdrs.CONNECTION: val = value.lower() # handle websocket if 'upgrade' in val: self.upgrade = True # connection keep-alive elif 'close' in val: self.keepalive = False elif 'keep-alive' in val: self.keepalive = True elif name == hdrs.UPGRADE: if 'websocket' in value.lower(): self.websocket = True self.headers[name] = value elif name not in self.HOP_HEADERS: # ignore hop-by-hop headers self.headers.add(name, value) def add_headers(self, *headers): """Adds headers to a HTTP message.""" for name, value in headers: self.add_header(name, value) def send_headers(self, _sep=': ', _end='\r\n'): """Writes headers to a stream. Constructs payload writer.""" # Chunked response is only for HTTP/1.1 clients or newer # and there is no Content-Length header is set. # Do not use chunked responses when the response is guaranteed to # not have a response body (304, 204). assert not self.headers_sent, 'headers have been sent already' self.headers_sent = True if not self.chunked and self.autochunked(): self.enable_chunking() if self.chunked: self.headers[hdrs.TRANSFER_ENCODING] = 'chunked' self._add_default_headers() # status + headers headers = self.status_line + ''.join( [k + _sep + v + _end for k, v in self.headers.items()]) headers = headers.encode('utf-8') + b'\r\n' self.buffer_data(headers) def _add_default_headers(self): # set the connection header connection = None if self.upgrade: connection = 'Upgrade' elif not self.closing if self.keepalive is None else self.keepalive: if self.version == HttpVersion10: connection = 'keep-alive' else: if self.version == HttpVersion11: connection = 'close' if connection is not None: self.headers[hdrs.CONNECTION] = connection
class HttpMessage(ABC, PayloadWriter): """HttpMessage allows to write headers and payload to a stream.""" HOP_HEADERS = None # Must be set by subclass. SERVER_SOFTWARE = 'Python/{0[0]}.{0[1]} aiohttp/{1}'.format( sys.version_info, aiohttp.__version__) upgrade = False # Connection: UPGRADE websocket = False # Upgrade: WEBSOCKET has_chunked_hdr = False # Transfer-encoding: chunked def __init__(self, transport, version, close, loop=None): super().__init__(transport, loop) self._version = version self.closing = close self.keepalive = None self.length = None self.headers = CIMultiDict() self.headers_sent = False @property @abstractmethod def status_line(self): return b'' @abstractmethod def autochunked(self): return False @property def version(self): return self._version @property def body_length(self): return self.output_length def force_close(self): self.closing = True self.keepalive = False def keep_alive(self): if self.keepalive is None: if self._version < HttpVersion10: # keep alive not supported at all return False if self._version == HttpVersion10: if self.headers.get(hdrs.CONNECTION) == 'keep-alive': return True else: # no headers means we close for Http 1.0 return False else: return not self.closing else: return self.keepalive def is_headers_sent(self): return self.headers_sent def add_header(self, name, value): """Analyze headers. Calculate content length, removes hop headers, etc.""" assert not self.headers_sent, 'headers have been sent already' assert isinstance(name, str), \ 'Header name should be a string, got {!r}'.format(name) assert set(name).issubset(ASCIISET), \ 'Header name should contain ASCII chars, got {!r}'.format(name) assert isinstance(value, str), \ 'Header {!r} should have string value, got {!r}'.format( name, value) name = istr(name) value = value.strip() if name == hdrs.CONTENT_LENGTH: self.length = int(value) if name == hdrs.TRANSFER_ENCODING: self.has_chunked_hdr = value.lower() == 'chunked' if name == hdrs.CONNECTION: val = value.lower() # handle websocket if 'upgrade' in val: self.upgrade = True # connection keep-alive elif 'close' in val: self.keepalive = False elif 'keep-alive' in val: self.keepalive = True elif name == hdrs.UPGRADE: if 'websocket' in value.lower(): self.websocket = True self.headers[name] = value elif name not in self.HOP_HEADERS: # ignore hop-by-hop headers self.headers.add(name, value) def add_headers(self, *headers): """Adds headers to a HTTP message.""" for name, value in headers: self.add_header(name, value) def send_headers(self, _sep=': ', _end='\r\n'): """Writes headers to a stream. Constructs payload writer.""" # Chunked response is only for HTTP/1.1 clients or newer # and there is no Content-Length header is set. # Do not use chunked responses when the response is guaranteed to # not have a response body (304, 204). assert not self.headers_sent, 'headers have been sent already' self.headers_sent = True if not self.chunked and self.autochunked(): self.enable_chunking() if self.chunked: self.headers[hdrs.TRANSFER_ENCODING] = 'chunked' self._add_default_headers() # status + headers headers = self.status_line + ''.join( [k + _sep + v + _end for k, v in self.headers.items()]) headers = headers.encode('utf-8') + b'\r\n' self.buffer_data(headers) def _add_default_headers(self): # set the connection header connection = None if self.upgrade: connection = 'Upgrade' elif not self.closing if self.keepalive is None else self.keepalive: if self._version == HttpVersion10: connection = 'keep-alive' else: if self._version == HttpVersion11: connection = 'close' if connection is not None: self.headers[hdrs.CONNECTION] = connection
def function767(self, arg211): 'Parses RFC 5322 headers from a stream.\n\n Line continuations are supported. Returns list of header name\n and value pairs. Header name is in upper case.\n ' var3066 = CIMultiDict() var2086 = [] var4022 = 1 var2842 = arg211[1] var1431 = len(arg211) while line: var1685 = len(var2842) try: (var3564, var724) = var2842.split(b':', 1) except ValueError: raise InvalidHeader(var2842) from None var3564 = var3564.strip(b' \t') if var4019.search(var3564): raise InvalidHeader(var3564) var4022 += 1 var2842 = arg211[var4022] var3115 = (line and (var2842[0] in (32, 9))) if var3115: var724 = [var724] while continuation: var1685 += len(var2842) if (var1685 > self.attribute1698): raise LineTooLong( 'request header field {}'.format( var3564.decode('utf8', 'xmlcharrefreplace')), self.attribute1698) var724.append(var2842) var4022 += 1 if (var4022 < var1431): var2842 = arg211[var4022] if var2842: var3115 = (var2842[0] in (32, 9)) else: var2842 = b'' break var724 = b''.join(var724) elif (var1685 > self.attribute1698): raise LineTooLong( 'request header field {}'.format( var3564.decode('utf8', 'xmlcharrefreplace')), self.attribute1698) var724 = var724.strip() var4704 = istr(var3564.decode('utf-8', 'surrogateescape')) var3249 = var724.decode('utf-8', 'surrogateescape') var3066.add(var4704, var3249) var2086.append((var3564, var724)) var4441 = None var1752 = None var4099 = False var1171 = False var2086 = tuple(var2086) var2967 = var3066.get(hdrs.CONNECTION) if var2967: var502 = var2967.lower() if (var502 == 'close'): var4441 = True elif (var502 == 'keep-alive'): var4441 = False elif (var502 == 'upgrade'): var4099 = True var2201 = var3066.get(hdrs.CONTENT_ENCODING) if var2201: var2201 = var2201.lower() if (var2201 in ('gzip', 'deflate')): var1752 = var2201 var768 = var3066.get(hdrs.TRANSFER_ENCODING) if (te and ('chunked' in var768.lower())): var1171 = True return (var3066, var2086, var4441, var1752, var4099, var1171)
class StreamResponse(BaseClass, HeadersMixin): _length_check = True def __init__(self, *, status: int = 200, reason: Optional[str] = None, headers: Optional[LooseHeaders] = None) -> None: self._body = None self._keep_alive = None # type: Optional[bool] self._chunked = False self._compression = False self._compression_force = None # type: Optional[ContentCoding] self._cookies = SimpleCookie() self._req = None # type: Optional[Request] self._payload_writer = None # type: Optional[AbstractStreamWriter] self._eof_sent = False self._body_length = 0 self._state = {} # type: Dict[str, Any] if headers is not None: self._headers = CIMultiDict(headers) # type: CIMultiDict[str] else: self._headers = CIMultiDict() # type: CIMultiDict[str] self.set_status(status, reason) @property def prepared(self) -> bool: return self._payload_writer is not None @property def task(self) -> 'asyncio.Task[None]': return getattr(self._req, 'task', None) @property def status(self) -> int: return self._status @property def chunked(self) -> bool: return self._chunked @property def compression(self) -> bool: return self._compression @property def reason(self) -> str: return self._reason def set_status( self, status: int, reason: Optional[str] = None, _RESPONSES: Mapping[int, Tuple[str, str]] = RESPONSES) -> None: assert not self.prepared, \ 'Cannot change the response status code after ' \ 'the headers have been sent' self._status = int(status) if reason is None: try: reason = _RESPONSES[self._status][0] except Exception: reason = '' self._reason = reason @property def keep_alive(self) -> Optional[bool]: return self._keep_alive def force_close(self) -> None: self._keep_alive = False @property def body_length(self) -> int: return self._body_length @property def output_length(self) -> int: warnings.warn('output_length is deprecated', DeprecationWarning) assert self._payload_writer return self._payload_writer.buffer_size def enable_chunked_encoding(self, chunk_size: Optional[int] = None) -> None: """Enables automatic chunked transfer encoding.""" self._chunked = True if hdrs.CONTENT_LENGTH in self._headers: raise RuntimeError("You can't enable chunked encoding when " "a content length is set") if chunk_size is not None: warnings.warn('Chunk size is deprecated #1615', DeprecationWarning) def enable_compression(self, force: Optional[Union[bool, ContentCoding]] = None ) -> None: """Enables response compression encoding.""" # Backwards compatibility for when force was a bool <0.17. if type(force) == bool: force = ContentCoding.deflate if force else ContentCoding.identity warnings.warn("Using boolean for force is deprecated #3318", DeprecationWarning) elif force is not None: assert isinstance(force, ContentCoding), ("force should one of " "None, bool or " "ContentEncoding") self._compression = True self._compression_force = force @property def headers(self) -> 'CIMultiDict[str]': return self._headers @property def cookies(self) -> SimpleCookie: return self._cookies def set_cookie(self, name: str, value: str, *, expires: Optional[str] = None, domain: Optional[str] = None, max_age: Optional[Union[int, str]] = None, path: str = '/', secure: Optional[str] = None, httponly: Optional[str] = None, version: Optional[str] = None) -> None: """Set or update response cookie. Sets new cookie or updates existent with new value. Also updates only those params which are not None. """ old = self._cookies.get(name) if old is not None and old.coded_value == '': # deleted cookie self._cookies.pop(name, None) self._cookies[name] = value c = self._cookies[name] if expires is not None: c['expires'] = expires elif c.get('expires') == 'Thu, 01 Jan 1970 00:00:00 GMT': del c['expires'] if domain is not None: c['domain'] = domain if max_age is not None: c['max-age'] = str(max_age) elif 'max-age' in c: del c['max-age'] c['path'] = path if secure is not None: c['secure'] = secure if httponly is not None: c['httponly'] = httponly if version is not None: c['version'] = version def del_cookie(self, name: str, *, domain: Optional[str] = None, path: str = '/') -> None: """Delete cookie. Creates new empty expired cookie. """ # TODO: do we need domain/path here? self._cookies.pop(name, None) self.set_cookie(name, '', max_age=0, expires="Thu, 01 Jan 1970 00:00:00 GMT", domain=domain, path=path) @property def content_length(self) -> Optional[int]: # Just a placeholder for adding setter return super().content_length @content_length.setter def content_length(self, value: Optional[int]) -> None: if value is not None: value = int(value) if self._chunked: raise RuntimeError("You can't set content length when " "chunked encoding is enable") self._headers[hdrs.CONTENT_LENGTH] = str(value) else: self._headers.pop(hdrs.CONTENT_LENGTH, None) @property def content_type(self) -> str: # Just a placeholder for adding setter return super().content_type @content_type.setter def content_type(self, value: str) -> None: self.content_type # read header values if needed self._content_type = str(value) self._generate_content_type_header() @property def charset(self) -> Optional[str]: # Just a placeholder for adding setter return super().charset @charset.setter def charset(self, value: Optional[str]) -> None: ctype = self.content_type # read header values if needed if ctype == 'application/octet-stream': raise RuntimeError("Setting charset for application/octet-stream " "doesn't make sense, setup content_type first") assert self._content_dict is not None if value is None: self._content_dict.pop('charset', None) else: self._content_dict['charset'] = str(value).lower() self._generate_content_type_header() @property def last_modified(self) -> Optional[datetime.datetime]: """The value of Last-Modified HTTP header, or None. This header is represented as a `datetime` object. """ httpdate = self._headers.get(hdrs.LAST_MODIFIED) if httpdate is not None: timetuple = parsedate(httpdate) if timetuple is not None: return datetime.datetime(*timetuple[:6], tzinfo=datetime.timezone.utc) return None @last_modified.setter def last_modified( self, value: Optional[Union[int, float, datetime.datetime, str]]) -> None: if value is None: self._headers.pop(hdrs.LAST_MODIFIED, None) elif isinstance(value, (int, float)): self._headers[hdrs.LAST_MODIFIED] = time.strftime( "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(math.ceil(value))) elif isinstance(value, datetime.datetime): self._headers[hdrs.LAST_MODIFIED] = time.strftime( "%a, %d %b %Y %H:%M:%S GMT", value.utctimetuple()) elif isinstance(value, str): self._headers[hdrs.LAST_MODIFIED] = value def _generate_content_type_header(self, CONTENT_TYPE: istr = hdrs.CONTENT_TYPE ) -> None: assert self._content_dict is not None assert self._content_type is not None params = '; '.join("{}={}".format(k, v) for k, v in self._content_dict.items()) if params: ctype = self._content_type + '; ' + params else: ctype = self._content_type self._headers[CONTENT_TYPE] = ctype def _do_start_compression(self, coding: ContentCoding) -> None: if coding != ContentCoding.identity: assert self._payload_writer is not None self._headers[hdrs.CONTENT_ENCODING] = coding.value self._payload_writer.enable_compression(coding.value) # Compressed payload may have different content length, # remove the header self._headers.popall(hdrs.CONTENT_LENGTH, None) def _start_compression(self, request: 'Request') -> None: if self._compression_force: self._do_start_compression(self._compression_force) else: accept_encoding = request.headers.get(hdrs.ACCEPT_ENCODING, '').lower() for coding in ContentCoding: if coding.value in accept_encoding: self._do_start_compression(coding) return async def prepare(self, request: 'Request') -> Optional[AbstractStreamWriter]: if self._eof_sent: return None if self._payload_writer is not None: return self._payload_writer await request._prepare_hook(self) return await self._start(request) async def _start(self, request: 'Request') -> AbstractStreamWriter: self._req = request keep_alive = self._keep_alive if keep_alive is None: keep_alive = request.keep_alive self._keep_alive = keep_alive version = request.version writer = self._payload_writer = request._payload_writer headers = self._headers for cookie in self._cookies.values(): value = cookie.output(header='')[1:] headers.add(hdrs.SET_COOKIE, value) if self._compression: self._start_compression(request) if self._chunked: if version != HttpVersion11: raise RuntimeError("Using chunked encoding is forbidden " "for HTTP/{0.major}.{0.minor}".format( request.version)) writer.enable_chunking() headers[hdrs.TRANSFER_ENCODING] = 'chunked' if hdrs.CONTENT_LENGTH in headers: del headers[hdrs.CONTENT_LENGTH] elif self._length_check: writer.length = self.content_length if writer.length is None: if version >= HttpVersion11: writer.enable_chunking() headers[hdrs.TRANSFER_ENCODING] = 'chunked' if hdrs.CONTENT_LENGTH in headers: del headers[hdrs.CONTENT_LENGTH] else: keep_alive = False headers.setdefault(hdrs.CONTENT_TYPE, 'application/octet-stream') headers.setdefault(hdrs.DATE, rfc822_formatted_time()) headers.setdefault(hdrs.SERVER, SERVER_SOFTWARE) # connection header if hdrs.CONNECTION not in headers: if keep_alive: if version == HttpVersion10: headers[hdrs.CONNECTION] = 'keep-alive' else: if version == HttpVersion11: headers[hdrs.CONNECTION] = 'close' # status line status_line = 'HTTP/{}.{} {} {}'.format(version[0], version[1], self._status, self._reason) await writer.write_headers(status_line, headers) return writer async def write(self, data: bytes) -> None: assert isinstance(data, (bytes, bytearray, memoryview)), \ "data argument must be byte-ish (%r)" % type(data) if self._eof_sent: raise RuntimeError("Cannot call write() after write_eof()") if self._payload_writer is None: raise RuntimeError("Cannot call write() before prepare()") await self._payload_writer.write(data) async def drain(self) -> None: assert not self._eof_sent, "EOF has already been sent" assert self._payload_writer is not None, \ "Response has not been started" warnings.warn("drain method is deprecated, use await resp.write()", DeprecationWarning, stacklevel=2) await self._payload_writer.drain() async def write_eof(self, data: bytes = b'') -> None: assert isinstance(data, (bytes, bytearray, memoryview)), \ "data argument must be byte-ish (%r)" % type(data) if self._eof_sent: return assert self._payload_writer is not None, \ "Response has not been started" await self._payload_writer.write_eof(data) self._eof_sent = True self._req = None self._body_length = self._payload_writer.output_size self._payload_writer = None def __repr__(self) -> str: if self._eof_sent: info = "eof" elif self.prepared: assert self._req is not None info = "{} {} ".format(self._req.method, self._req.path) else: info = "not prepared" return "<{} {} {}>".format(self.__class__.__name__, self.reason, info) def __getitem__(self, key: str) -> Any: return self._state[key] def __setitem__(self, key: str, value: Any) -> None: self._state[key] = value def __delitem__(self, key: str) -> None: del self._state[key] def __len__(self) -> int: return len(self._state) def __iter__(self) -> Iterator[str]: return iter(self._state) def __hash__(self) -> int: return hash(id(self)) def __eq__(self, other: object) -> bool: return self is other
class HttpMessage(ABC): """HttpMessage allows to write headers and payload to a stream. For example, lets say we want to read file then compress it with deflate compression and then send it with chunked transfer encoding, code may look like this: >>> response = aiohttp.Response(transport, 200) We have to use deflate compression first: >>> response.add_compression_filter('deflate') Then we want to split output stream into chunks of 1024 bytes size: >>> response.add_chunking_filter(1024) We can add headers to response with add_headers() method. add_headers() does not send data to transport, send_headers() sends request/response line and then sends headers: >>> response.add_headers( ... ('Content-Disposition', 'attachment; filename="..."')) >>> response.send_headers() Now we can use chunked writer to write stream to a network stream. First call to write() method sends response status line and headers, add_header() and add_headers() method unavailable at this stage: >>> with open('...', 'rb') as f: ... chunk = fp.read(8192) ... while chunk: ... response.write(chunk) ... chunk = fp.read(8192) >>> response.write_eof() """ writer = None # 'filter' is being used for altering write() behaviour, # add_chunking_filter adds deflate/gzip compression and # add_compression_filter splits incoming data into a chunks. filter = None HOP_HEADERS = None # Must be set by subclass. SERVER_SOFTWARE = 'Python/{0[0]}.{0[1]} aiohttp/{1}'.format( sys.version_info, aiohttp.__version__) upgrade = False # Connection: UPGRADE websocket = False # Upgrade: WEBSOCKET has_chunked_hdr = False # Transfer-encoding: chunked # subclass can enable auto sending headers with write() call, # this is useful for wsgi's start_response implementation. _send_headers = False def __init__(self, transport, version, close): self.transport = transport self._version = version self.closing = close self.keepalive = None self.chunked = False self.length = None self.headers = CIMultiDict() self.headers_sent = False self.output_length = 0 self.headers_length = 0 self._output_size = 0 self._cache = {} @property @abstractmethod def status_line(self): return b'' @abstractmethod def autochunked(self): return False @property def version(self): return self._version @property def body_length(self): return self.output_length - self.headers_length def force_close(self): self.closing = True self.keepalive = False def enable_chunked_encoding(self): self.chunked = True def keep_alive(self): if self.keepalive is None: if self.version < HttpVersion10: # keep alive not supported at all return False if self.version == HttpVersion10: if self.headers.get(hdrs.CONNECTION) == 'keep-alive': return True else: # no headers means we close for Http 1.0 return False else: return not self.closing else: return self.keepalive def is_headers_sent(self): return self.headers_sent def add_header(self, name, value): """Analyze headers. Calculate content length, removes hop headers, etc.""" assert not self.headers_sent, 'headers have been sent already' assert isinstance(name, str), \ 'Header name should be a string, got {!r}'.format(name) assert set(name).issubset(ASCIISET), \ 'Header name should contain ASCII chars, got {!r}'.format(name) assert isinstance(value, str), \ 'Header {!r} should have string value, got {!r}'.format( name, value) name = istr(name) value = value.strip() if name == hdrs.CONTENT_LENGTH: self.length = int(value) if name == hdrs.TRANSFER_ENCODING: self.has_chunked_hdr = value.lower().strip() == 'chunked' if name == hdrs.CONNECTION: val = value.lower() # handle websocket if 'upgrade' in val: self.upgrade = True # connection keep-alive elif 'close' in val: self.keepalive = False elif 'keep-alive' in val: self.keepalive = True elif name == hdrs.UPGRADE: if 'websocket' in value.lower(): self.websocket = True self.headers[name] = value elif name not in self.HOP_HEADERS: # ignore hop-by-hop headers self.headers.add(name, value) def add_headers(self, *headers): """Adds headers to a HTTP message.""" for name, value in headers: self.add_header(name, value) def send_headers(self, _sep=': ', _end='\r\n'): """Writes headers to a stream. Constructs payload writer.""" # Chunked response is only for HTTP/1.1 clients or newer # and there is no Content-Length header is set. # Do not use chunked responses when the response is guaranteed to # not have a response body (304, 204). assert not self.headers_sent, 'headers have been sent already' self.headers_sent = True if self.chunked or self.autochunked(): self.writer = self._write_chunked_payload() self.headers[hdrs.TRANSFER_ENCODING] = 'chunked' elif self.length is not None: self.writer = self._write_length_payload(self.length) else: self.writer = self._write_eof_payload() next(self.writer) self._add_default_headers() # status + headers headers = self.status_line + ''.join( [k + _sep + v + _end for k, v in self.headers.items()]) headers = headers.encode('utf-8') + b'\r\n' self.output_length += len(headers) self.headers_length = len(headers) self.transport.write(headers) def _add_default_headers(self): # set the connection header connection = None if self.upgrade: connection = 'Upgrade' elif not self.closing if self.keepalive is None else self.keepalive: if self.version == HttpVersion10: connection = 'keep-alive' else: if self.version == HttpVersion11: connection = 'close' if connection is not None: self.headers[hdrs.CONNECTION] = connection def write(self, chunk, *, drain=False, EOF_MARKER=EOF_MARKER, EOL_MARKER=EOL_MARKER): """Writes chunk of data to a stream by using different writers. writer uses filter to modify chunk of data. write_eof() indicates end of stream. writer can't be used after write_eof() method being called. write() return drain future. """ assert (isinstance(chunk, (bytes, bytearray)) or chunk is EOF_MARKER), chunk size = self.output_length if self._send_headers and not self.headers_sent: self.send_headers() if self.filter: chunk = self.filter.send(chunk) while chunk not in (EOF_MARKER, EOL_MARKER): if chunk: self.writer.send(chunk) chunk = next(self.filter) else: if chunk is not EOF_MARKER: self.writer.send(chunk) self._output_size += self.output_length - size if self._output_size > 64 * 1024: if drain: self._output_size = 0 return self.transport.drain() return () def write_eof(self): self.write(EOF_MARKER) try: self.writer.throw(aiohttp.EofStream()) except StopIteration: pass return self.transport.drain() def _write_chunked_payload(self): """Write data in chunked transfer encoding.""" while True: try: chunk = yield except aiohttp.EofStream: self.transport.write(b'0\r\n\r\n') self.output_length += 5 break chunk = bytes(chunk) chunk_len = '{:x}\r\n'.format(len(chunk)).encode('ascii') self.transport.write(chunk_len + chunk + b'\r\n') self.output_length += len(chunk_len) + len(chunk) + 2 def _write_length_payload(self, length): """Write specified number of bytes to a stream.""" while True: try: chunk = yield except aiohttp.EofStream: break if length: l = len(chunk) if length >= l: self.transport.write(chunk) self.output_length += l length = length - l else: self.transport.write(chunk[:length]) self.output_length += length length = 0 def _write_eof_payload(self): while True: try: chunk = yield except aiohttp.EofStream: break self.transport.write(chunk) self.output_length += len(chunk) @wrap_payload_filter def add_chunking_filter(self, chunk_size=16 * 1024, *, EOF_MARKER=EOF_MARKER, EOL_MARKER=EOL_MARKER): """Split incoming stream into chunks.""" buf = bytearray() chunk = yield while True: if chunk is EOF_MARKER: if buf: yield buf yield EOF_MARKER else: buf.extend(chunk) while len(buf) >= chunk_size: chunk = bytes(buf[:chunk_size]) del buf[:chunk_size] yield chunk chunk = yield EOL_MARKER @wrap_payload_filter def add_compression_filter(self, encoding='deflate', *, EOF_MARKER=EOF_MARKER, EOL_MARKER=EOL_MARKER): """Compress incoming stream with deflate or gzip encoding.""" zlib_mode = (16 + zlib.MAX_WBITS if encoding == 'gzip' else -zlib.MAX_WBITS) zcomp = zlib.compressobj(wbits=zlib_mode) chunk = yield while True: if chunk is EOF_MARKER: yield zcomp.flush() chunk = yield EOF_MARKER else: yield zcomp.compress(chunk) chunk = yield EOL_MARKER
class WsgiResponse: """A WSGI response. Instances are callable using the standard WSGI call and, importantly, iterable:: response = WsgiResponse(200) A :class:`WsgiResponse` is an iterable over bytes to send back to the requesting client. .. attribute:: status_code Integer indicating the HTTP status, (i.e. 200) .. attribute:: response String indicating the HTTP status (i.e. 'OK') .. attribute:: status String indicating the HTTP status code and response (i.e. '200 OK') .. attribute:: content_type The content type of this response. Can be ``None``. .. attribute:: headers The :class:`.Headers` container for this response. .. attribute:: cookies A python :class:`SimpleCookie` container of cookies included in the request as well as cookies set during the response. """ _iterated = False __wsgi_started__ = False def __init__(self, status_code=200, content=None, response_headers=None, content_type=None, encoding=None, can_store_cookies=True): self.status_code = status_code self.encoding = encoding self.headers = CIMultiDict(response_headers or ()) self.content = content self._cookies = None self._can_store_cookies = can_store_cookies if content_type is not None: self.content_type = content_type @property def started(self): return self.__wsgi_started__ @property def iterated(self): return self._iterated @property def cookies(self): if self._cookies is None: self._cookies = SimpleCookie() return self._cookies @property def content(self): return self._content @content.setter def content(self, content): self.set_content(content) def set_content(self, content): if self._iterated: raise RuntimeError('Cannot set content. Already iterated') if content is None: self._content = () elif isinstance(content, str): if not self.encoding: # use utf-8 if not set self.encoding = 'utf-8' self._content = content.encode(self.encoding), elif isinstance(content, bytes): self._content = content, else: self._content = content def _get_content_type(self): return self.headers.get('content-type') def _set_content_type(self, typ): if typ: self.headers['content-type'] = typ else: self.headers.pop('content-type', None) content_type = property(_get_content_type, _set_content_type) @property def response(self): return responses.get(self.status_code) @property def status(self): return '%s %s' % (self.status_code, responses.get(self.status_code)) def __str__(self): return self.status def __repr__(self): return '%s(%s)' % (self.__class__.__name__, self) def is_streamed(self): """Check if the response is streamed. A streamed response is an iterable with no length information. In this case streamed means that there is no information about the number of iterations. This is usually `True` if a generator is passed to the response object. """ try: len(self._content) except TypeError: return True return False def length(self): try: len(self._content) except TypeError: return return reduce(count_len, self._content, 0) def can_set_cookies(self): return self.status_code < 400 and self._can_store_cookies def start(self, environ, start_response, exc_info=None): self.__wsgi_started__ = True headers = self._get_headers(environ) return start_response(self.status, headers, exc_info) def __iter__(self): if self._iterated: raise RuntimeError('WsgiResponse can be iterated once only') self.__wsgi_started__ = True self._iterated = True iterable = iter(self._content) self._content = None return iterable def close(self): """Close this response, required by WSGI """ if hasattr(self._content, 'close'): self._content.close() def set_cookie(self, key, **kwargs): """ Sets a cookie. ``expires`` can be a string in the correct format or a ``datetime.datetime`` object in UTC. If ``expires`` is a datetime object then ``max_age`` will be calculated. """ set_cookie(self.cookies, key, **kwargs) def delete_cookie(self, key, path='/', domain=None): set_cookie(self.cookies, key, max_age=0, path=path, domain=domain, expires='Thu, 01-Jan-1970 00:00:00 GMT') def has_header(self, header): return header in self.headers __contains__ = has_header def __setitem__(self, header, value): self.headers[header] = value def __getitem__(self, header): return self.headers[header] def _get_headers(self, environ): """The list of headers for this response """ headers = self.headers method = environ['REQUEST_METHOD'] if has_empty_content(self.status_code, method) and method != HEAD: headers.pop('content-type', None) headers.pop('content-length', None) self._content = () else: if not self.is_streamed(): cl = reduce(count_len, self._content, 0) headers['content-length'] = str(cl) ct = headers.get('content-type') # content type encoding available if self.encoding: ct = ct or 'text/plain' if ';' not in ct: ct = '%s; charset=%s' % (ct, self.encoding) if ct: headers['content-type'] = ct if method == HEAD: self._content = () # Cookies if (self.status_code < 400 and self._can_store_cookies and self._cookies): for c in self.cookies.values(): headers.add('set-cookie', c.OutputString()) return headers.items()
class StreamingHTTPResponse(BaseHTTPResponse): __slots__ = ('transport', 'streaming_fn', 'status', 'content_type', 'headers', '_cookies') def __init__(self, streaming_fn, status=200, headers=None, content_type='text/plain'): self.content_type = content_type self.streaming_fn = streaming_fn self.status = status self.headers = CIMultiDict(headers or {}) self._cookies = None def write(self, data): """Writes a chunk of data to the streaming response. :param data: bytes-ish data to be written. """ if type(data) != bytes: data = self._encode_body(data) self.transport.write(b"%x\r\n%b\r\n" % (len(data), data)) async def stream(self, version="1.1", keep_alive=False, keep_alive_timeout=None): """Streams headers, runs the `streaming_fn` callback that writes content to the response body, then finalizes the response body. """ headers = self.get_headers(version, keep_alive=keep_alive, keep_alive_timeout=keep_alive_timeout) self.transport.write(headers) await self.streaming_fn(self) self.transport.write(b'0\r\n\r\n') def get_headers(self, version="1.1", keep_alive=False, keep_alive_timeout=None): # This is all returned in a kind-of funky way # We tried to make this as fast as possible in pure python timeout_header = b'' if keep_alive and keep_alive_timeout is not None: timeout_header = b'Keep-Alive: %d\r\n' % keep_alive_timeout self.headers['Transfer-Encoding'] = 'chunked' self.headers.pop('Content-Length', None) self.headers['Content-Type'] = self.headers.get( 'Content-Type', self.content_type) headers = self._parse_headers() if self.status is 200: status = b'OK' else: status = http.STATUS_CODES.get(self.status) return (b'HTTP/%b %d %b\r\n' b'%b' b'%b\r\n') % (version.encode(), self.status, status, timeout_header, headers)
def make_mocked_request(method, path, headers=None, *, match_info=sentinel, version=HttpVersion(1, 1), closing=False, app=None, writer=sentinel, protocol=sentinel, transport=sentinel, payload=sentinel, sslcontext=None, client_max_size=1024**2, loop=...): """Creates mocked web.Request testing purposes. Useful in unit tests, when spinning full web server is overkill or specific conditions and errors are hard to trigger. """ task = mock.Mock() if loop is ...: loop = mock.Mock() loop.create_future.return_value = () if version < HttpVersion(1, 1): closing = True if headers: headers = CIMultiDict(headers) raw_hdrs = tuple( (k.encode('utf-8'), v.encode('utf-8')) for k, v in headers.items()) else: headers = CIMultiDict() raw_hdrs = () chunked = 'chunked' in headers.get(hdrs.TRANSFER_ENCODING, '').lower() message = RawRequestMessage(method, path, version, headers, raw_hdrs, closing, False, False, chunked, URL(path)) if app is None: app = _create_app_mock() if protocol is sentinel: protocol = mock.Mock() if transport is sentinel: transport = _create_transport(sslcontext) if writer is sentinel: writer = mock.Mock() writer.write_headers = make_mocked_coro(None) writer.write = make_mocked_coro(None) writer.write_eof = make_mocked_coro(None) writer.drain = make_mocked_coro(None) writer.transport = transport protocol.transport = transport protocol.writer = writer if payload is sentinel: payload = mock.Mock() req = Request(message, payload, protocol, writer, task, loop, client_max_size=client_max_size) match_info = UrlMappingMatchInfo( {} if match_info is sentinel else match_info, mock.Mock()) match_info.add_app(app) req._match_info = match_info return req
class StreamResponse(BaseClass, HeadersMixin, CookieMixin): __slots__ = ( "_length_check", "_body", "_keep_alive", "_chunked", "_compression", "_compression_force", "_req", "_payload_writer", "_eof_sent", "_body_length", "_state", "_headers", "_status", "_reason", "__weakref__", ) def __init__( self, *, status: int = 200, reason: Optional[str] = None, headers: Optional[LooseHeaders] = None, ) -> None: super().__init__() self._length_check = True self._body = None self._keep_alive = None # type: Optional[bool] self._chunked = False self._compression = False self._compression_force = None # type: Optional[ContentCoding] self._req = None # type: Optional[BaseRequest] self._payload_writer = None # type: Optional[AbstractStreamWriter] self._eof_sent = False self._body_length = 0 self._state = {} # type: Dict[str, Any] if headers is not None: self._headers = CIMultiDict(headers) # type: CIMultiDict[str] else: self._headers = CIMultiDict() self.set_status(status, reason) @property def prepared(self) -> bool: return self._payload_writer is not None @property def task(self) -> "asyncio.Task[None]": return getattr(self._req, "task", None) @property def status(self) -> int: return self._status @property def chunked(self) -> bool: return self._chunked @property def compression(self) -> bool: return self._compression @property def reason(self) -> str: return self._reason def set_status( self, status: int, reason: Optional[str] = None, _RESPONSES: Mapping[int, Tuple[str, str]] = RESPONSES, ) -> None: assert not self.prepared, ( "Cannot change the response status code after " "the headers have been sent") self._status = int(status) if reason is None: try: reason = _RESPONSES[self._status][0] except Exception: reason = "" self._reason = reason @property def keep_alive(self) -> Optional[bool]: return self._keep_alive def force_close(self) -> None: self._keep_alive = False @property def body_length(self) -> int: return self._body_length def enable_chunked_encoding(self) -> None: """Enables automatic chunked transfer encoding.""" self._chunked = True if hdrs.CONTENT_LENGTH in self._headers: raise RuntimeError("You can't enable chunked encoding when " "a content length is set") def enable_compression(self, force: Optional[ContentCoding] = None) -> None: """Enables response compression encoding.""" # Backwards compatibility for when force was a bool <0.17. self._compression = True self._compression_force = force @property def headers(self) -> "CIMultiDict[str]": return self._headers @property def content_length(self) -> Optional[int]: # Just a placeholder for adding setter return super().content_length @content_length.setter def content_length(self, value: Optional[int]) -> None: if value is not None: value = int(value) if self._chunked: raise RuntimeError("You can't set content length when " "chunked encoding is enable") self._headers[hdrs.CONTENT_LENGTH] = str(value) else: self._headers.pop(hdrs.CONTENT_LENGTH, None) @property def content_type(self) -> str: # Just a placeholder for adding setter return super().content_type @content_type.setter def content_type(self, value: str) -> None: self.content_type # read header values if needed self._content_type = str(value) self._generate_content_type_header() @property def charset(self) -> Optional[str]: # Just a placeholder for adding setter return super().charset @charset.setter def charset(self, value: Optional[str]) -> None: ctype = self.content_type # read header values if needed if ctype == "application/octet-stream": raise RuntimeError("Setting charset for application/octet-stream " "doesn't make sense, setup content_type first") assert self._content_dict is not None if value is None: self._content_dict.pop("charset", None) else: self._content_dict["charset"] = str(value).lower() self._generate_content_type_header() @property def last_modified(self) -> Optional[datetime.datetime]: """The value of Last-Modified HTTP header, or None. This header is represented as a `datetime` object. """ httpdate = self._headers.get(hdrs.LAST_MODIFIED) if httpdate is not None: timetuple = parsedate(httpdate) if timetuple is not None: return datetime.datetime(*timetuple[:6], tzinfo=datetime.timezone.utc) return None @last_modified.setter def last_modified( self, value: Optional[Union[int, float, datetime.datetime, str]]) -> None: if value is None: self._headers.pop(hdrs.LAST_MODIFIED, None) elif isinstance(value, (int, float)): self._headers[hdrs.LAST_MODIFIED] = time.strftime( "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(math.ceil(value))) elif isinstance(value, datetime.datetime): self._headers[hdrs.LAST_MODIFIED] = time.strftime( "%a, %d %b %Y %H:%M:%S GMT", value.utctimetuple()) elif isinstance(value, str): self._headers[hdrs.LAST_MODIFIED] = value def _generate_content_type_header(self, CONTENT_TYPE: istr = hdrs.CONTENT_TYPE ) -> None: assert self._content_dict is not None assert self._content_type is not None params = "; ".join(f"{k}={v}" for k, v in self._content_dict.items()) if params: ctype = self._content_type + "; " + params else: ctype = self._content_type self._headers[CONTENT_TYPE] = ctype async def _do_start_compression(self, coding: ContentCoding) -> None: if coding != ContentCoding.identity: assert self._payload_writer is not None self._headers[hdrs.CONTENT_ENCODING] = coding.value self._payload_writer.enable_compression(coding.value) # Compressed payload may have different content length, # remove the header self._headers.popall(hdrs.CONTENT_LENGTH, None) async def _start_compression(self, request: "BaseRequest") -> None: if self._compression_force: await self._do_start_compression(self._compression_force) else: accept_encoding = request.headers.get(hdrs.ACCEPT_ENCODING, "").lower() for coding in ContentCoding: if coding.value in accept_encoding: await self._do_start_compression(coding) return async def prepare( self, request: "BaseRequest") -> Optional[AbstractStreamWriter]: if self._eof_sent: return None if self._payload_writer is not None: return self._payload_writer return await self._start(request) async def _start(self, request: "BaseRequest") -> AbstractStreamWriter: self._req = request writer = self._payload_writer = request._payload_writer await self._prepare_headers() await request._prepare_hook(self) await self._write_headers() return writer async def _prepare_headers(self) -> None: request = self._req assert request is not None writer = self._payload_writer assert writer is not None keep_alive = self._keep_alive if keep_alive is None: keep_alive = request.keep_alive self._keep_alive = keep_alive version = request.version headers = self._headers populate_with_cookies(headers, self.cookies) if self._compression: await self._start_compression(request) if self._chunked: if version != HttpVersion11: raise RuntimeError("Using chunked encoding is forbidden " "for HTTP/{0.major}.{0.minor}".format( request.version)) writer.enable_chunking() headers[hdrs.TRANSFER_ENCODING] = "chunked" if hdrs.CONTENT_LENGTH in headers: del headers[hdrs.CONTENT_LENGTH] elif self._length_check: writer.length = self.content_length if writer.length is None: if version >= HttpVersion11 and self.status != 204: writer.enable_chunking() headers[hdrs.TRANSFER_ENCODING] = "chunked" if hdrs.CONTENT_LENGTH in headers: del headers[hdrs.CONTENT_LENGTH] else: keep_alive = False # HTTP 1.1: https://tools.ietf.org/html/rfc7230#section-3.3.2 # HTTP 1.0: https://tools.ietf.org/html/rfc1945#section-10.4 elif version >= HttpVersion11 and self.status in (100, 101, 102, 103, 204): del headers[hdrs.CONTENT_LENGTH] if self.status != 204: headers.setdefault(hdrs.CONTENT_TYPE, "application/octet-stream") headers.setdefault(hdrs.DATE, rfc822_formatted_time()) headers.setdefault(hdrs.SERVER, SERVER_SOFTWARE) # connection header if hdrs.CONNECTION not in headers: if keep_alive: if version == HttpVersion10: headers[hdrs.CONNECTION] = "keep-alive" else: if version == HttpVersion11: headers[hdrs.CONNECTION] = "close" async def _write_headers(self) -> None: request = self._req assert request is not None writer = self._payload_writer assert writer is not None # status line version = request.version status_line = "HTTP/{}.{} {} {}".format(version[0], version[1], self._status, self._reason) await writer.write_headers(status_line, self._headers) async def write(self, data: bytes) -> None: assert isinstance( data, (bytes, bytearray, memoryview)), "data argument must be byte-ish (%r)" % type(data) if self._eof_sent: raise RuntimeError("Cannot call write() after write_eof()") if self._payload_writer is None: raise RuntimeError("Cannot call write() before prepare()") await self._payload_writer.write(data) async def drain(self) -> None: assert not self._eof_sent, "EOF has already been sent" assert self._payload_writer is not None, "Response has not been started" warnings.warn( "drain method is deprecated, use await resp.write()", DeprecationWarning, stacklevel=2, ) await self._payload_writer.drain() async def write_eof(self, data: bytes = b"") -> None: assert isinstance( data, (bytes, bytearray, memoryview)), "data argument must be byte-ish (%r)" % type(data) if self._eof_sent: return assert self._payload_writer is not None, "Response has not been started" await self._payload_writer.write_eof(data) self._eof_sent = True self._req = None self._body_length = self._payload_writer.output_size self._payload_writer = None def __repr__(self) -> str: if self._eof_sent: info = "eof" elif self.prepared: assert self._req is not None info = f"{self._req.method} {self._req.path} " else: info = "not prepared" return f"<{self.__class__.__name__} {self.reason} {info}>" def __getitem__(self, key: str) -> Any: return self._state[key] def __setitem__(self, key: str, value: Any) -> None: self._state[key] = value def __delitem__(self, key: str) -> None: del self._state[key] def __len__(self) -> int: return len(self._state) def __iter__(self) -> Iterator[str]: return iter(self._state) def __hash__(self) -> int: return hash(id(self)) def __eq__(self, other: object) -> bool: return self is other
class ClientRequest: GET_METHODS = {hdrs.METH_GET, hdrs.METH_HEAD, hdrs.METH_OPTIONS} POST_METHODS = {hdrs.METH_PATCH, hdrs.METH_POST, hdrs.METH_PUT} ALL_METHODS = GET_METHODS.union(POST_METHODS).union( {hdrs.METH_DELETE, hdrs.METH_TRACE}) DEFAULT_HEADERS = { hdrs.ACCEPT: '*/*', hdrs.ACCEPT_ENCODING: 'gzip, deflate', } SERVER_SOFTWARE = HttpMessage.SERVER_SOFTWARE body = b'' auth = None response = None response_class = None _writer = None # async task for streaming data _continue = None # waiter future for '100 Continue' response # N.B. # Adding __del__ method with self._writer closing doesn't make sense # because _writer is instance method, thus it keeps a reference to self. # Until writer has finished finalizer will not be called. def __init__(self, method, url, *, params=None, headers=None, skip_auto_headers=frozenset(), data=None, cookies=None, auth=None, encoding='utf-8', version=aiohttp.HttpVersion11, compress=None, chunked=None, expect100=False, loop=None, response_class=None): if loop is None: loop = asyncio.get_event_loop() self.url = url self.method = method.upper() self.encoding = encoding self.chunked = chunked self.compress = compress self.loop = loop self.response_class = response_class or ClientResponse if loop.get_debug(): self._source_traceback = traceback.extract_stack(sys._getframe(1)) self.update_version(version) self.update_host(url) self.update_path(params) self.update_headers(headers) self.update_auto_headers(skip_auto_headers) self.update_cookies(cookies) self.update_content_encoding() self.update_auth(auth) self.update_body_from_data(data, skip_auto_headers) self.update_transfer_encoding() self.update_expect_continue(expect100) def update_host(self, url): """Update destination host, port and connection type (ssl).""" url_parsed = urllib.parse.urlsplit(url) # check for network location part netloc = url_parsed.netloc if not netloc: raise ValueError('Host could not be detected.') # get host/port host = url_parsed.hostname if not host: raise ValueError('Host could not be detected.') try: port = url_parsed.port except ValueError: raise ValueError('Port number could not be converted.') from None # check domain idna encoding try: netloc = netloc.encode('idna').decode('utf-8') host = host.encode('idna').decode('utf-8') except UnicodeError: raise ValueError('URL has an invalid label.') # basic auth info username, password = url_parsed.username, url_parsed.password if username: self.auth = helpers.BasicAuth(username, password or '') netloc = netloc.split('@', 1)[1] # Record entire netloc for usage in host header self.netloc = netloc scheme = url_parsed.scheme self.ssl = scheme in ('https', 'wss') # set port number if it isn't already set if not port: if self.ssl: port = HTTPS_PORT else: port = HTTP_PORT self.host, self.port, self.scheme = host, port, scheme def update_version(self, version): """Convert request version to two elements tuple. parser HTTP version '1.1' => (1, 1) """ if isinstance(version, str): v = [l.strip() for l in version.split('.', 1)] try: version = int(v[0]), int(v[1]) except ValueError: raise ValueError( 'Can not parse http version number: {}'.format( version)) from None self.version = version def update_path(self, params): """Build path.""" # extract path scheme, netloc, path, query, fragment = urllib.parse.urlsplit(self.url) if not path: path = '/' if isinstance(params, collections.Mapping): params = list(params.items()) if params: if not isinstance(params, str): params = urllib.parse.urlencode(params) if query: query = '%s&%s' % (query, params) else: query = params self.path = urllib.parse.urlunsplit( ('', '', helpers.requote_uri(path), query, fragment)) self.url = urllib.parse.urlunsplit((scheme, netloc, self.path, '', '')) def update_headers(self, headers): """Update request headers.""" self.headers = CIMultiDict() if headers: if isinstance(headers, dict): headers = headers.items() elif isinstance(headers, (MultiDictProxy, MultiDict)): headers = headers.items() for key, value in headers: self.headers.add(key, value) def update_auto_headers(self, skip_auto_headers): self.skip_auto_headers = skip_auto_headers used_headers = set(self.headers) | skip_auto_headers for hdr, val in self.DEFAULT_HEADERS.items(): if hdr not in used_headers: self.headers.add(hdr, val) # add host if hdrs.HOST not in used_headers: self.headers[hdrs.HOST] = self.netloc if hdrs.USER_AGENT not in used_headers: self.headers[hdrs.USER_AGENT] = self.SERVER_SOFTWARE def update_cookies(self, cookies): """Update request cookies header.""" if not cookies: return c = http.cookies.SimpleCookie() if hdrs.COOKIE in self.headers: c.load(self.headers.get(hdrs.COOKIE, '')) del self.headers[hdrs.COOKIE] if isinstance(cookies, dict): cookies = cookies.items() for name, value in cookies: if isinstance(value, http.cookies.Morsel): c[value.key] = value.value else: c[name] = value self.headers[hdrs.COOKIE] = c.output(header='', sep=';').strip() def update_content_encoding(self): """Set request content encoding.""" enc = self.headers.get(hdrs.CONTENT_ENCODING, '').lower() if enc: if self.compress is not False: self.compress = enc # enable chunked, no need to deal with length self.chunked = True elif self.compress: if not isinstance(self.compress, str): self.compress = 'deflate' self.headers[hdrs.CONTENT_ENCODING] = self.compress self.chunked = True # enable chunked, no need to deal with length def update_auth(self, auth): """Set basic auth.""" if auth is None: auth = self.auth if auth is None: return if not isinstance(auth, helpers.BasicAuth): warnings.warn('BasicAuth() tuple is required instead ', DeprecationWarning) auth = helpers.BasicAuth(*auth) self.headers[hdrs.AUTHORIZATION] = auth.encode() def update_body_from_data(self, data, skip_auto_headers): if not data: return if isinstance(data, str): data = data.encode(self.encoding) if isinstance(data, (bytes, bytearray)): self.body = data if (hdrs.CONTENT_TYPE not in self.headers and hdrs.CONTENT_TYPE not in skip_auto_headers): self.headers[hdrs.CONTENT_TYPE] = 'application/octet-stream' if hdrs.CONTENT_LENGTH not in self.headers and not self.chunked: self.headers[hdrs.CONTENT_LENGTH] = str(len(self.body)) elif isinstance(data, (asyncio.StreamReader, streams.DataQueue)): self.body = data elif asyncio.iscoroutine(data): self.body = data if (hdrs.CONTENT_LENGTH not in self.headers and self.chunked is None): self.chunked = True elif isinstance(data, io.IOBase): assert not isinstance(data, io.StringIO), \ 'attempt to send text data instead of binary' self.body = data if not self.chunked and isinstance(data, io.BytesIO): # Not chunking if content-length can be determined size = len(data.getbuffer()) self.headers[hdrs.CONTENT_LENGTH] = str(size) self.chunked = False elif not self.chunked and isinstance(data, io.BufferedReader): # Not chunking if content-length can be determined try: size = os.fstat(data.fileno()).st_size - data.tell() self.headers[hdrs.CONTENT_LENGTH] = str(size) self.chunked = False except OSError: # data.fileno() is not supported, e.g. # io.BufferedReader(io.BytesIO(b'data')) self.chunked = True else: self.chunked = True if hasattr(data, 'mode'): if data.mode == 'r': raise ValueError('file {!r} should be open in binary mode' ''.format(data)) if (hdrs.CONTENT_TYPE not in self.headers and hdrs.CONTENT_TYPE not in skip_auto_headers and hasattr(data, 'name')): mime = mimetypes.guess_type(data.name)[0] mime = 'application/octet-stream' if mime is None else mime self.headers[hdrs.CONTENT_TYPE] = mime elif isinstance(data, MultipartWriter): self.body = data.serialize() self.headers.update(data.headers) self.chunked = self.chunked or 8192 else: if not isinstance(data, helpers.FormData): data = helpers.FormData(data) self.body = data(self.encoding) if (hdrs.CONTENT_TYPE not in self.headers and hdrs.CONTENT_TYPE not in skip_auto_headers): self.headers[hdrs.CONTENT_TYPE] = data.content_type if data.is_multipart: self.chunked = self.chunked or 8192 else: if (hdrs.CONTENT_LENGTH not in self.headers and not self.chunked): self.headers[hdrs.CONTENT_LENGTH] = str(len(self.body)) def update_transfer_encoding(self): """Analyze transfer-encoding header.""" te = self.headers.get(hdrs.TRANSFER_ENCODING, '').lower() if self.chunked: if hdrs.CONTENT_LENGTH in self.headers: del self.headers[hdrs.CONTENT_LENGTH] if 'chunked' not in te: self.headers[hdrs.TRANSFER_ENCODING] = 'chunked' self.chunked = self.chunked if type(self.chunked) is int else 8192 else: if 'chunked' in te: self.chunked = 8192 else: self.chunked = None if hdrs.CONTENT_LENGTH not in self.headers: self.headers[hdrs.CONTENT_LENGTH] = str(len(self.body)) def update_expect_continue(self, expect=False): if expect: self.headers[hdrs.EXPECT] = '100-continue' elif self.headers.get(hdrs.EXPECT, '').lower() == '100-continue': expect = True if expect: self._continue = asyncio.Future(loop=self.loop) @asyncio.coroutine def write_bytes(self, request, reader): """Support coroutines that yields bytes objects.""" # 100 response if self._continue is not None: yield from self._continue try: if asyncio.iscoroutine(self.body): request.transport.set_tcp_nodelay(True) exc = None value = None stream = self.body while True: try: if exc is not None: result = stream.throw(exc) else: result = stream.send(value) except StopIteration as exc: if isinstance(exc.value, bytes): yield from request.write(exc.value, drain=True) break except: self.response.close() raise if isinstance(result, asyncio.Future): exc = None value = None try: value = yield result except Exception as err: exc = err elif isinstance(result, (bytes, bytearray)): yield from request.write(result, drain=True) value = None else: raise ValueError('Bytes object is expected, got: %s.' % type(result)) elif isinstance(self.body, asyncio.StreamReader): request.transport.set_tcp_nodelay(True) chunk = yield from self.body.read(streams.DEFAULT_LIMIT) while chunk: yield from request.write(chunk, drain=True) chunk = yield from self.body.read(streams.DEFAULT_LIMIT) elif isinstance(self.body, streams.DataQueue): request.transport.set_tcp_nodelay(True) while True: try: chunk = yield from self.body.read() if chunk is EOF_MARKER: break yield from request.write(chunk, drain=True) except streams.EofStream: break elif isinstance(self.body, io.IOBase): chunk = self.body.read(self.chunked) while chunk: request.write(chunk) chunk = self.body.read(self.chunked) request.transport.set_tcp_nodelay(True) else: if isinstance(self.body, (bytes, bytearray)): self.body = (self.body, ) for chunk in self.body: request.write(chunk) request.transport.set_tcp_nodelay(True) except Exception as exc: new_exc = aiohttp.ClientRequestError( 'Can not write request body for %s' % self.url) new_exc.__context__ = exc new_exc.__cause__ = exc reader.set_exception(new_exc) else: assert request.transport.tcp_nodelay try: ret = request.write_eof() # NB: in asyncio 3.4.1+ StreamWriter.drain() is coroutine # see bug #170 if (asyncio.iscoroutine(ret) or isinstance(ret, asyncio.Future)): yield from ret except Exception as exc: new_exc = aiohttp.ClientRequestError( 'Can not write request body for %s' % self.url) new_exc.__context__ = exc new_exc.__cause__ = exc reader.set_exception(new_exc) self._writer = None def send(self, writer, reader): writer.set_tcp_cork(True) request = aiohttp.Request(writer, self.method, self.path, self.version) if self.compress: request.add_compression_filter(self.compress) if self.chunked is not None: request.enable_chunked_encoding() request.add_chunking_filter(self.chunked) # set default content-type if (self.method in self.POST_METHODS and hdrs.CONTENT_TYPE not in self.skip_auto_headers and hdrs.CONTENT_TYPE not in self.headers): self.headers[hdrs.CONTENT_TYPE] = 'application/octet-stream' for k, value in self.headers.items(): request.add_header(k, value) request.send_headers() self._writer = helpers.ensure_future(self.write_bytes(request, reader), loop=self.loop) self.response = self.response_class(self.method, self.url, self.host, writer=self._writer, continue100=self._continue) self.response._post_init(self.loop) return self.response @asyncio.coroutine def close(self): if self._writer is not None: try: yield from self._writer finally: self._writer = None def terminate(self): if self._writer is not None: if hasattr(self.loop, 'is_closed'): if not self.loop.is_closed(): self._writer.cancel() else: self._writer.cancel() self._writer = None
class StreamResponse(HeadersMixin): def __init__(self, *, status=200, reason=None, headers=None): self._body = None self._keep_alive = None self._chunked = False self._chunk_size = None self._compression = False self._compression_force = False self._headers = CIMultiDict() self._cookies = http.cookies.SimpleCookie() self.set_status(status, reason) self._req = None self._resp_impl = None self._eof_sent = False self._tcp_nodelay = True self._tcp_cork = False if headers is not None: self._headers.extend(headers) self._parse_content_type(self._headers.get(hdrs.CONTENT_TYPE)) self._generate_content_type_header() def _copy_cookies(self): for cookie in self._cookies.values(): value = cookie.output(header='')[1:] self.headers.add(hdrs.SET_COOKIE, value) @property def prepared(self): return self._resp_impl is not None @property def started(self): warnings.warn('use Response.prepared instead', DeprecationWarning) return self.prepared @property def status(self): return self._status @property def chunked(self): return self._chunked @property def compression(self): return self._compression @property def reason(self): return self._reason def set_status(self, status, reason=None): self._status = int(status) if reason is None: reason = ResponseImpl.calc_reason(status) self._reason = reason @property def keep_alive(self): return self._keep_alive def force_close(self): self._keep_alive = False def enable_chunked_encoding(self, chunk_size=None): """Enables automatic chunked transfer encoding.""" self._chunked = True self._chunk_size = chunk_size def enable_compression(self, force=None): """Enables response compression encoding.""" # Backwards compatibility for when force was a bool <0.17. if type(force) == bool: force = ContentCoding.deflate if force else ContentCoding.identity elif force is not None: assert isinstance(force, ContentCoding), ("force should one of " "None, bool or " "ContentEncoding") self._compression = True self._compression_force = force @property def headers(self): return self._headers @property def cookies(self): return self._cookies def set_cookie(self, name, value, *, expires=None, domain=None, max_age=None, path='/', secure=None, httponly=None, version=None): """Set or update response cookie. Sets new cookie or updates existent with new value. Also updates only those params which are not None. """ old = self._cookies.get(name) if old is not None and old.coded_value == '': # deleted cookie self._cookies.pop(name, None) self._cookies[name] = value c = self._cookies[name] if expires is not None: c['expires'] = expires elif c.get('expires') == 'Thu, 01 Jan 1970 00:00:00 GMT': del c['expires'] if domain is not None: c['domain'] = domain if max_age is not None: c['max-age'] = max_age elif 'max-age' in c: del c['max-age'] c['path'] = path if secure is not None: c['secure'] = secure if httponly is not None: c['httponly'] = httponly if version is not None: c['version'] = version def del_cookie(self, name, *, domain=None, path='/'): """Delete cookie. Creates new empty expired cookie. """ # TODO: do we need domain/path here? self._cookies.pop(name, None) self.set_cookie(name, '', max_age=0, expires="Thu, 01 Jan 1970 00:00:00 GMT", domain=domain, path=path) @property def content_length(self): # Just a placeholder for adding setter return super().content_length @content_length.setter def content_length(self, value): if value is not None: value = int(value) # TODO: raise error if chunked enabled self.headers[hdrs.CONTENT_LENGTH] = str(value) else: self.headers.pop(hdrs.CONTENT_LENGTH, None) @property def content_type(self): # Just a placeholder for adding setter return super().content_type @content_type.setter def content_type(self, value): self.content_type # read header values if needed self._content_type = str(value) self._generate_content_type_header() @property def charset(self): # Just a placeholder for adding setter return super().charset @charset.setter def charset(self, value): ctype = self.content_type # read header values if needed if ctype == 'application/octet-stream': raise RuntimeError("Setting charset for application/octet-stream " "doesn't make sense, setup content_type first") if value is None: self._content_dict.pop('charset', None) else: self._content_dict['charset'] = str(value).lower() self._generate_content_type_header() @property def last_modified(self, _LAST_MODIFIED=hdrs.LAST_MODIFIED): """The value of Last-Modified HTTP header, or None. This header is represented as a `datetime` object. """ httpdate = self.headers.get(_LAST_MODIFIED) if httpdate is not None: timetuple = parsedate(httpdate) if timetuple is not None: return datetime.datetime(*timetuple[:6], tzinfo=datetime.timezone.utc) return None @last_modified.setter def last_modified(self, value): if value is None: self.headers.pop(hdrs.LAST_MODIFIED, None) elif isinstance(value, (int, float)): self.headers[hdrs.LAST_MODIFIED] = time.strftime( "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(math.ceil(value))) elif isinstance(value, datetime.datetime): self.headers[hdrs.LAST_MODIFIED] = time.strftime( "%a, %d %b %Y %H:%M:%S GMT", value.utctimetuple()) elif isinstance(value, str): self.headers[hdrs.LAST_MODIFIED] = value @property def tcp_nodelay(self): return self._tcp_nodelay def set_tcp_nodelay(self, value): value = bool(value) self._tcp_nodelay = value if value: self._tcp_cork = False if self._resp_impl is None: return if value: self._resp_impl.transport.set_tcp_cork(False) self._resp_impl.transport.set_tcp_nodelay(value) @property def tcp_cork(self): return self._tcp_cork def set_tcp_cork(self, value): value = bool(value) self._tcp_cork = value if value: self._tcp_nodelay = False if self._resp_impl is None: return if value: self._resp_impl.transport.set_tcp_nodelay(False) self._resp_impl.transport.set_tcp_cork(value) def _generate_content_type_header(self, CONTENT_TYPE=hdrs.CONTENT_TYPE): params = '; '.join("%s=%s" % i for i in self._content_dict.items()) if params: ctype = self._content_type + '; ' + params else: ctype = self._content_type self.headers[CONTENT_TYPE] = ctype def _start_pre_check(self, request): if self._resp_impl is not None: if self._req is not request: raise RuntimeError( "Response has been started with different request.") else: return self._resp_impl else: return None def _do_start_compression(self, coding): if coding != ContentCoding.identity: self.headers[hdrs.CONTENT_ENCODING] = coding.value self._resp_impl.add_compression_filter(coding.value) self.content_length = None def _start_compression(self, request): if self._compression_force: self._do_start_compression(self._compression_force) else: accept_encoding = request.headers.get( hdrs.ACCEPT_ENCODING, '').lower() for coding in ContentCoding: if coding.value in accept_encoding: self._do_start_compression(coding) return def start(self, request): warnings.warn('use .prepare(request) instead', DeprecationWarning) resp_impl = self._start_pre_check(request) if resp_impl is not None: return resp_impl return self._start(request) @asyncio.coroutine def prepare(self, request): resp_impl = self._start_pre_check(request) if resp_impl is not None: return resp_impl for app in request.match_info.apps: yield from app.on_response_prepare.send(request, self) return self._start(request) def _start(self, request): self._req = request keep_alive = self._keep_alive if keep_alive is None: keep_alive = request.keep_alive self._keep_alive = keep_alive resp_impl = self._resp_impl = ResponseImpl( request._writer, self._status, request.version, not keep_alive, self._reason) self._copy_cookies() if self._compression: self._start_compression(request) if self._chunked: if request.version != HttpVersion11: raise RuntimeError("Using chunked encoding is forbidden " "for HTTP/{0.major}.{0.minor}".format( request.version)) resp_impl.enable_chunked_encoding() if self._chunk_size: resp_impl.add_chunking_filter(self._chunk_size) headers = self.headers.items() for key, val in headers: resp_impl.add_header(key, val) resp_impl.transport.set_tcp_nodelay(self._tcp_nodelay) resp_impl.transport.set_tcp_cork(self._tcp_cork) self._send_headers(resp_impl) return resp_impl def _send_headers(self, resp_impl): # Durty hack required for # https://github.com/KeepSafe/aiohttp/issues/1093 # File sender may override it resp_impl.send_headers() def write(self, data): assert isinstance(data, (bytes, bytearray, memoryview)), \ "data argument must be byte-ish (%r)" % type(data) if self._eof_sent: raise RuntimeError("Cannot call write() after write_eof()") if self._resp_impl is None: raise RuntimeError("Cannot call write() before start()") if data: return self._resp_impl.write(data) else: return () @asyncio.coroutine def drain(self): if self._resp_impl is None: raise RuntimeError("Response has not been started") yield from self._resp_impl.transport.drain() @asyncio.coroutine def write_eof(self): if self._eof_sent: return if self._resp_impl is None: raise RuntimeError("Response has not been started") yield from self._resp_impl.write_eof() self._eof_sent = True def __repr__(self): if self.started: info = "{} {} ".format(self._req.method, self._req.path) else: info = "not started" return "<{} {} {}>".format(self.__class__.__name__, self.reason, info)
class HTTPResponse(object): def __init__(self, body=None, status=200, headers=None, content_type="text/plain", body_bytes=b""): self.content_type = content_type self.headers = headers if body is not None: self.body = self.encode_body(body) else: self.body = body_bytes self.status = status self.headers = CIMultiDict(headers or {}) def encode_body(self, data): try: return data.encode() except AttributeError: return str(data).encode() def parse_headers(self): headers = b"" for key, value in self.headers.items(): try: headers += b"%b: %b\r\n" % (key.encode(), value.encode("utf-8")) except AttributeError: headers += b"%b: %b\r\n" % (str(key).encode(), str(value).encode("utf-8")) return headers def has_message_body(self): """ According to the following RFC message body and length SHOULD NOT be included in responses status 1XX, 204 and 304. https://tools.ietf.org/html/rfc2616#section-4.4 https://tools.ietf.org/html/rfc2616#section-4.3 """ return self.status not in (204, 304) and not (100 <= self.status < 200) def output(self, version="1.1", keep_alive=False, keep_alive_timeout=None): """ TODO 先暂时不支持 keep alive :param version: :return: """ timeout_header = b"" if keep_alive and keep_alive_timeout is not None: timeout_header = b"Keep-Alive: %d\r\n" % keep_alive_timeout body = b"" if self.has_message_body(): body = self.body self.headers["Content-Length"] = self.headers.get( "Content-Length", len(self.body)) self.headers["Content-Type"] = self.headers.get( "Content-Type", self.content_type) if self.status in (304, 412): self.headers = remove_entity_headers(self.headers) headers = self.parse_headers() if self.status == 200: status_msg = b"OK" else: status_msg = STATUS_CODES.get(self.status, b"UNKNOWN RESPONSE") """ 消息结构如下: HTTP/1.1 200 OK # 状态行 Connection: close # headers Content-Type: text/plain # 空行 xxxxxbody part # body内容 """ return (b"HTTP/%b %d %b\r\n" b"Connection: %b\r\n" b"%b" b"%b\r\n" b"%b") % (version.encode(), self.status, status_msg, b"keep-alive" if keep_alive else b"close", timeout_header, headers, body)
class HttpRequest(RequestBase): """An :class:`HttpClient` request for an HTTP resource. This class has a similar interface to :class:`urllib.request.Request`. :param files: optional dictionary of name, file-like-objects. :param allow_redirects: allow the response to follow redirects. .. attribute:: method The request method .. attribute:: version HTTP version for this request, usually ``HTTP/1.1`` .. attribute:: history List of past :class:`.HttpResponse` (collected during redirects). .. attribute:: wait_continue if ``True``, the :class:`HttpRequest` includes the ``Expect: 100-Continue`` header. .. attribute:: stream Allow for streaming body """ _proxy = None _ssl = None _tunnel = None _write_done = False def __init__(self, client, url, method, inp_params=None, headers=None, data=None, files=None, json=None, history=None, auth=None, charset=None, max_redirects=10, source_address=None, allow_redirects=False, decompress=True, version=None, wait_continue=False, websocket_handler=None, cookies=None, params=None, stream=False, proxies=None, verify=True, cert=None, **extra): self.client = client self.method = method.upper() self.inp_params = inp_params or {} self.unredirected_headers = CIMultiDict() self.history = history self.wait_continue = wait_continue self.max_redirects = max_redirects self.allow_redirects = allow_redirects self.charset = charset or 'utf-8' self.version = version self.decompress = decompress self.websocket_handler = websocket_handler self.source_address = source_address self.stream = stream self.verify = verify self.cert = cert if auth and not isinstance(auth, Auth): auth = HTTPBasicAuth(*auth) self.auth = auth self.url = full_url(url, params, method=self.method) self._set_proxy(proxies) self.key = RequestKey.create(self) self.headers = client.get_headers(self, headers) self.body = self._encode_body(data, files, json) self.unredirected_headers['host'] = self.key.netloc cookies = cookiejar_from_dict(client.cookies, cookies) if cookies: cookies.add_cookie_header(self) @property def _loop(self): return self.client._loop @property def ssl(self): """Context for TLS connections. If this is a tunneled request and the tunnel connection is not yet established, it returns ``None``. """ return self._ssl @property def proxy(self): """Proxy server for this request. """ return self._proxy @property def tunnel(self): """Tunnel for this request. """ return self._tunnel def __repr__(self): return self.first_line() __str__ = __repr__ def first_line(self): if self.method == 'CONNECT': url = self.key.netloc elif self._proxy: url = self.url else: p = urlparse(self.url) url = urlunparse(('', '', p.path or '/', p.params, p.query, p.fragment)) return '%s %s %s' % (self.method, url, self.version) def is_chunked(self): return self.body and 'content-length' not in self.headers def encode(self): """The bytes representation of this :class:`HttpRequest`. Called by :class:`HttpResponse` when it needs to encode this :class:`HttpRequest` before sending it to the HTTP resource. """ # Call body before fist_line in case the query is changes. first_line = self.first_line() if self.body and self.wait_continue: self.headers['expect'] = '100-continue' headers = self.headers if self.unredirected_headers: headers = self.unredirected_headers.copy() headers.update(self.headers) buffer = [first_line.encode('ascii'), b'\r\n'] buffer.extend((('%s: %s\r\n' % (name, value)).encode(CHARSET) for name, value in headers.items())) buffer.append(b'\r\n') return b''.join(buffer) def add_header(self, key, value): self.headers[key] = value def has_header(self, header_name): """Check ``header_name`` is in this request headers. """ return (header_name in self.headers or header_name in self.unredirected_headers) def get_header(self, header_name, default=None): """Retrieve ``header_name`` from this request headers. """ return self.headers.get( header_name, self.unredirected_headers.get(header_name, default)) def remove_header(self, header_name): """Remove ``header_name`` from this request. """ val1 = self.headers.pop(header_name, None) val2 = self.unredirected_headers.pop(header_name, None) return val1 or val2 def add_unredirected_header(self, header_name, header_value): self.unredirected_headers[header_name] = header_value def write_body(self, transport): assert not self._write_done, 'Body already sent' self._write_done = True if not self.body: return if is_streamed(self.body): self._loop.create_task(self._write_streamed_data(transport)) else: self._write_body_data(transport, self.body, True) # INTERNAL ENCODING METHODS def _encode_body(self, data, files, json): body = None ct = None if isinstance(data, (str, bytes)): if files: raise ValueError('data cannot be a string or bytes when ' 'files are present') body = to_bytes(data, self.charset) elif data and is_streamed(data): if files: raise ValueError('data cannot be an iterator when ' 'files are present') if 'content-length' not in self.headers: self.headers['transfer-encoding'] = 'chunked' return data elif data or files: if files: body, ct = self._encode_files(data, files) else: body, ct = self._encode_params(data) elif json: body = _json.dumps(json).encode(self.charset) ct = 'application/json' if not self.headers.get('content-type') and ct: self.headers['Content-Type'] = ct if body: self.headers['content-length'] = str(len(body)) return body def _encode_files(self, data, files): fields = [] for field, val in mapping_iterator(data or ()): if (isinstance(val, str) or isinstance(val, bytes) or not hasattr(val, '__iter__')): val = [val] for v in val: if v is not None: if not isinstance(v, bytes): v = str(v) fields.append( (field.decode('utf-8') if isinstance(field, bytes) else field, v.encode('utf-8') if isinstance(v, str) else v)) for (k, v) in mapping_iterator(files): # support for explicit filename ft = None if isinstance(v, (tuple, list)): if len(v) == 2: fn, fp = v else: fn, fp, ft = v else: fn = guess_filename(v) or k fp = v if isinstance(fp, bytes): fp = BytesIO(fp) elif isinstance(fp, str): fp = StringIO(fp) if ft: new_v = (fn, fp.read(), ft) else: new_v = (fn, fp.read()) fields.append((k, new_v)) # return encode_multipart_formdata(fields, charset=self.charset) def _encode_params(self, params): content_type = self.headers.get('content-type') # No content type given, chose one if not content_type: content_type = FORM_URL_ENCODED if hasattr(params, 'read'): params = params.read() if content_type in JSON_CONTENT_TYPES: body = _json.dumps(params) elif content_type == FORM_URL_ENCODED: body = urlencode(tuple(split_url_params(params))) elif content_type == MULTIPART_FORM_DATA: body, content_type = encode_multipart_formdata( params, charset=self.charset) else: body = params return to_bytes(body, self.charset), content_type def _write_body_data(self, transport, data, finish=False): if self.is_chunked(): data = http_chunks(data, finish) elif data: data = (data, ) else: return for chunk in data: transport.write(chunk) async def _write_streamed_data(self, transport): for data in self.body: try: data = await data except TypeError: pass self._write_body_data(transport, data) self._write_body_data(transport, b'', True) # PROXY INTERNALS def _set_proxy(self, proxies): url = urlparse(self.url) request_proxies = self.client.proxies.copy() if proxies: request_proxies.update(proxies) self.proxies = request_proxies # if url.scheme in request_proxies: host, port = get_hostport(url.scheme, url.netloc) no_proxy = [ n for n in request_proxies.get('no', '').split(',') if n ] if not any(map(host.endswith, no_proxy)): proxy_url = request_proxies[url.scheme] if url.scheme in tls_schemes: self._tunnel = proxy_url else: self._proxy = proxy_url
class HTTPResponse(BaseHTTPResponse): __slots__ = ("body", "status", "content_type", "headers", "_cookies") def __init__( self, body=None, status=200, headers=None, content_type="text/plain", body_bytes=b"", ): self.content_type = content_type if body is not None: self.body = self._encode_body(body) else: self.body = body_bytes self.status = status self.headers = CIMultiDict(headers or {}) self._cookies = None def output(self, version="1.1", keep_alive=False, keep_alive_timeout=None): # This is all returned in a kind-of funky way # We tried to make this as fast as possible in pure python timeout_header = b"" if keep_alive and keep_alive_timeout is not None: timeout_header = b"Keep-Alive: %d\r\n" % keep_alive_timeout body = b"" if has_message_body(self.status): body = self.body self.headers["Content-Length"] = self.headers.get( "Content-Length", len(self.body) ) self.headers["Content-Type"] = self.headers.get( "Content-Type", self.content_type ) if self.status in (304, 412): self.headers = remove_entity_headers(self.headers) headers = self._parse_headers() if self.status == 200: status = b"OK" else: status = STATUS_CODES.get(self.status, b"UNKNOWN RESPONSE") return ( b"HTTP/%b %d %b\r\n" b"Connection: %b\r\n" b"%b" b"%b\r\n" b"%b" ) % ( version.encode(), self.status, status, b"keep-alive" if keep_alive else b"close", timeout_header, headers, body, ) @property def cookies(self): if self._cookies is None: self._cookies = CookieJar(self.headers) return self._cookies
async def create( cls, sanic_app, scope: ASGIScope, receive: ASGIReceive, send: ASGISend ) -> "ASGIApp": instance = cls() instance.sanic_app = sanic_app instance.transport = MockTransport(scope, receive, send) instance.transport.add_task = sanic_app.loop.create_task instance.transport.loop = sanic_app.loop headers = CIMultiDict( [ (key.decode("latin-1"), value.decode("latin-1")) for key, value in scope.get("headers", []) ] ) instance.do_stream = ( True if headers.get("expect") == "100-continue" else False ) instance.lifespan = Lifespan(instance) if scope["type"] == "lifespan": await instance.lifespan(scope, receive, send) else: url_bytes = scope.get("root_path", "") + quote(scope["path"]) url_bytes = url_bytes.encode("latin-1") url_bytes += b"?" + scope["query_string"] if scope["type"] == "http": version = scope["http_version"] method = scope["method"] elif scope["type"] == "websocket": version = "1.1" method = "GET" instance.ws = instance.transport.create_websocket_connection( send, receive ) await instance.ws.accept() else: pass # TODO: # - close connection instance.request = Request( url_bytes, headers, version, method, instance.transport, sanic_app, ) if sanic_app.is_request_stream: is_stream_handler = sanic_app.router.is_stream_handler( instance.request ) if is_stream_handler: instance.request.stream = StreamBuffer( sanic_app.config.REQUEST_BUFFER_QUEUE_SIZE ) instance.do_stream = True return instance
def make_mocked_request(method, path, headers=None, *, version=HttpVersion(1, 1), closing=False, app=None, writer=sentinel, payload_writer=sentinel, protocol=sentinel, transport=sentinel, payload=sentinel, sslcontext=None, client_max_size=1024**2): """ XXX copied from aiohttp but using guillotina request object Creates mocked web.Request testing purposes. Useful in unit tests, when spinning full web server is overkill or specific conditions and errors are hard to trigger. """ task = mock.Mock() loop = mock.Mock() loop.create_future.return_value = () if version < HttpVersion(1, 1): closing = True if headers: headers = CIMultiDict(headers) raw_hdrs = tuple( (k.encode('utf-8'), v.encode('utf-8')) for k, v in headers.items()) else: headers = CIMultiDict() raw_hdrs = () chunked = 'chunked' in headers.get(hdrs.TRANSFER_ENCODING, '').lower() message = RawRequestMessage( method, path, version, headers, raw_hdrs, closing, False, False, chunked, URL(path)) if app is None: app = test_utils._create_app_mock() if protocol is sentinel: protocol = mock.Mock() if transport is sentinel: transport = test_utils._create_transport(sslcontext) if writer is sentinel: writer = mock.Mock() writer.transport = transport if payload_writer is sentinel: payload_writer = mock.Mock() payload_writer.write_eof.side_effect = noop payload_writer.drain.side_effect = noop protocol.transport = transport protocol.writer = writer if payload is sentinel: payload = mock.Mock() time_service = mock.Mock() time_service.time.return_value = 12345 time_service.strtime.return_value = "Tue, 15 Nov 1994 08:12:31 GMT" @contextmanager def timeout(*args, **kw): yield time_service.timeout = mock.Mock() time_service.timeout.side_effect = timeout req = Request(message, payload, protocol, payload_writer, time_service, task, client_max_size=client_max_size) match_info = UrlMappingMatchInfo({}, mock.Mock()) match_info.add_app(app) req._match_info = match_info return req
class HttpMessage(ABC): """HttpMessage allows to write headers and payload to a stream. For example, lets say we want to read file then compress it with deflate compression and then send it with chunked transfer encoding, code may look like this: >>> response = aiohttp.Response(transport, 200) We have to use deflate compression first: >>> response.add_compression_filter('deflate') Then we want to split output stream into chunks of 1024 bytes size: >>> response.add_chunking_filter(1024) We can add headers to response with add_headers() method. add_headers() does not send data to transport, send_headers() sends request/response line and then sends headers: >>> response.add_headers( ... ('Content-Disposition', 'attachment; filename="..."')) >>> response.send_headers() Now we can use chunked writer to write stream to a network stream. First call to write() method sends response status line and headers, add_header() and add_headers() method unavailable at this stage: >>> with open('...', 'rb') as f: ... chunk = fp.read(8192) ... while chunk: ... response.write(chunk) ... chunk = fp.read(8192) >>> response.write_eof() """ writer = None # 'filter' is being used for altering write() behaviour, # add_chunking_filter adds deflate/gzip compression and # add_compression_filter splits incoming data into a chunks. filter = None HOP_HEADERS = None # Must be set by subclass. SERVER_SOFTWARE = 'Python/{0[0]}.{0[1]} aiohttp/{1}'.format( sys.version_info, aiohttp.__version__) upgrade = False # Connection: UPGRADE websocket = False # Upgrade: WEBSOCKET has_chunked_hdr = False # Transfer-encoding: chunked # subclass can enable auto sending headers with write() call, # this is useful for wsgi's start_response implementation. _send_headers = False def __init__(self, transport, version, close): self.transport = transport self._version = version self.closing = close self.keepalive = None self.chunked = False self.length = None self.headers = CIMultiDict() self.headers_sent = False self.output_length = 0 self.headers_length = 0 self._output_size = 0 self._cache = {} @property @abstractmethod def status_line(self): return b'' @abstractmethod def autochunked(self): return False @property def version(self): return self._version @property def body_length(self): return self.output_length - self.headers_length def force_close(self): self.closing = True self.keepalive = False def enable_chunked_encoding(self): self.chunked = True def keep_alive(self): if self.keepalive is None: if self.version < HttpVersion10: # keep alive not supported at all return False if self.version == HttpVersion10: if self.headers.get(hdrs.CONNECTION) == 'keep-alive': return True else: # no headers means we close for Http 1.0 return False else: return not self.closing else: return self.keepalive def is_headers_sent(self): return self.headers_sent def add_header(self, name, value): """Analyze headers. Calculate content length, removes hop headers, etc.""" assert not self.headers_sent, 'headers have been sent already' assert isinstance(name, str), \ 'Header name should be a string, got {!r}'.format(name) assert set(name).issubset(ASCIISET), \ 'Header name should contain ASCII chars, got {!r}'.format(name) assert isinstance(value, str), \ 'Header {!r} should have string value, got {!r}'.format( name, value) name = istr(name) value = value.strip() if name == hdrs.CONTENT_LENGTH: self.length = int(value) if name == hdrs.TRANSFER_ENCODING: self.has_chunked_hdr = value.lower().strip() == 'chunked' if name == hdrs.CONNECTION: val = value.lower() # handle websocket if 'upgrade' in val: self.upgrade = True # connection keep-alive elif 'close' in val: self.keepalive = False elif 'keep-alive' in val: self.keepalive = True elif name == hdrs.UPGRADE: if 'websocket' in value.lower(): self.websocket = True self.headers[name] = value elif name not in self.HOP_HEADERS: # ignore hop-by-hop headers self.headers.add(name, value) def add_headers(self, *headers): """Adds headers to a HTTP message.""" for name, value in headers: self.add_header(name, value) def send_headers(self, _sep=': ', _end='\r\n'): """Writes headers to a stream. Constructs payload writer.""" # Chunked response is only for HTTP/1.1 clients or newer # and there is no Content-Length header is set. # Do not use chunked responses when the response is guaranteed to # not have a response body (304, 204). assert not self.headers_sent, 'headers have been sent already' self.headers_sent = True if self.chunked or self.autochunked(): self.writer = self._write_chunked_payload() self.headers[hdrs.TRANSFER_ENCODING] = 'chunked' elif self.length is not None: self.writer = self._write_length_payload(self.length) else: self.writer = self._write_eof_payload() next(self.writer) self._add_default_headers() # status + headers headers = self.status_line + ''.join( [k + _sep + v + _end for k, v in self.headers.items()]) headers = headers.encode('utf-8') + b'\r\n' self.output_length += len(headers) self.headers_length = len(headers) self.transport.write(headers) def _add_default_headers(self): # set the connection header connection = None if self.upgrade: connection = 'Upgrade' elif not self.closing if self.keepalive is None else self.keepalive: if self.version == HttpVersion10: connection = 'keep-alive' else: if self.version == HttpVersion11: connection = 'close' if connection is not None: self.headers[hdrs.CONNECTION] = connection def write(self, chunk, *, drain=False, EOF_MARKER=EOF_MARKER, EOL_MARKER=EOL_MARKER): """Writes chunk of data to a stream by using different writers. writer uses filter to modify chunk of data. write_eof() indicates end of stream. writer can't be used after write_eof() method being called. write() return drain future. """ assert (isinstance(chunk, (bytes, bytearray)) or chunk is EOF_MARKER), chunk size = self.output_length if self._send_headers and not self.headers_sent: self.send_headers() if self.filter: chunk = self.filter.send(chunk) while chunk not in (EOF_MARKER, EOL_MARKER): if chunk: self.writer.send(chunk) chunk = next(self.filter) else: if chunk is not EOF_MARKER: self.writer.send(chunk) self._output_size += self.output_length - size if self._output_size > 64 * 1024: if drain: self._output_size = 0 return self.transport.drain() return () def write_eof(self): self.write(EOF_MARKER) try: self.writer.throw(aiohttp.EofStream()) except StopIteration: pass return self.transport.drain() def _write_chunked_payload(self): """Write data in chunked transfer encoding.""" while True: try: chunk = yield except aiohttp.EofStream: self.transport.write(b'0\r\n\r\n') self.output_length += 5 break chunk = bytes(chunk) chunk_len = '{:x}\r\n'.format(len(chunk)).encode('ascii') self.transport.write(chunk_len + chunk + b'\r\n') self.output_length += len(chunk_len) + len(chunk) + 2 def _write_length_payload(self, length): """Write specified number of bytes to a stream.""" while True: try: chunk = yield except aiohttp.EofStream: break if length: l = len(chunk) if length >= l: self.transport.write(chunk) self.output_length += l length = length-l else: self.transport.write(chunk[:length]) self.output_length += length length = 0 def _write_eof_payload(self): while True: try: chunk = yield except aiohttp.EofStream: break self.transport.write(chunk) self.output_length += len(chunk) @wrap_payload_filter def add_chunking_filter(self, chunk_size=16*1024, *, EOF_MARKER=EOF_MARKER, EOL_MARKER=EOL_MARKER): """Split incoming stream into chunks.""" buf = bytearray() chunk = yield while True: if chunk is EOF_MARKER: if buf: yield buf yield EOF_MARKER else: buf.extend(chunk) while len(buf) >= chunk_size: chunk = bytes(buf[:chunk_size]) del buf[:chunk_size] yield chunk chunk = yield EOL_MARKER @wrap_payload_filter def add_compression_filter(self, encoding='deflate', *, EOF_MARKER=EOF_MARKER, EOL_MARKER=EOL_MARKER): """Compress incoming stream with deflate or gzip encoding.""" zlib_mode = (16 + zlib.MAX_WBITS if encoding == 'gzip' else -zlib.MAX_WBITS) zcomp = zlib.compressobj(wbits=zlib_mode) chunk = yield while True: if chunk is EOF_MARKER: yield zcomp.flush() chunk = yield EOF_MARKER else: yield zcomp.compress(chunk) chunk = yield EOL_MARKER
class ClientRequest: GET_METHODS = { hdrs.METH_GET, hdrs.METH_HEAD, hdrs.METH_OPTIONS, hdrs.METH_TRACE, } POST_METHODS = {hdrs.METH_PATCH, hdrs.METH_POST, hdrs.METH_PUT} ALL_METHODS = GET_METHODS.union(POST_METHODS).union({hdrs.METH_DELETE}) DEFAULT_HEADERS = { hdrs.ACCEPT: '*/*', hdrs.ACCEPT_ENCODING: 'gzip, deflate', } body = b'' auth = None response = None response_class = None _writer = None # async task for streaming data _continue = None # waiter future for '100 Continue' response # N.B. # Adding __del__ method with self._writer closing doesn't make sense # because _writer is instance method, thus it keeps a reference to self. # Until writer has finished finalizer will not be called. def __init__(self, method: str, url: URL, *, params: Optional[Mapping[str, str]] = None, headers: Optional[LooseHeaders] = None, skip_auto_headers: Iterable[str] = frozenset(), data: Any = None, cookies: Optional[LooseCookies] = None, auth: Optional[BasicAuth] = None, version: http.HttpVersion = http.HttpVersion11, compress: Optional[str] = None, chunked: Optional[bool] = None, expect100: bool = False, loop: Optional[asyncio.AbstractEventLoop] = None, response_class: Optional[Type['ClientResponse']] = None, proxy: Optional[URL] = None, proxy_auth: Optional[BasicAuth] = None, timer: Optional[BaseTimerContext] = None, session: Optional['ClientSession'] = None, ssl: Union[SSLContext, bool, Fingerprint, None] = None, proxy_headers: Optional[LooseHeaders] = None, traces: Optional[List['Trace']] = None): if loop is None: loop = asyncio.get_event_loop() assert isinstance(url, URL), url assert isinstance(proxy, (URL, type(None))), proxy # FIXME: session is None in tests only, need to fix tests # assert session is not None self._session = cast('ClientSession', session) if params: q = MultiDict(url.query) url2 = url.with_query(params) q.extend(url2.query) url = url.with_query(q) self.original_url = url self.url = url.with_fragment(None) self.method = method.upper() self.chunked = chunked self.compress = compress self.loop = loop self.length = None if response_class is None: real_response_class = ClientResponse else: real_response_class = response_class self.response_class = real_response_class # type: Type[ClientResponse] self._timer = timer if timer is not None else TimerNoop() self._ssl = ssl if loop.get_debug(): self._source_traceback = traceback.extract_stack(sys._getframe(1)) self.update_version(version) self.update_host(url) self.update_headers(headers) self.update_auto_headers(skip_auto_headers) self.update_cookies(cookies) self.update_content_encoding(data) self.update_auth(auth) self.update_proxy(proxy, proxy_auth, proxy_headers) self.update_body_from_data(data) if data or self.method not in self.GET_METHODS: self.update_transfer_encoding() self.update_expect_continue(expect100) if traces is None: traces = [] self._traces = traces def is_ssl(self) -> bool: return self.url.scheme in ('https', 'wss') @property def ssl(self) -> Union['SSLContext', None, bool, Fingerprint]: return self._ssl @property def connection_key(self) -> ConnectionKey: proxy_headers = self.proxy_headers if proxy_headers: h = hash(tuple( (k, v) for k, v in proxy_headers.items())) # type: Optional[int] # noqa else: h = None return ConnectionKey(self.host, self.port, self.is_ssl(), self.ssl, self.proxy, self.proxy_auth, h) @property def host(self) -> str: ret = self.url.host assert ret is not None return ret @property def port(self) -> Optional[int]: return self.url.port @property def request_info(self) -> RequestInfo: headers = CIMultiDictProxy(self.headers) # type: CIMultiDictProxy[str] return RequestInfo(self.url, self.method, headers, self.original_url) def update_host(self, url: URL) -> None: """Update destination host, port and connection type (ssl).""" # get host/port if not url.host: raise InvalidURL(url) # basic auth info username, password = url.user, url.password if username: self.auth = helpers.BasicAuth(username, password or '') def update_version(self, version: Union[http.HttpVersion, str]) -> None: """Convert request version to two elements tuple. parser HTTP version '1.1' => (1, 1) """ if isinstance(version, str): v = [l.strip() for l in version.split('.', 1)] try: version = http.HttpVersion(int(v[0]), int(v[1])) except ValueError: raise ValueError( 'Can not parse http version number: {}'.format( version)) from None self.version = version def update_headers(self, headers: Optional[LooseHeaders]) -> None: """Update request headers.""" self.headers = CIMultiDict() # type: CIMultiDict[str] # add host netloc = cast(str, self.url.raw_host) if helpers.is_ipv6_address(netloc): netloc = '[{}]'.format(netloc) if not self.url.is_default_port(): netloc += ':' + str(self.url.port) self.headers[hdrs.HOST] = netloc if headers: if isinstance(headers, (dict, MultiDictProxy, MultiDict)): headers = headers.items() # type: ignore for key, value in headers: # A special case for Host header if key.lower() == 'host': self.headers[key] = value else: self.headers.add(key, value) def update_auto_headers(self, skip_auto_headers: Iterable[str]) -> None: self.skip_auto_headers = CIMultiDict( (hdr, None) for hdr in sorted(skip_auto_headers)) used_headers = self.headers.copy() used_headers.extend(self.skip_auto_headers) # type: ignore for hdr, val in self.DEFAULT_HEADERS.items(): if hdr not in used_headers: self.headers.add(hdr, val) if hdrs.USER_AGENT not in used_headers: self.headers[hdrs.USER_AGENT] = SERVER_SOFTWARE def update_cookies(self, cookies: Optional[LooseCookies]) -> None: """Update request cookies header.""" if not cookies: return c = SimpleCookie() if hdrs.COOKIE in self.headers: c.load(self.headers.get(hdrs.COOKIE, '')) del self.headers[hdrs.COOKIE] if isinstance(cookies, Mapping): iter_cookies = cookies.items() else: iter_cookies = cookies # type: ignore for name, value in iter_cookies: if isinstance(value, Morsel): # Preserve coded_value mrsl_val = value.get(value.key, Morsel()) mrsl_val.set(value.key, value.value, value.coded_value) # type: ignore # noqa c[name] = mrsl_val else: c[name] = value # type: ignore self.headers[hdrs.COOKIE] = c.output(header='', sep=';').strip() def update_content_encoding(self, data: Any) -> None: """Set request content encoding.""" if not data: return enc = self.headers.get(hdrs.CONTENT_ENCODING, '').lower() if enc: if self.compress: raise ValueError('compress can not be set ' 'if Content-Encoding header is set') elif self.compress: if not isinstance(self.compress, str): self.compress = 'deflate' self.headers[hdrs.CONTENT_ENCODING] = self.compress self.chunked = True # enable chunked, no need to deal with length def update_transfer_encoding(self) -> None: """Analyze transfer-encoding header.""" te = self.headers.get(hdrs.TRANSFER_ENCODING, '').lower() if 'chunked' in te: if self.chunked: raise ValueError( 'chunked can not be set ' 'if "Transfer-Encoding: chunked" header is set') elif self.chunked: if hdrs.CONTENT_LENGTH in self.headers: raise ValueError('chunked can not be set ' 'if Content-Length header is set') self.headers[hdrs.TRANSFER_ENCODING] = 'chunked' else: if hdrs.CONTENT_LENGTH not in self.headers: self.headers[hdrs.CONTENT_LENGTH] = str(len(self.body)) def update_auth(self, auth: Optional[BasicAuth]) -> None: """Set basic auth.""" if auth is None: auth = self.auth if auth is None: return if not isinstance(auth, helpers.BasicAuth): raise TypeError('BasicAuth() tuple is required instead') self.headers[hdrs.AUTHORIZATION] = auth.encode() def update_body_from_data(self, body: Any) -> None: if not body: return # FormData if isinstance(body, FormData): body = body() try: body = payload.PAYLOAD_REGISTRY.get(body, disposition=None) except payload.LookupError: body = FormData(body)() self.body = body # enable chunked encoding if needed if not self.chunked: if hdrs.CONTENT_LENGTH not in self.headers: size = body.size if size is None: self.chunked = True else: if hdrs.CONTENT_LENGTH not in self.headers: self.headers[hdrs.CONTENT_LENGTH] = str(size) # set content-type if (hdrs.CONTENT_TYPE not in self.headers and hdrs.CONTENT_TYPE not in self.skip_auto_headers): self.headers[hdrs.CONTENT_TYPE] = body.content_type # copy payload headers if body.headers: for (key, value) in body.headers.items(): if key not in self.headers: self.headers[key] = value def update_expect_continue(self, expect: bool = False) -> None: if expect: self.headers[hdrs.EXPECT] = '100-continue' elif self.headers.get(hdrs.EXPECT, '').lower() == '100-continue': expect = True if expect: self._continue = self.loop.create_future() def update_proxy(self, proxy: Optional[URL], proxy_auth: Optional[BasicAuth], proxy_headers: Optional[LooseHeaders]) -> None: if proxy and not proxy.scheme == 'http': raise ValueError("Only http proxies are supported") if proxy_auth and not isinstance(proxy_auth, helpers.BasicAuth): raise ValueError("proxy_auth must be None or BasicAuth() tuple") self.proxy = proxy self.proxy_auth = proxy_auth self.proxy_headers = proxy_headers def keep_alive(self) -> bool: if self.version < HttpVersion10: # keep alive not supported at all return False if self.version == HttpVersion10: if self.headers.get(hdrs.CONNECTION) == 'keep-alive': return True else: # no headers means we close for Http 1.0 return False elif self.headers.get(hdrs.CONNECTION) == 'close': return False return True async def write_bytes(self, writer: AbstractStreamWriter, conn: 'Connection') -> None: """Support coroutines that yields bytes objects.""" # 100 response if self._continue is not None: await writer.drain() await self._continue protocol = conn.protocol assert protocol is not None try: if isinstance(self.body, payload.Payload): await self.body.write(writer) else: if isinstance(self.body, (bytes, bytearray)): self.body = (self.body, ) # type: ignore for chunk in self.body: await writer.write(chunk) # type: ignore await writer.write_eof() except OSError as exc: new_exc = ClientOSError( exc.errno, 'Can not write request body for %s' % self.url) new_exc.__context__ = exc new_exc.__cause__ = exc protocol.set_exception(new_exc) except asyncio.CancelledError as exc: if not conn.closed: protocol.set_exception(exc) except Exception as exc: protocol.set_exception(exc) finally: self._writer = None async def send(self, conn: 'Connection') -> 'ClientResponse': # Specify request target: # - CONNECT request must send authority form URI # - not CONNECT proxy must send absolute form URI # - most common is origin form URI if self.method == hdrs.METH_CONNECT: path = '{}:{}'.format(self.url.raw_host, self.url.port) elif self.proxy and not self.is_ssl(): path = str(self.url) else: path = self.url.raw_path if self.url.raw_query_string: path += '?' + self.url.raw_query_string protocol = conn.protocol assert protocol is not None writer = StreamWriter(protocol, self.loop, on_chunk_sent=self._on_chunk_request_sent) if self.compress: writer.enable_compression(self.compress) if self.chunked is not None: writer.enable_chunking() # set default content-type if (self.method in self.POST_METHODS and hdrs.CONTENT_TYPE not in self.skip_auto_headers and hdrs.CONTENT_TYPE not in self.headers): self.headers[hdrs.CONTENT_TYPE] = 'application/octet-stream' # set the connection header connection = self.headers.get(hdrs.CONNECTION) if not connection: if self.keep_alive(): if self.version == HttpVersion10: connection = 'keep-alive' else: if self.version == HttpVersion11: connection = 'close' if connection is not None: self.headers[hdrs.CONNECTION] = connection # status + headers status_line = '{0} {1} HTTP/{2[0]}.{2[1]}'.format( self.method, path, self.version) await writer.write_headers(status_line, self.headers) self._writer = self.loop.create_task(self.write_bytes(writer, conn)) response_class = self.response_class assert response_class is not None self.response = response_class(self.method, self.original_url, writer=self._writer, continue100=self._continue, timer=self._timer, request_info=self.request_info, traces=self._traces, loop=self.loop, session=self._session) return self.response async def close(self) -> None: if self._writer is not None: try: await self._writer finally: self._writer = None def terminate(self) -> None: if self._writer is not None: if not self.loop.is_closed(): self._writer.cancel() self._writer = None async def _on_chunk_request_sent(self, chunk: bytes) -> None: for trace in self._traces: await trace.send_request_chunk_sent(chunk)
class ClientRequest: GET_METHODS = {hdrs.METH_GET, hdrs.METH_HEAD, hdrs.METH_OPTIONS} POST_METHODS = {hdrs.METH_PATCH, hdrs.METH_POST, hdrs.METH_PUT} ALL_METHODS = GET_METHODS.union(POST_METHODS).union({hdrs.METH_DELETE, hdrs.METH_TRACE}) DEFAULT_HEADERS = {hdrs.ACCEPT: "*/*", hdrs.ACCEPT_ENCODING: "gzip, deflate"} SERVER_SOFTWARE = HttpMessage.SERVER_SOFTWARE body = b"" auth = None response = None response_class = None _writer = None # async task for streaming data _continue = None # waiter future for '100 Continue' response # N.B. # Adding __del__ method with self._writer closing doesn't make sense # because _writer is instance method, thus it keeps a reference to self. # Until writer has finished finalizer will not be called. def __init__( self, method, url, *, params=None, headers=None, skip_auto_headers=frozenset(), data=None, cookies=None, auth=None, encoding="utf-8", version=aiohttp.HttpVersion11, compress=None, chunked=None, expect100=False, loop=None, response_class=None ): if loop is None: loop = asyncio.get_event_loop() self.url = url self.method = method.upper() self.encoding = encoding self.chunked = chunked self.compress = compress self.loop = loop self.response_class = response_class or ClientResponse if loop.get_debug(): self._source_traceback = traceback.extract_stack(sys._getframe(1)) self.update_version(version) self.update_host(url) self.update_path(params) self.update_headers(headers) self.update_auto_headers(skip_auto_headers) self.update_cookies(cookies) self.update_content_encoding() self.update_auth(auth) self.update_body_from_data(data, skip_auto_headers) self.update_transfer_encoding() self.update_expect_continue(expect100) def update_host(self, url): """Update destination host, port and connection type (ssl).""" url_parsed = urllib.parse.urlsplit(url) # check for network location part netloc = url_parsed.netloc if not netloc: raise ValueError("Host could not be detected.") # get host/port host = url_parsed.hostname if not host: raise ValueError("Host could not be detected.") try: port = url_parsed.port except ValueError: raise ValueError("Port number could not be converted.") from None # check domain idna encoding try: netloc = netloc.encode("idna").decode("utf-8") host = host.encode("idna").decode("utf-8") except UnicodeError: raise ValueError("URL has an invalid label.") # basic auth info username, password = url_parsed.username, url_parsed.password if username: self.auth = helpers.BasicAuth(username, password or "") netloc = netloc.split("@", 1)[1] # Record entire netloc for usage in host header self.netloc = netloc scheme = url_parsed.scheme self.ssl = scheme in ("https", "wss") # set port number if it isn't already set if not port: if self.ssl: port = HTTPS_PORT else: port = HTTP_PORT self.host, self.port, self.scheme = host, port, scheme def update_version(self, version): """Convert request version to two elements tuple. parser HTTP version '1.1' => (1, 1) """ if isinstance(version, str): v = [l.strip() for l in version.split(".", 1)] try: version = int(v[0]), int(v[1]) except ValueError: raise ValueError("Can not parse http version number: {}".format(version)) from None self.version = version def update_path(self, params): """Build path.""" # extract path scheme, netloc, path, query, fragment = urllib.parse.urlsplit(self.url) if not path: path = "/" if isinstance(params, collections.Mapping): params = list(params.items()) if params: if not isinstance(params, str): params = urllib.parse.urlencode(params) if query: query = "%s&%s" % (query, params) else: query = params self.path = urllib.parse.urlunsplit(("", "", helpers.requote_uri(path), query, fragment)) self.url = urllib.parse.urlunsplit((scheme, netloc, self.path, "", "")) def update_headers(self, headers): """Update request headers.""" self.headers = CIMultiDict() if headers: if isinstance(headers, dict): headers = headers.items() elif isinstance(headers, (MultiDictProxy, MultiDict)): headers = headers.items() for key, value in headers: self.headers.add(key, value) def update_auto_headers(self, skip_auto_headers): self.skip_auto_headers = skip_auto_headers used_headers = set(self.headers) | skip_auto_headers for hdr, val in self.DEFAULT_HEADERS.items(): if hdr not in used_headers: self.headers.add(hdr, val) # add host if hdrs.HOST not in used_headers: self.headers[hdrs.HOST] = self.netloc if hdrs.USER_AGENT not in used_headers: self.headers[hdrs.USER_AGENT] = self.SERVER_SOFTWARE def update_cookies(self, cookies): """Update request cookies header.""" if not cookies: return c = http.cookies.SimpleCookie() if hdrs.COOKIE in self.headers: c.load(self.headers.get(hdrs.COOKIE, "")) del self.headers[hdrs.COOKIE] if isinstance(cookies, dict): cookies = cookies.items() for name, value in cookies: if isinstance(value, http.cookies.Morsel): c[value.key] = value.value else: c[name] = value self.headers[hdrs.COOKIE] = c.output(header="", sep=";").strip() def update_content_encoding(self): """Set request content encoding.""" enc = self.headers.get(hdrs.CONTENT_ENCODING, "").lower() if enc: if self.compress is not False: self.compress = enc # enable chunked, no need to deal with length self.chunked = True elif self.compress: if not isinstance(self.compress, str): self.compress = "deflate" self.headers[hdrs.CONTENT_ENCODING] = self.compress self.chunked = True # enable chunked, no need to deal with length def update_auth(self, auth): """Set basic auth.""" if auth is None: auth = self.auth if auth is None: return if not isinstance(auth, helpers.BasicAuth): warnings.warn("BasicAuth() tuple is required instead ", DeprecationWarning) auth = helpers.BasicAuth(*auth) self.headers[hdrs.AUTHORIZATION] = auth.encode() def update_body_from_data(self, data, skip_auto_headers): if not data: return if isinstance(data, str): data = data.encode(self.encoding) if isinstance(data, (bytes, bytearray)): self.body = data if hdrs.CONTENT_TYPE not in self.headers and hdrs.CONTENT_TYPE not in skip_auto_headers: self.headers[hdrs.CONTENT_TYPE] = "application/octet-stream" if hdrs.CONTENT_LENGTH not in self.headers and not self.chunked: self.headers[hdrs.CONTENT_LENGTH] = str(len(self.body)) elif isinstance(data, (asyncio.StreamReader, streams.DataQueue)): self.body = data elif asyncio.iscoroutine(data): self.body = data if hdrs.CONTENT_LENGTH not in self.headers and self.chunked is None: self.chunked = True elif isinstance(data, io.IOBase): assert not isinstance(data, io.StringIO), "attempt to send text data instead of binary" self.body = data if not self.chunked and isinstance(data, io.BytesIO): # Not chunking if content-length can be determined size = len(data.getbuffer()) self.headers[hdrs.CONTENT_LENGTH] = str(size) self.chunked = False elif not self.chunked and isinstance(data, io.BufferedReader): # Not chunking if content-length can be determined try: size = os.fstat(data.fileno()).st_size - data.tell() self.headers[hdrs.CONTENT_LENGTH] = str(size) self.chunked = False except OSError: # data.fileno() is not supported, e.g. # io.BufferedReader(io.BytesIO(b'data')) self.chunked = True else: self.chunked = True if hasattr(data, "mode"): if data.mode == "r": raise ValueError("file {!r} should be open in binary mode" "".format(data)) if ( hdrs.CONTENT_TYPE not in self.headers and hdrs.CONTENT_TYPE not in skip_auto_headers and hasattr(data, "name") ): mime = mimetypes.guess_type(data.name)[0] mime = "application/octet-stream" if mime is None else mime self.headers[hdrs.CONTENT_TYPE] = mime elif isinstance(data, MultipartWriter): self.body = data.serialize() self.headers.update(data.headers) self.chunked = self.chunked or 8192 else: if not isinstance(data, helpers.FormData): data = helpers.FormData(data) self.body = data(self.encoding) if hdrs.CONTENT_TYPE not in self.headers and hdrs.CONTENT_TYPE not in skip_auto_headers: self.headers[hdrs.CONTENT_TYPE] = data.content_type if data.is_multipart: self.chunked = self.chunked or 8192 else: if hdrs.CONTENT_LENGTH not in self.headers and not self.chunked: self.headers[hdrs.CONTENT_LENGTH] = str(len(self.body)) def update_transfer_encoding(self): """Analyze transfer-encoding header.""" te = self.headers.get(hdrs.TRANSFER_ENCODING, "").lower() if self.chunked: if hdrs.CONTENT_LENGTH in self.headers: del self.headers[hdrs.CONTENT_LENGTH] if "chunked" not in te: self.headers[hdrs.TRANSFER_ENCODING] = "chunked" self.chunked = self.chunked if type(self.chunked) is int else 8192 else: if "chunked" in te: self.chunked = 8192 else: self.chunked = None if hdrs.CONTENT_LENGTH not in self.headers: self.headers[hdrs.CONTENT_LENGTH] = str(len(self.body)) def update_expect_continue(self, expect=False): if expect: self.headers[hdrs.EXPECT] = "100-continue" elif self.headers.get(hdrs.EXPECT, "").lower() == "100-continue": expect = True if expect: self._continue = asyncio.Future(loop=self.loop) @asyncio.coroutine def write_bytes(self, request, reader): """Support coroutines that yields bytes objects.""" # 100 response if self._continue is not None: yield from self._continue try: if asyncio.iscoroutine(self.body): request.transport.set_tcp_nodelay(True) exc = None value = None stream = self.body while True: try: if exc is not None: result = stream.throw(exc) else: result = stream.send(value) except StopIteration as exc: if isinstance(exc.value, bytes): yield from request.write(exc.value, drain=True) break except: self.response.close() raise if isinstance(result, asyncio.Future): exc = None value = None try: value = yield result except Exception as err: exc = err elif isinstance(result, (bytes, bytearray)): yield from request.write(result, drain=True) value = None else: raise ValueError("Bytes object is expected, got: %s." % type(result)) elif isinstance(self.body, asyncio.StreamReader): request.transport.set_tcp_nodelay(True) chunk = yield from self.body.read(streams.DEFAULT_LIMIT) while chunk: yield from request.write(chunk, drain=True) chunk = yield from self.body.read(streams.DEFAULT_LIMIT) elif isinstance(self.body, streams.DataQueue): request.transport.set_tcp_nodelay(True) while True: try: chunk = yield from self.body.read() if chunk is EOF_MARKER: break yield from request.write(chunk, drain=True) except streams.EofStream: break elif isinstance(self.body, io.IOBase): chunk = self.body.read(self.chunked) while chunk: request.write(chunk) chunk = self.body.read(self.chunked) request.transport.set_tcp_nodelay(True) else: if isinstance(self.body, (bytes, bytearray)): self.body = (self.body,) for chunk in self.body: request.write(chunk) request.transport.set_tcp_nodelay(True) except Exception as exc: new_exc = aiohttp.ClientRequestError("Can not write request body for %s" % self.url) new_exc.__context__ = exc new_exc.__cause__ = exc reader.set_exception(new_exc) else: assert request.transport.tcp_nodelay try: ret = request.write_eof() # NB: in asyncio 3.4.1+ StreamWriter.drain() is coroutine # see bug #170 if asyncio.iscoroutine(ret) or isinstance(ret, asyncio.Future): yield from ret except Exception as exc: new_exc = aiohttp.ClientRequestError("Can not write request body for %s" % self.url) new_exc.__context__ = exc new_exc.__cause__ = exc reader.set_exception(new_exc) self._writer = None def send(self, writer, reader): writer.set_tcp_cork(True) request = aiohttp.Request(writer, self.method, self.path, self.version) if self.compress: request.add_compression_filter(self.compress) if self.chunked is not None: request.enable_chunked_encoding() request.add_chunking_filter(self.chunked) # set default content-type if ( self.method in self.POST_METHODS and hdrs.CONTENT_TYPE not in self.skip_auto_headers and hdrs.CONTENT_TYPE not in self.headers ): self.headers[hdrs.CONTENT_TYPE] = "application/octet-stream" for k, value in self.headers.items(): request.add_header(k, value) request.send_headers() self._writer = helpers.ensure_future(self.write_bytes(request, reader), loop=self.loop) self.response = self.response_class( self.method, self.url, self.host, writer=self._writer, continue100=self._continue ) self.response._post_init(self.loop) return self.response @asyncio.coroutine def close(self): if self._writer is not None: try: yield from self._writer finally: self._writer = None def terminate(self): if self._writer is not None: if hasattr(self.loop, "is_closed"): if not self.loop.is_closed(): self._writer.cancel() else: self._writer.cancel() self._writer = None
class MultipartWriter(object): """Multipart body writer.""" #: Body part reader class for non multipart/* content types. part_writer_cls = BodyPartWriter def __init__(self, subtype='mixed', boundary=None): boundary = boundary if boundary is not None else uuid.uuid4().hex try: boundary.encode('us-ascii') except UnicodeEncodeError: raise ValueError('boundary should contains ASCII only chars') self.headers = CIMultiDict() self.headers[CONTENT_TYPE] = 'multipart/{}; boundary="{}"'.format( subtype, boundary ) self.parts = [] def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): pass def __iter__(self): return iter(self.parts) def __len__(self): return len(self.parts) @property def boundary(self): *_, params = parse_mimetype(self.headers.get(CONTENT_TYPE)) return params['boundary'].encode('us-ascii') def append(self, obj, headers=None): """Adds a new body part to multipart writer.""" if isinstance(obj, self.part_writer_cls): if headers: obj.headers.update(headers) self.parts.append(obj) else: if not headers: headers = CIMultiDict() self.parts.append(self.part_writer_cls(obj, headers)) return self.parts[-1] def append_json(self, obj, headers=None): """Helper to append JSON part.""" if not headers: headers = CIMultiDict() headers[CONTENT_TYPE] = 'application/json' return self.append(obj, headers) def append_form(self, obj, headers=None): """Helper to append form urlencoded part.""" if not headers: headers = CIMultiDict() headers[CONTENT_TYPE] = 'application/x-www-form-urlencoded' assert isinstance(obj, (Sequence, Mapping)) return self.append(obj, headers) def serialize(self): """Yields multipart byte chunks.""" if not self.parts: yield b'' return for part in self.parts: yield b'--' + self.boundary + b'\r\n' yield from part.serialize() else: yield b'--' + self.boundary + b'--\r\n' yield b''
class MultipartWriter(object): """Multipart body writer.""" #: Body part reader class for non multipart/* content types. part_writer_cls = BodyPartWriter def __init__(self, subtype='mixed', boundary=None): boundary = boundary if boundary is not None else uuid.uuid4().hex try: boundary.encode('us-ascii') except UnicodeEncodeError: raise ValueError('boundary should contains ASCII only chars') self.headers = CIMultiDict() self.headers[CONTENT_TYPE] = 'multipart/{}; boundary="{}"'.format( subtype, boundary) self.parts = [] def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): pass def __iter__(self): return iter(self.parts) def __len__(self): return len(self.parts) @property def boundary(self): *_, params = parse_mimetype(self.headers.get(CONTENT_TYPE)) return params['boundary'].encode('us-ascii') def append(self, obj, headers=None): """Adds a new body part to multipart writer.""" if isinstance(obj, self.part_writer_cls): if headers: obj.headers.update(headers) self.parts.append(obj) else: if not headers: headers = CIMultiDict() self.parts.append(self.part_writer_cls(obj, headers)) return self.parts[-1] def append_json(self, obj, headers=None): """Helper to append JSON part.""" if not headers: headers = CIMultiDict() headers[CONTENT_TYPE] = 'application/json' return self.append(obj, headers) def append_form(self, obj, headers=None): """Helper to append form urlencoded part.""" if not headers: headers = CIMultiDict() headers[CONTENT_TYPE] = 'application/x-www-form-urlencoded' assert isinstance(obj, (Sequence, Mapping)) return self.append(obj, headers) def serialize(self): """Yields multipart byte chunks.""" if not self.parts: yield b'' return for part in self.parts: yield b'--' + self.boundary + b'\r\n' yield from part.serialize() else: yield b'--' + self.boundary + b'--\r\n' yield b''
def parse_headers(self, lines): """Parses RFC 5322 headers from a stream. Line continuations are supported. Returns list of header name and value pairs. Header name is in upper case. """ headers = CIMultiDict() raw_headers = [] lines_idx = 1 line = lines[1] line_count = len(lines) while line: header_length = len(line) # Parse initial header name : value pair. try: bname, bvalue = line.split(b':', 1) except ValueError: raise InvalidHeader(line) from None bname = bname.strip(b' \t') if HDRRE.search(bname): raise InvalidHeader(bname) # next line lines_idx += 1 line = lines[lines_idx] # consume continuation lines continuation = line and line[0] in (32, 9) # (' ', '\t') if continuation: bvalue = [bvalue] while continuation: header_length += len(line) if header_length > self.max_field_size: raise LineTooLong( 'request header field {}'.format( bname.decode("utf8", "xmlcharrefreplace")), self.max_field_size) bvalue.append(line) # next line lines_idx += 1 if lines_idx < line_count: line = lines[lines_idx] if line: continuation = line[0] in (32, 9) # (' ', '\t') else: line = b'' break bvalue = b''.join(bvalue) else: if header_length > self.max_field_size: raise LineTooLong( 'request header field {}'.format( bname.decode("utf8", "xmlcharrefreplace")), self.max_field_size) bvalue = bvalue.strip() name = bname.decode('utf-8', 'surrogateescape') value = bvalue.decode('utf-8', 'surrogateescape') headers.add(name, value) raw_headers.append((bname, bvalue)) close_conn = None encoding = None upgrade = False chunked = False raw_headers = tuple(raw_headers) # keep-alive conn = headers.get(hdrs.CONNECTION) if conn: v = conn.lower() if v == 'close': close_conn = True elif v == 'keep-alive': close_conn = False elif v == 'upgrade': upgrade = True # encoding enc = headers.get(hdrs.CONTENT_ENCODING) if enc: enc = enc.lower() if enc in ('gzip', 'deflate'): encoding = enc # chunking te = headers.get(hdrs.TRANSFER_ENCODING) if te and 'chunked' in te.lower(): chunked = True return headers, raw_headers, close_conn, encoding, upgrade, chunked
class BaseRequest(object): charset = 'utf-8' encoding_errors = 'replace' def __init__(self, app, environ, headers=None): # Request state self.app = app self.environ = environ self.headers = CIMultiDict(headers or {}) # Response state self._cookies = None self._cached_data = None self._parsed_content_type = None self.disconnected = False self.response_complete = False self.match_headers() @property def META(self): return dict(self.headers, **self.environ) def match_headers(self): for value in self.environ["headers"]: self.headers[value[0]] = value[1] @property def content_type(self): """Like :attr:`content_type`, but without parameters (eg, without charset, type etc.) and always lowercase. For example if the content type is ``text/HTML; charset=utf-8`` the mimetype would be ``'text/html'``. """ if self.headers.get('content-type'): self._parsed_content_type = parse_options_header( self.headers['content-type']) else: self._parsed_content_type = parse_options_header( self.headers.get('accept', '')) return self._parsed_content_type[0].lower() @cached_property def cookies(self): if self._cookies is None: cookies = {} cookie_header = self.headers.get("cookie") if cookie_header: cookie = http.cookies.SimpleCookie() cookie.load(cookie_header) for key, morsel in cookie.items(): cookies[key] = morsel.value self._cookies = cookies return self._cookies def get_host(self): if self.headers.get(self.app.config['FORWARDED_FOR_HEADER']): rv = self.headers[self.app.config['FORWARDED_FOR_HEADER']].split( ',', 1)[0].strip() elif self.headers.get(self.app.config['HTTP_HOST']): rv = self.headers[self.app.config['HTTP_HOST']] elif self.headers.get('host'): rv = self.headers['host'].split(':', 1)[0].strip() else: rv = self.environ.get('host') return rv @cached_property def version(self): """ The http request version. """ return self.environ.get("http_version", "1.1") @cached_property def path(self): """ Requested path as unicode. This works a bit like the regular path info in the WSGI environment but will always include a leading slash, even if the URL root is accessed. """ raw_path = self.environ.get('path') or '' return '/' + raw_path.lstrip('/') @property def url_charset(self): """ The charset that is assumed for URLs. Defaults to the value of :attr:`charset`. .. versionadded:: 0.6 """ return self.charset @cached_property def full_path(self): """ Requested path as unicode, including the query string. """ return self.path + u'?' + to_unicode(self.query_string, self.url_charset) @cached_property def body(self): """ The reconstructed current URL as IRI. """ return self.environ.get("body") @cached_property def data(self): return self.get_data() @cached_property def form(self): return self.get_data(parse_form_data=True) @cached_property def url(self): """ The reconstructed current URL as IRI. """ return self.environ.get("url") @cached_property def base_url(self): """ Like :attr:`url` but without the querystring """ return get_request_url(self, strip_querystring=True) @cached_property def root_url(self): """ The full URL root (with hostname), this is the application root as IRI. """ return get_request_url(self, root_only=True) @cached_property def ip(self): """ Just the host including the ip if available. """ return self.environ.get("ip") @cached_property def host(self): """ Just the host including the host if available. """ return self.get_host() @cached_property def port(self): """ Just the host including the port if available. """ return self.environ.get("port") @cached_property def query_string(self): """ Just the request query string. """ return self.environ.get("query_string") @cached_property def args(self): """ Just the request query string. """ return dict(parse.parse_qsl(parse.urlsplit(self.full_path).query)) @cached_property def method(self): """ Just the request method string. """ return self.environ.get("method") @cached_property def scheme(self): """ Just the request scheme string. """ return self.environ.get("scheme") @cached_property def server(self): """ Just the request server string. """ return self.environ.get("server") @cached_property def client(self): """ Just the request client string. """ return self.environ.get("client") @cached_property def remote_addr(self): return self.headers.get(self.app.config['REAL_IP_HEADER']) \ or self.headers.get(self.app.config['FORWARDED_FOR_HEADER']) \ or self.headers.get(self.app.config['REMOTE_ADDR']) @cached_property def root_path(self): """ Just the request root_path string. """ return self.environ.get("root_path") @cached_property def transport(self): """ Just the request transport string. """ return self.environ.get("transport") def get_data(self, cache=True, as_text=False, parse_form_data=False): rv = getattr(self, '_cached_data', None) if rv is None: rv = self.body if cache: self._cached_data = rv if as_text: rv = rv.decode(self.charset, self.encoding_errors) return rv