def test_single_forwarded_header_injection2() -> None:
header = "very bad syntax, for=_real"
req = make_mocked_request("GET", "/", headers=CIMultiDict({"Forwarded": header}))
assert len(req.forwarded) == 2
assert "for" not in req.forwarded[0]
assert req.forwarded[1]["for"] == "_real"
async def _ws_connect(
self,
url: StrOrURL,
*,
method: str = hdrs.METH_GET,
protocols: Iterable[str] = (),
timeout: float = 10.0,
receive_timeout: Optional[float] = None,
autoclose: bool = True,
autoping: bool = True,
heartbeat: Optional[float] = None,
auth: Optional[BasicAuth] = None,
origin: Optional[str] = None,
headers: Optional[LooseHeaders] = None,
proxy: Optional[StrOrURL] = None,
proxy_auth: Optional[BasicAuth] = None,
ssl: Union[SSLContext, bool, None, Fingerprint] = None,
verify_ssl: Optional[bool] = None,
fingerprint: Optional[bytes] = None,
ssl_context: Optional[SSLContext] = None,
proxy_headers: Optional[LooseHeaders] = None,
compress: int = 0,
max_msg_size: int = 4 * 1024 * 1024) -> ClientWebSocketResponse:
if headers is None:
real_headers = CIMultiDict() # type: CIMultiDict[str]
else:
real_headers = CIMultiDict(headers)
default_headers = {
hdrs.UPGRADE: hdrs.WEBSOCKET,
hdrs.CONNECTION: hdrs.UPGRADE,
hdrs.SEC_WEBSOCKET_VERSION: '13',
}
for key, value in default_headers.items():
real_headers.setdefault(key, value)
sec_key = base64.b64encode(os.urandom(16))
real_headers[hdrs.SEC_WEBSOCKET_KEY] = sec_key.decode()
if protocols:
real_headers[hdrs.SEC_WEBSOCKET_PROTOCOL] = ','.join(protocols)
if origin is not None:
real_headers[hdrs.ORIGIN] = origin
if compress:
extstr = ws_ext_gen(compress=compress)
real_headers[hdrs.SEC_WEBSOCKET_EXTENSIONS] = extstr
ssl = _merge_ssl_params(ssl, verify_ssl, ssl_context, fingerprint)
# send request
resp = await self.request(method,
url,
headers=real_headers,
read_until_eof=False,
auth=auth,
proxy=proxy,
proxy_auth=proxy_auth,
ssl=ssl,
proxy_headers=proxy_headers)
try:
# check handshake
if resp.status != 101:
raise WSServerHandshakeError(resp.request_info,
resp.history,
message='Invalid response status',
status=resp.status,
headers=resp.headers)
if resp.headers.get(hdrs.UPGRADE, '').lower() != 'websocket':
raise WSServerHandshakeError(resp.request_info,
resp.history,
message='Invalid upgrade header',
status=resp.status,
headers=resp.headers)
if resp.headers.get(hdrs.CONNECTION, '').lower() != 'upgrade':
raise WSServerHandshakeError(
resp.request_info,
resp.history,
message='Invalid connection header',
status=resp.status,
headers=resp.headers)
# key calculation
key = resp.headers.get(hdrs.SEC_WEBSOCKET_ACCEPT, '')
match = base64.b64encode(hashlib.sha1(sec_key +
WS_KEY).digest()).decode()
if key != match:
raise WSServerHandshakeError(
resp.request_info,
resp.history,
message='Invalid challenge response',
status=resp.status,
headers=resp.headers)
# websocket protocol
protocol = None
if protocols and hdrs.SEC_WEBSOCKET_PROTOCOL in resp.headers:
resp_protocols = [
proto.strip() for proto in resp.headers[
hdrs.SEC_WEBSOCKET_PROTOCOL].split(',')
]
for proto in resp_protocols:
if proto in protocols:
protocol = proto
break
# websocket compress
notakeover = False
if compress:
compress_hdrs = resp.headers.get(hdrs.SEC_WEBSOCKET_EXTENSIONS)
if compress_hdrs:
try:
compress, notakeover = ws_ext_parse(compress_hdrs)
except WSHandshakeError as exc:
raise WSServerHandshakeError(resp.request_info,
resp.history,
message=exc.args[0],
status=resp.status,
headers=resp.headers)
else:
compress = 0
notakeover = False
conn = resp.connection
assert conn is not None
proto = conn.protocol
assert proto is not None
transport = conn.transport
assert transport is not None
reader = FlowControlDataQueue(
proto, limit=2**16, loop=self._loop
) # type: FlowControlDataQueue[WSMessage] # noqa
proto.set_parser(WebSocketReader(reader, max_msg_size), reader)
writer = WebSocketWriter(proto,
transport,
use_mask=True,
compress=compress,
notakeover=notakeover)
except BaseException:
resp.close()
raise
else:
return self._ws_response_class(reader,
writer,
protocol,
resp,
timeout,
autoclose,
autoping,
self._loop,
receive_timeout=receive_timeout,
heartbeat=heartbeat,
compress=compress,
client_notakeover=notakeover)
def __init__(self,
*,
body: Any = None,
status: int = 200,
reason: Optional[str] = None,
text: Optional[str] = None,
headers: Optional[LooseHeaders] = None,
content_type: Optional[str] = None,
charset: Optional[str] = None,
zlib_executor_size: Optional[int] = None,
zlib_executor: Executor = None) -> None:
if body is not None and text is not None:
raise ValueError("body and text are not allowed together")
if headers is None:
real_headers = CIMultiDict() # type: CIMultiDict[str]
elif not isinstance(headers, CIMultiDict):
real_headers = CIMultiDict(headers)
else:
real_headers = headers # = cast('CIMultiDict[str]', headers)
if content_type is not None and "charset" in content_type:
raise ValueError("charset must not be in content_type " "argument")
if text is not None:
if hdrs.CONTENT_TYPE in real_headers:
if content_type or charset:
raise ValueError("passing both Content-Type header and "
"content_type or charset params "
"is forbidden")
else:
# fast path for filling headers
if not isinstance(text, str):
raise TypeError("text argument must be str (%r)" %
type(text))
if content_type is None:
content_type = 'text/plain'
if charset is None:
charset = 'utf-8'
real_headers[hdrs.CONTENT_TYPE] = (content_type +
'; charset=' + charset)
body = text.encode(charset)
text = None
else:
if hdrs.CONTENT_TYPE in real_headers:
if content_type is not None or charset is not None:
raise ValueError("passing both Content-Type header and "
"content_type or charset params "
"is forbidden")
else:
if content_type is not None:
if charset is not None:
content_type += '; charset=' + charset
real_headers[hdrs.CONTENT_TYPE] = content_type
super().__init__(status=status, reason=reason, headers=real_headers)
if text is not None:
self.text = text
else:
self.body = body
self._compressed_body = None # type: Optional[bytes]
self._zlib_executor_size = zlib_executor_size
self._zlib_executor = zlib_executor
def test_merge_headers_with_list_of_tuples(create_session):
session = create_session(headers={"h1": "header1", "h2": "header2"})
headers = session._prepare_headers([("h1", "h1")])
assert isinstance(headers, CIMultiDict)
assert headers == CIMultiDict([("h2", "header2"), ("h1", "h1")])
def test_init_headers_list_of_tuples_with_duplicates(create_session):
session = create_session(
headers=[("h1", "header11"), ("h2", "header21"), ("h1", "header12")])
assert (session._default_headers == CIMultiDict([("H1", "header11"),
("H2", "header21"),
("H1", "header12")]))
def __init__(self, *, connector=None, loop=None, cookies=None,
headers=None, skip_auto_headers=None,
auth=None, json_serialize=json.dumps,
request_class=ClientRequest, response_class=ClientResponse,
ws_response_class=ClientWebSocketResponse,
version=http.HttpVersion11,
cookie_jar=None, connector_owner=True, raise_for_status=False,
read_timeout=sentinel, conn_timeout=None,
auto_decompress=True, trust_env=False,
trace_configs=None):
implicit_loop = False
if loop is None:
if connector is not None:
loop = connector._loop
else:
implicit_loop = True
loop = asyncio.get_event_loop()
if connector is None:
connector = TCPConnector(loop=loop)
if connector._loop is not loop:
raise RuntimeError(
"Session and connector has to use same event loop")
self._loop = loop
if loop.get_debug():
self._source_traceback = traceback.extract_stack(sys._getframe(1))
if implicit_loop and not loop.is_running():
warnings.warn("Creating a client session outside of coroutine is "
"a very dangerous idea",
stacklevel=2)
context = {'client_session': self,
'message': 'Creating a client session outside '
'of coroutine'}
if self._source_traceback is not None:
context['source_traceback'] = self._source_traceback
loop.call_exception_handler(context)
if cookie_jar is None:
cookie_jar = CookieJar(loop=loop)
self._cookie_jar = cookie_jar
if cookies is not None:
self._cookie_jar.update_cookies(cookies)
self._connector = connector
self._connector_owner = connector_owner
self._default_auth = auth
self._version = version
self._json_serialize = json_serialize
self._read_timeout = (read_timeout if read_timeout is not sentinel
else DEFAULT_TIMEOUT)
self._conn_timeout = conn_timeout
self._raise_for_status = raise_for_status
self._auto_decompress = auto_decompress
self._trust_env = trust_env
# Convert to list of tuples
if headers:
headers = CIMultiDict(headers)
else:
headers = CIMultiDict()
self._default_headers = headers
if skip_auto_headers is not None:
self._skip_auto_headers = frozenset([istr(i)
for i in skip_auto_headers])
else:
self._skip_auto_headers = frozenset()
self._request_class = request_class
self._response_class = response_class
self._ws_response_class = ws_response_class
self._trace_configs = trace_configs or []
for trace_config in self._trace_configs:
trace_config.freeze()
async def send(self, request, **config):
response = await super(AiohttpTestTransport, self).send(request, **config)
if not isinstance(response.headers, CIMultiDictProxy):
response.headers = CIMultiDictProxy(CIMultiDict(response.internal_response.headers))
response.content_type = response.headers.get("content-type")
return response
def make_request(app, method, path, headers=CIMultiDict()):
message = RawRequestMessage(method, path, HttpVersion11, headers,
[(k.encode('utf-8'), v.encode('utf-8'))
for k, v in headers.items()],
False, False)
return request_from_message(message, app)
def append_json(self, obj, headers=None):
"""Helper to append JSON part."""
if not headers:
headers = CIMultiDict()
headers[CONTENT_TYPE] = 'application/json'
return self.append(obj, headers)
def test_clone_headers_dict() -> None:
req = make_mocked_request("GET", "/path", headers={"A": "B"})
req2 = req.clone(headers={"B": "C"})
assert req2.headers == CIMultiDict({"B": "C"})
assert req2.raw_headers == ((b"B", b"C"),)
def make_mocked_request(method,
path,
headers=None,
*,
version=HttpVersion(1, 1),
closing=False,
app=None,
writer=sentinel,
payload_writer=sentinel,
protocol=sentinel,
transport=sentinel,
payload=sentinel,
sslcontext=None,
secure_proxy_ssl_header=None,
client_max_size=1024**2):
"""Creates mocked web.Request testing purposes.
Useful in unit tests, when spinning full web server is overkill or
specific conditions and errors are hard to trigger.
"""
task = mock.Mock()
loop = mock.Mock()
loop.create_future.return_value = ()
if version < HttpVersion(1, 1):
closing = True
if headers:
headers = CIMultiDict(headers)
raw_hdrs = tuple(
(k.encode('utf-8'), v.encode('utf-8')) for k, v in headers.items())
else:
headers = CIMultiDict()
raw_hdrs = ()
chunked = 'chunked' in headers.get(hdrs.TRANSFER_ENCODING, '').lower()
message = RawRequestMessage(method, path, version, headers, raw_hdrs,
closing, False, False, chunked, URL(path))
if app is None:
app = _create_app_mock()
if protocol is sentinel:
protocol = mock.Mock()
if transport is sentinel:
transport = _create_transport(sslcontext)
if writer is sentinel:
writer = mock.Mock()
writer.transport = transport
if payload_writer is sentinel:
payload_writer = mock.Mock()
payload_writer.write_eof.side_effect = noop
payload_writer.drain.side_effect = noop
protocol.transport = transport
protocol.writer = writer
if payload is sentinel:
payload = mock.Mock()
time_service = mock.Mock()
time_service.time.return_value = 12345
time_service.strtime.return_value = "Tue, 15 Nov 1994 08:12:31 GMT"
@contextmanager
def timeout(*args, **kw):
yield
time_service.timeout = mock.Mock()
time_service.timeout.side_effect = timeout
req = Request(message,
payload,
protocol,
payload_writer,
time_service,
task,
secure_proxy_ssl_header=secure_proxy_ssl_header,
client_max_size=client_max_size)
match_info = UrlMappingMatchInfo({}, mock.Mock())
match_info.add_app(app)
req._match_info = match_info
return req
def test_raw_headers() -> None:
req = make_mocked_request("GET", "/", headers=CIMultiDict({"X-HEADER": "aaa"}))
assert req.raw_headers == ((b"X-HEADER", b"aaa"),)
def test_host_by_host_header() -> None:
req = make_mocked_request("GET", "/", headers=CIMultiDict({"Host": "example.com"}))
assert req.host == "example.com"
def test_single_forwarded_header_long_quoted_string() -> None:
header = 'for="' + "\\\\" * 5000 + '"'
req = make_mocked_request("GET", "/", headers=CIMultiDict({"Forwarded": header}))
assert req.forwarded[0]["for"] == "\\" * 5000
def parse_headers(self, lines):
"""Parses RFC 5322 headers from a stream.
Line continuations are supported. Returns list of header name
and value pairs. Header name is in upper case.
"""
headers = CIMultiDict()
raw_headers = []
lines_idx = 1
line = lines[1]
while line:
header_length = len(line)
# Parse initial header name : value pair.
try:
bname, bvalue = line.split(b':', 1)
except ValueError:
raise errors.InvalidHeader(line) from None
bname = bname.strip(b' \t').upper()
if HDRRE.search(bname):
raise errors.InvalidHeader(bname)
# next line
lines_idx += 1
line = lines[lines_idx]
# consume continuation lines
continuation = line and line[0] in (32, 9) # (' ', '\t')
if continuation:
bvalue = [bvalue]
while continuation:
header_length += len(line)
if header_length > self.max_field_size:
raise errors.LineTooLong(
'request header field {}'.format(
bname.decode("utf8", "xmlcharrefreplace")),
self.max_field_size)
bvalue.append(line)
# next line
lines_idx += 1
line = lines[lines_idx]
continuation = line[0] in (32, 9) # (' ', '\t')
bvalue = b'\r\n'.join(bvalue)
else:
if header_length > self.max_field_size:
raise errors.LineTooLong(
'request header field {}'.format(
bname.decode("utf8", "xmlcharrefreplace")),
self.max_field_size)
bvalue = bvalue.strip()
name = istr(bname.decode('utf-8', 'surrogateescape'))
value = bvalue.decode('utf-8', 'surrogateescape')
headers.add(name, value)
raw_headers.append((bname, bvalue))
close_conn = None
encoding = None
upgrade = False
chunked = False
# keep-alive
conn = headers.get(hdrs.CONNECTION)
if conn:
v = conn.lower()
if v == 'close':
close_conn = True
elif v == 'keep-alive':
close_conn = False
elif v == 'upgrade':
upgrade = True
# encoding
enc = headers.get(hdrs.CONTENT_ENCODING)
if enc:
enc = enc.lower()
if enc in ('gzip', 'deflate'):
encoding = enc
# chunking
te = headers.get(hdrs.TRANSFER_ENCODING)
if te and 'chunked' in te.lower():
chunked = True
return headers, raw_headers, close_conn, encoding, upgrade, chunked
def append_json(self, obj, headers=None):
"""Helper to append JSON part."""
if headers is None:
headers = CIMultiDict()
return self.append_payload(JsonPayload(obj, headers=headers))
async def _ws_connect(self, url, *,
protocols=(),
timeout=10.0,
receive_timeout=None,
autoclose=True,
autoping=True,
heartbeat=None,
auth=None,
origin=None,
headers=None,
proxy=None,
proxy_auth=None,
ssl=None,
verify_ssl=None,
fingerprint=None,
ssl_context=None,
proxy_headers=None,
compress=0):
if headers is None:
headers = CIMultiDict()
default_headers = {
hdrs.UPGRADE: hdrs.WEBSOCKET,
hdrs.CONNECTION: hdrs.UPGRADE,
hdrs.SEC_WEBSOCKET_VERSION: '13',
}
for key, value in default_headers.items():
if key not in headers:
headers[key] = value
sec_key = base64.b64encode(os.urandom(16))
headers[hdrs.SEC_WEBSOCKET_KEY] = sec_key.decode()
if protocols:
headers[hdrs.SEC_WEBSOCKET_PROTOCOL] = ','.join(protocols)
if origin is not None:
headers[hdrs.ORIGIN] = origin
if compress:
extstr = ws_ext_gen(compress=compress)
headers[hdrs.SEC_WEBSOCKET_EXTENSIONS] = extstr
ssl = _merge_ssl_params(ssl, verify_ssl, ssl_context, fingerprint)
# send request
resp = await self.get(url, headers=headers,
read_until_eof=False,
auth=auth,
proxy=proxy,
proxy_auth=proxy_auth,
ssl=ssl,
proxy_headers=proxy_headers)
try:
# check handshake
if resp.status != 101:
raise WSServerHandshakeError(
resp.request_info,
resp.history,
message='Invalid response status',
code=resp.status,
headers=resp.headers)
if resp.headers.get(hdrs.UPGRADE, '').lower() != 'websocket':
raise WSServerHandshakeError(
resp.request_info,
resp.history,
message='Invalid upgrade header',
code=resp.status,
headers=resp.headers)
if resp.headers.get(hdrs.CONNECTION, '').lower() != 'upgrade':
raise WSServerHandshakeError(
resp.request_info,
resp.history,
message='Invalid connection header',
code=resp.status,
headers=resp.headers)
# key calculation
key = resp.headers.get(hdrs.SEC_WEBSOCKET_ACCEPT, '')
match = base64.b64encode(
hashlib.sha1(sec_key + WS_KEY).digest()).decode()
if key != match:
raise WSServerHandshakeError(
resp.request_info,
resp.history,
message='Invalid challenge response',
code=resp.status,
headers=resp.headers)
# websocket protocol
protocol = None
if protocols and hdrs.SEC_WEBSOCKET_PROTOCOL in resp.headers:
resp_protocols = [
proto.strip() for proto in
resp.headers[hdrs.SEC_WEBSOCKET_PROTOCOL].split(',')]
for proto in resp_protocols:
if proto in protocols:
protocol = proto
break
# websocket compress
notakeover = False
if compress:
compress_hdrs = resp.headers.get(hdrs.SEC_WEBSOCKET_EXTENSIONS)
if compress_hdrs:
try:
compress, notakeover = ws_ext_parse(compress_hdrs)
except WSHandshakeError as exc:
raise WSServerHandshakeError(
resp.request_info,
resp.history,
message=exc.args[0],
code=resp.status,
headers=resp.headers)
else:
compress = 0
notakeover = False
proto = resp.connection.protocol
transport = resp.connection.transport
reader = FlowControlDataQueue(
proto, limit=2 ** 16, loop=self._loop)
proto.set_parser(WebSocketReader(reader), reader)
tcp_nodelay(transport, True)
writer = WebSocketWriter(
proto, transport, use_mask=True,
compress=compress, notakeover=notakeover)
except BaseException:
resp.close()
raise
else:
return self._ws_response_class(reader,
writer,
protocol,
resp,
timeout,
autoclose,
autoping,
self._loop,
receive_timeout=receive_timeout,
heartbeat=heartbeat,
compress=compress,
client_notakeover=notakeover)
def make_mocked_request(method,
path,
headers=None,
*,
version=HttpVersion(1, 1),
closing=False,
app=None,
reader=sentinel,
writer=sentinel,
transport=sentinel,
payload=sentinel,
sslcontext=None,
secure_proxy_ssl_header=None):
"""Creates mocked web.Request testing purposes.
Useful in unit tests, when spinning full web server is overkill or
specific conditions and errors are hard to trigger.
:param method: str, that represents HTTP method, like; GET, POST.
:type method: str
:param path: str, The URL including *PATH INFO* without the host or scheme
:type path: str
:param headers: mapping containing the headers. Can be anything accepted
by the multidict.CIMultiDict constructor.
:type headers: dict, multidict.CIMultiDict, list of pairs
:param version: namedtuple with encoded HTTP version
:type version: aiohttp.protocol.HttpVersion
:param closing: flag indicates that connection should be closed after
response.
:type closing: bool
:param app: the aiohttp.web application attached for fake request
:type app: aiohttp.web.Application
:param reader: object for storing and managing incoming data
:type reader: aiohttp.parsers.StreamParser
:param writer: object for managing outcoming data
:type wirter: aiohttp.parsers.StreamWriter
:param transport: asyncio transport instance
:type transport: asyncio.transports.Transport
:param payload: raw payload reader object
:type payload: aiohttp.streams.FlowControlStreamReader
:param sslcontext: ssl.SSLContext object, for HTTPS connection
:type sslcontext: ssl.SSLContext
:param secure_proxy_ssl_header: A tuple representing a HTTP header/value
combination that signifies a request is secure.
:type secure_proxy_ssl_header: tuple
"""
if version < HttpVersion(1, 1):
closing = True
if headers:
hdrs = CIMultiDict(headers)
raw_hdrs = [(k.encode('utf-8'), v.encode('utf-8'))
for k, v in headers.items()]
else:
hdrs = CIMultiDict()
raw_hdrs = []
message = RawRequestMessage(method, path, version, hdrs, raw_hdrs, closing,
False)
if app is None:
app = _create_app_mock()
if reader is sentinel:
reader = mock.Mock()
if writer is sentinel:
writer = mock.Mock()
if transport is sentinel:
transport = _create_transport(sslcontext)
if payload is sentinel:
payload = mock.Mock()
req = Request(app,
message,
payload,
transport,
reader,
writer,
secure_proxy_ssl_header=secure_proxy_ssl_header)
return req
def _make_headers(headers: t.Optional[LooseHeaders]) -> CIMultiDict:
headers = CIMultiDict(headers or {})
return headers
from multidict import CIMultiDict
import asyncio
import requests
import aiohttp
import json
import re
import hashlib
DEFAULT_HEADERS = CIMultiDict({
"Accept": "application/json, text/plain, */*",
"Cookie": "xm_sg_tk=5d6900addb98a4cf409ae66a8f035d38_1589250735020; xm_sg_tk.sig=04dDg_DG3Q9hb4BjNc6PaCBm9FAjrwEofChRb4gsuxE",
"Referer": "https://www.xiami.com/collect/1132898094",
"User-Agent": ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) "
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36"),
})
def parse_index():
bs_url = 'https://www.xiami.com/api/collect/getCollectStaticUrl'
_q = json.dumps({"listId":1132898094}).replace(' ', '')
print(_q)
_s = get_sign(_q)
print(_s)
params = {
'_q': _q,
'_s': _s
}
res = requests.get(url=bs_url, params=params, headers=DEFAULT_HEADERS, verify=False).json()
print(res)
def make_mocked_request(method,
path,
headers=None,
*,
match_info=sentinel,
version=HttpVersion(1, 1),
closing=False,
app=None,
writer=sentinel,
protocol=sentinel,
transport=sentinel,
payload=sentinel,
sslcontext=None,
client_max_size=1024**2,
loop=...):
"""Creates mocked web.Request testing purposes.
Useful in unit tests, when spinning full web server is overkill or
specific conditions and errors are hard to trigger.
"""
task = mock.Mock()
if loop is ...:
loop = mock.Mock()
loop.create_future.return_value = ()
if version < HttpVersion(1, 1):
closing = True
if headers:
headers = CIMultiDict(headers)
raw_hdrs = tuple(
(k.encode('utf-8'), v.encode('utf-8')) for k, v in headers.items())
else:
headers = CIMultiDict()
raw_hdrs = ()
chunked = 'chunked' in headers.get(hdrs.TRANSFER_ENCODING, '').lower()
message = RawRequestMessage(method, path, version, headers, raw_hdrs,
closing, False, False, chunked, URL(path))
if app is None:
app = _create_app_mock()
if protocol is sentinel:
protocol = mock.Mock()
if transport is sentinel:
transport = _create_transport(sslcontext)
if writer is sentinel:
writer = mock.Mock()
writer.write_headers = make_mocked_coro(None)
writer.write = make_mocked_coro(None)
writer.write_eof = make_mocked_coro(None)
writer.drain = make_mocked_coro(None)
writer.transport = transport
protocol.transport = transport
protocol.writer = writer
if payload is sentinel:
payload = mock.Mock()
req = Request(message,
payload,
protocol,
writer,
task,
loop,
client_max_size=client_max_size)
match_info = UrlMappingMatchInfo(
{} if match_info is sentinel else match_info, mock.Mock())
match_info.add_app(app)
req._match_info = match_info
return req
def parse_headers(
self, lines: List[bytes]
) -> Tuple["CIMultiDictProxy[str]", RawHeaders]:
headers = CIMultiDict() # type: CIMultiDict[str]
raw_headers = []
lines_idx = 1
line = lines[1]
line_count = len(lines)
while line:
# Parse initial header name : value pair.
try:
bname, bvalue = line.split(b":", 1)
except ValueError:
raise InvalidHeader(line) from None
bname = bname.strip(b" \t")
bvalue = bvalue.lstrip()
if HDRRE.search(bname):
raise InvalidHeader(bname)
if len(bname) > self.max_field_size:
raise LineTooLong(
"request header name {}".format(
bname.decode("utf8", "xmlcharrefreplace")
),
str(self.max_field_size),
str(len(bname)),
)
header_length = len(bvalue)
# next line
lines_idx += 1
line = lines[lines_idx]
# consume continuation lines
continuation = line and line[0] in (32, 9) # (' ', '\t')
if continuation:
bvalue_lst = [bvalue]
while continuation:
header_length += len(line)
if header_length > self.max_field_size:
raise LineTooLong(
"request header field {}".format(
bname.decode("utf8", "xmlcharrefreplace")
),
str(self.max_field_size),
str(header_length),
)
bvalue_lst.append(line)
# next line
lines_idx += 1
if lines_idx < line_count:
line = lines[lines_idx]
if line:
continuation = line[0] in (32, 9) # (' ', '\t')
else:
line = b""
break
bvalue = b"".join(bvalue_lst)
else:
if header_length > self.max_field_size:
raise LineTooLong(
"request header field {}".format(
bname.decode("utf8", "xmlcharrefreplace")
),
str(self.max_field_size),
str(header_length),
)
bvalue = bvalue.strip()
name = bname.decode("utf-8", "surrogateescape")
value = bvalue.decode("utf-8", "surrogateescape")
headers.add(name, value)
raw_headers.append((bname, bvalue))
return (CIMultiDictProxy(headers), tuple(raw_headers))
def test_init_headers_list_of_tuples(create_session):
session = create_session(
headers=[("h1", "header1"), ("h2", "header2"), ("h3", "header3")])
assert (session._default_headers == CIMultiDict([("h1", "header1"),
("h2", "header2"),
("h3", "header3")]))
def test_body_in_ctor_with_content_type_header_multidict():
headers = CIMultiDict({'Content-Type': 'text/html; charset=koi8-r'})
resp = Response(body='текст'.encode('koi8-r'), headers=headers)
assert 'текст'.encode('koi8-r') == resp.body
assert 'text/html' == resp.content_type
assert 'koi8-r' == resp.charset
def __init__(
self,
*,
connector: Optional[BaseConnector] = None,
loop: Optional[asyncio.AbstractEventLoop] = None,
cookies: Optional[LooseCookies] = None,
headers: Optional[LooseHeaders] = None,
skip_auto_headers: Optional[Iterable[str]] = None,
auth: Optional[BasicAuth] = None,
json_serialize: JSONEncoder = json.dumps,
request_class: Type[ClientRequest] = ClientRequest,
response_class: Type[ClientResponse] = ClientResponse,
ws_response_class: Type[
ClientWebSocketResponse] = ClientWebSocketResponse, # noqa
version: HttpVersion = http.HttpVersion11,
cookie_jar: Optional[AbstractCookieJar] = None,
connector_owner: bool = True,
raise_for_status: bool = False,
read_timeout: Union[float, object] = sentinel,
conn_timeout: Optional[float] = None,
timeout: Union[object, ClientTimeout] = sentinel,
auto_decompress: bool = True,
trust_env: bool = False,
requote_redirect_url: bool = True,
trace_configs: Optional[List[TraceConfig]] = None) -> None:
if loop is None:
if connector is not None:
loop = connector._loop
loop = get_running_loop(loop)
if connector is None:
connector = TCPConnector(loop=loop)
if connector._loop is not loop:
raise RuntimeError(
"Session and connector has to use same event loop")
self._loop = loop
if loop.get_debug():
self._source_traceback = traceback.extract_stack(sys._getframe(1))
if cookie_jar is None:
cookie_jar = CookieJar(loop=loop)
self._cookie_jar = cookie_jar
if cookies is not None:
self._cookie_jar.update_cookies(cookies)
self._connector = connector # type: BaseConnector
self._connector_owner = connector_owner
self._default_auth = auth
self._version = version
self._json_serialize = json_serialize
if timeout is sentinel:
self._timeout = DEFAULT_TIMEOUT
if read_timeout is not sentinel:
warnings.warn(
"read_timeout is deprecated, "
"use timeout argument instead",
DeprecationWarning,
stacklevel=2)
self._timeout = attr.evolve(self._timeout, total=read_timeout)
if conn_timeout is not None:
self._timeout = attr.evolve(self._timeout,
connect=conn_timeout)
warnings.warn(
"conn_timeout is deprecated, "
"use timeout argument instead",
DeprecationWarning,
stacklevel=2)
else:
self._timeout = timeout # type: ignore
if read_timeout is not sentinel:
raise ValueError("read_timeout and timeout parameters "
"conflict, please setup "
"timeout.read")
if conn_timeout is not None:
raise ValueError("conn_timeout and timeout parameters "
"conflict, please setup "
"timeout.connect")
self._raise_for_status = raise_for_status
self._auto_decompress = auto_decompress
self._trust_env = trust_env
self._requote_redirect_url = requote_redirect_url
# Convert to list of tuples
if headers:
headers = CIMultiDict(headers)
else:
headers = CIMultiDict()
self._default_headers = headers
if skip_auto_headers is not None:
self._skip_auto_headers = frozenset(
[istr(i) for i in skip_auto_headers])
else:
self._skip_auto_headers = frozenset()
self._request_class = request_class
self._response_class = response_class
self._ws_response_class = ws_response_class
self._trace_configs = trace_configs or []
for trace_config in self._trace_configs:
trace_config.freeze()
async def test_keep_alive_http09():
headers = CIMultiDict(Connection='keep-alive')
req = make_request('GET', '/', version=HttpVersion(0, 9), headers=headers)
resp = StreamResponse()
await resp.prepare(req)
assert not resp.keep_alive
def _handshake(
self, request: BaseRequest
) -> Tuple['CIMultiDict[str]', str, bool, bool]:
headers = request.headers
if request.method != hdrs.METH_GET:
raise HTTPMethodNotAllowed(request.method, [hdrs.METH_GET])
if 'websocket' != headers.get(hdrs.UPGRADE, '').lower().strip():
raise HTTPBadRequest(text=('No WebSocket UPGRADE hdr: {}\n Can '
'"Upgrade" only to "WebSocket".'
).format(headers.get(hdrs.UPGRADE)))
if 'upgrade' not in headers.get(hdrs.CONNECTION, '').lower():
raise HTTPBadRequest(text='No CONNECTION upgrade hdr: {}'.format(
headers.get(hdrs.CONNECTION)))
# find common sub-protocol between client and server
protocol = None
if hdrs.SEC_WEBSOCKET_PROTOCOL in headers:
req_protocols = [
str(proto.strip())
for proto in headers[hdrs.SEC_WEBSOCKET_PROTOCOL].split(',')
]
for proto in req_protocols:
if proto in self._protocols:
protocol = proto
break
else:
# No overlap found: Return no protocol as per spec
ws_logger.warning(
'Client protocols %r don’t overlap server-known ones %r',
req_protocols, self._protocols)
# check supported version
version = headers.get(hdrs.SEC_WEBSOCKET_VERSION, '')
if version not in ('13', '8', '7'):
raise HTTPBadRequest(
text='Unsupported version: {}'.format(version))
# check client handshake for validity
key = headers.get(hdrs.SEC_WEBSOCKET_KEY)
try:
if not key or len(base64.b64decode(key)) != 16:
raise HTTPBadRequest(text='Handshake error: {!r}'.format(key))
except binascii.Error:
raise HTTPBadRequest(
text='Handshake error: {!r}'.format(key)) from None
accept_val = base64.b64encode(
hashlib.sha1(key.encode() + WS_KEY).digest()).decode()
response_headers = CIMultiDict( # type: ignore
{hdrs.UPGRADE: 'websocket',
hdrs.CONNECTION: 'upgrade',
hdrs.TRANSFER_ENCODING: 'chunked',
hdrs.SEC_WEBSOCKET_ACCEPT: accept_val})
notakeover = False
compress = 0
if self._compress:
extensions = headers.get(hdrs.SEC_WEBSOCKET_EXTENSIONS)
# Server side always get return with no exception.
# If something happened, just drop compress extension
compress, notakeover = ws_ext_parse(extensions, isserver=True)
if compress:
enabledext = ws_ext_gen(compress=compress,
isserver=True,
server_notakeover=notakeover)
response_headers[hdrs.SEC_WEBSOCKET_EXTENSIONS] = enabledext
if protocol:
response_headers[hdrs.SEC_WEBSOCKET_PROTOCOL] = protocol
return (
response_headers, # type: ignore
protocol,
compress,
notakeover)
"method", ["get", "post", "options", "post", "put", "patch", "delete"])
async def test_test_client_methods(method, loop, test_client) -> None:
resp = await getattr(test_client, method)("/")
assert resp.status == 200
text = await resp.text()
assert _hello_world_str == text
async def test_test_client_head(loop, test_client) -> None:
resp = await test_client.head("/")
assert resp.status == 200
@pytest.mark.parametrize("headers", [{
'token': 'x'
}, CIMultiDict({'token': 'x'}), {}])
def test_make_mocked_request(headers) -> None:
req = make_mocked_request('GET', '/', headers=headers)
assert req.method == "GET"
assert req.path == "/"
assert isinstance(req, web.Request)
assert isinstance(req.headers, CIMultiDictProxy)
def test_make_mocked_request_sslcontext() -> None:
req = make_mocked_request('GET', '/')
assert req.transport.get_extra_info('sslcontext') is None
def test_make_mocked_request_unknown_extra_info() -> None:
req = make_mocked_request('GET', '/')
"get", "post", "options", "post", "put", "patch", "delete"
])
async def test_test_client_methods(method, loop, test_client) -> None:
resp = await getattr(test_client, method)("/")
assert resp.status == 200
text = await resp.text()
assert _hello_world_str == text
async def test_test_client_head(loop, test_client) -> None:
resp = await test_client.head("/")
assert resp.status == 200
@pytest.mark.parametrize(
"headers", [{'token': 'x'}, CIMultiDict({'token': 'x'}), {}])
def test_make_mocked_request(headers) -> None:
req = make_mocked_request('GET', '/', headers=headers)
assert req.method == "GET"
assert req.path == "/"
assert isinstance(req, web.Request)
assert isinstance(req.headers, CIMultiDictProxy)
def test_make_mocked_request_sslcontext() -> None:
req = make_mocked_request('GET', '/')
assert req.transport.get_extra_info('sslcontext') is None
def test_make_mocked_request_unknown_extra_info() -> None:
req = make_mocked_request('GET', '/')
def test_single_forwarded_header_bad_separator() -> None:
header = "BY=identifier PROTO=https"
req = make_mocked_request("GET", "/", headers=CIMultiDict({"Forwarded": header}))
assert "proto" not in req.forwarded[0]
