async def log_ticket(request):
user_agent = request.headers.get(istr('USER-AGENT'))
referer = request.headers.get(istr('REFERER'))
if user_agent is None:
user_agent = ''
if referer is None:
referer = ''
post_data = await request.post()
try:
name = post_data['name']
except KeyError:
return web.HTTPBadRequest()
if not 1 <= len(name) <= 64:
return web.HTTPBadRequest(text='name -- Invalid string format (1~64)')
with request.app['db']() as session:
log_ticket = m.LogTicket(
ticket=str(uuid.uuid4()),
ip=request.remote,
user_agent=user_agent,
referer=referer,
name=name
)
session.add(log_ticket)
return web.json_response({
'ticket': log_ticket.ticket
})
def _init_header(request: web.Request,
addon: str) -> Union[CIMultiDict, Dict[str, str]]:
"""Create initial header."""
headers = {}
# filter flags
for name, value in request.headers.items():
if name in (
hdrs.CONTENT_LENGTH,
hdrs.CONTENT_ENCODING,
hdrs.SEC_WEBSOCKET_EXTENSIONS,
hdrs.SEC_WEBSOCKET_PROTOCOL,
hdrs.SEC_WEBSOCKET_VERSION,
hdrs.SEC_WEBSOCKET_KEY,
istr(HEADER_TOKEN),
istr(HEADER_TOKEN_OLD),
):
continue
headers[name] = value
# Update X-Forwarded-For
forward_for = request.headers.get(hdrs.X_FORWARDED_FOR)
connected_ip = ip_address(request.transport.get_extra_info("peername")[0])
headers[hdrs.X_FORWARDED_FOR] = f"{forward_for}, {connected_ip!s}"
return headers
def _init_header(request: web.Request) -> dict[istr, str]:
"""Create initial header."""
headers = {
AUTHORIZATION: f"Bearer {os.environ.get('SUPERVISOR_TOKEN', '')}",
CONTENT_TYPE: request.content_type,
}
# Add user data
if request.get("hass_user") is not None:
headers[istr(X_HASS_USER_ID)] = request["hass_user"].id
headers[istr(X_HASS_IS_ADMIN)] = str(int(request["hass_user"].is_admin))
return headers
def __init__(self, *, connector=None, loop=None, cookies=None,
headers=None, skip_auto_headers=None,
auth=None, request_class=ClientRequest,
response_class=ClientResponse,
ws_response_class=ClientWebSocketResponse,
version=aiohttp.HttpVersion11,
cookie_jar=None):
if connector is None:
connector = aiohttp.TCPConnector(loop=loop)
loop = connector._loop # never None
else:
if loop is None:
loop = connector._loop # never None
elif connector._loop is not loop:
raise ValueError("loop argument must agree with connector")
self._loop = loop
if loop.get_debug():
self._source_traceback = traceback.extract_stack(sys._getframe(1))
if not loop.is_running():
warnings.warn("Creating a client session outside of coroutine is "
"a very dangerous idea", ResourceWarning,
stacklevel=2)
context = {'client_session': self,
'message': 'Creating a client session outside '
'of coroutine'}
if self._source_traceback is not None:
context['source_traceback'] = self._source_traceback
loop.call_exception_handler(context)
if cookie_jar is None:
cookie_jar = CookieJar(loop=loop)
self._cookie_jar = cookie_jar
if cookies is not None:
self._cookie_jar.update_cookies(cookies)
self._connector = connector
self._default_auth = auth
self._version = version
# Convert to list of tuples
if headers:
headers = CIMultiDict(headers)
else:
headers = CIMultiDict()
self._default_headers = headers
if skip_auto_headers is not None:
self._skip_auto_headers = frozenset([istr(i)
for i in skip_auto_headers])
else:
self._skip_auto_headers = frozenset()
self._request_class = request_class
self._response_class = response_class
self._ws_response_class = ws_response_class
async def _start(self, request: "Request"):
self._req: "Request" = request
self._prepared = True
headers = self.headers
headers.setdefault(istr("Content-Type"), "application/octet-stream")
await request.send({
"type": "http.response.start",
"headers": self._headers_to_list(headers),
"status": self.status_code,
})
def __init__(self,
*,
connector=None,
loop=None,
cookies=None,
headers=None,
skip_auto_headers=None,
auth=None,
request_class=ClientRequest,
response_class=ClientResponse,
ws_response_class=ClientWebSocketResponse,
version=aiohttp.HttpVersion11,
cookie_jar=None):
if connector is None:
connector = aiohttp.TCPConnector(loop=loop)
loop = connector._loop # never None
else:
if loop is None:
loop = connector._loop # never None
elif connector._loop is not loop:
raise ValueError("loop argument must agree with connector")
self._loop = loop
if loop.get_debug():
self._source_traceback = traceback.extract_stack(sys._getframe(1))
if cookie_jar is None:
cookie_jar = CookieJar(loop=loop)
self._cookie_jar = cookie_jar
if cookies is not None:
self._cookie_jar.update_cookies(cookies)
self._connector = connector
self._default_auth = auth
self._version = version
# Convert to list of tuples
if headers:
headers = CIMultiDict(headers)
else:
headers = CIMultiDict()
self._default_headers = headers
if skip_auto_headers is not None:
self._skip_auto_headers = frozenset(
[istr(i) for i in skip_auto_headers])
else:
self._skip_auto_headers = frozenset()
self._request_class = request_class
self._response_class = response_class
self._ws_response_class = ws_response_class
async def files(self) -> AsyncGenerator[Any, None]:
"""
Iterator over attached files in multipart uploads
"""
assert not self._iterated, "Request fields already extracted"
self._iterated = True
if self._multipart_reader is not None:
async for field in self._multipart_reader:
if field.name is not None:
if field.filename:
self._args[field.name] = field.filename
yield self.file_hook_factory(name=field.name, file_name=field.filename, data=field)
elif field.headers.get(istr("Content-Type")) == 'application/json':
self._args[field.name] = await field.json()
else:
self._args[field.name] = await field.text()
def __init__(self, *, connector=None, loop=None, cookies=None,
headers=None, skip_auto_headers=None,
auth=None, request_class=ClientRequest,
response_class=ClientResponse,
ws_response_class=ClientWebSocketResponse,
version=aiohttp.HttpVersion11,
cookie_jar=None):
if connector is None:
connector = aiohttp.TCPConnector(loop=loop)
loop = connector._loop # never None
else:
if loop is None:
loop = connector._loop # never None
elif connector._loop is not loop:
raise ValueError("loop argument must agree with connector")
self._loop = loop
if loop.get_debug():
self._source_traceback = traceback.extract_stack(sys._getframe(1))
if cookie_jar is None:
cookie_jar = CookieJar(loop=loop)
self._cookie_jar = cookie_jar
if cookies is not None:
self._cookie_jar.update_cookies(cookies)
self._connector = connector
self._default_auth = auth
self._version = version
# Convert to list of tuples
if headers:
headers = CIMultiDict(headers)
else:
headers = CIMultiDict()
self._default_headers = headers
if skip_auto_headers is not None:
self._skip_auto_headers = frozenset([istr(i)
for i in skip_auto_headers])
else:
self._skip_auto_headers = frozenset()
self._request_class = request_class
self._response_class = response_class
self._ws_response_class = ws_response_class
def add_header(self, name, value):
"""Analyze headers. Calculate content length,
removes hop headers, etc."""
assert not self.headers_sent, 'headers have been sent already'
assert isinstance(name, str), \
'Header name should be a string, got {!r}'.format(name)
assert set(name).issubset(ASCIISET), \
'Header name should contain ASCII chars, got {!r}'.format(name)
assert isinstance(value, str), \
'Header {!r} should have string value, got {!r}'.format(
name, value)
name = istr(name)
value = value.strip()
if name == hdrs.CONTENT_LENGTH:
self.length = int(value)
if name == hdrs.TRANSFER_ENCODING:
self.has_chunked_hdr = value.lower().strip() == 'chunked'
if name == hdrs.CONNECTION:
val = value.lower()
# handle websocket
if 'upgrade' in val:
self.upgrade = True
# connection keep-alive
elif 'close' in val:
self.keepalive = False
elif 'keep-alive' in val:
self.keepalive = True
elif name == hdrs.UPGRADE:
if 'websocket' in value.lower():
self.websocket = True
self.headers[name] = value
elif name not in self.HOP_HEADERS:
# ignore hop-by-hop headers
self.headers.add(name, value)
def add_header(self, name, value):
"""Analyze headers. Calculate content length,
removes hop headers, etc."""
assert not self.headers_sent, 'headers have been sent already'
assert isinstance(name, str), \
'Header name should be a string, got {!r}'.format(name)
assert set(name).issubset(ASCIISET), \
'Header name should contain ASCII chars, got {!r}'.format(name)
assert isinstance(value, str), \
'Header {!r} should have string value, got {!r}'.format(
name, value)
name = istr(name)
value = value.strip()
if name == hdrs.CONTENT_LENGTH:
self.length = int(value)
if name == hdrs.TRANSFER_ENCODING:
self.has_chunked_hdr = value.lower() == 'chunked'
if name == hdrs.CONNECTION:
val = value.lower()
# handle websocket
if 'upgrade' in val:
self.upgrade = True
# connection keep-alive
elif 'close' in val:
self.keepalive = False
elif 'keep-alive' in val:
self.keepalive = True
elif name == hdrs.UPGRADE:
if 'websocket' in value.lower():
self.websocket = True
self.headers[name] = value
elif name not in self.HOP_HEADERS:
# ignore hop-by-hop headers
self.headers.add(name, value)
def on_header(self, name, value):
header = istr(name.decode(CHARSET))
header_value = value.decode(CHARSET)
header_env = header.upper().replace('-', '_')
if 'SERVER_PROTOCOL' not in self.environ:
self.environ['SERVER_PROTOCOL'] = ("HTTP/%s" %
self.parser.get_http_version())
if header in HOP_HEADERS:
if header == CONNECTION:
if (self.environ['SERVER_PROTOCOL'] == 'HTTP/1.0'
or header_value.lower() != 'keep-alive'):
self.headers[header] = header_value
else:
self.headers[header] = header_value
else:
hnd = getattr(self.header_wsgi, header_env, None)
if hnd and hnd(self.environ, header_value):
return
self.environ['HTTP_%s' % header_env] = header_value
def content_type(self, content_type):
self.headers[istr("Content-Type")] = content_type
async def _request(
self,
method: str,
str_or_url: StrOrURL,
*,
params: Optional[Mapping[str, str]] = None,
data: Any = None,
json: Any = None,
cookies: Optional[LooseCookies] = None,
headers: Optional[LooseHeaders] = None,
skip_auto_headers: Optional[Iterable[str]] = None,
auth: Optional[BasicAuth] = None,
allow_redirects: bool = True,
max_redirects: int = 10,
compress: Optional[str] = None,
chunked: Optional[bool] = None,
expect100: bool = False,
raise_for_status: Union[
None, bool, Callable[[ClientResponse], Awaitable[None]]
] = None,
read_until_eof: bool = True,
proxy: Optional[StrOrURL] = None,
proxy_auth: Optional[BasicAuth] = None,
timeout: Union[ClientTimeout, _SENTINEL] = sentinel,
ssl: Optional[Union[SSLContext, bool, Fingerprint]] = None,
proxy_headers: Optional[LooseHeaders] = None,
trace_request_ctx: Optional[SimpleNamespace] = None,
read_bufsize: Optional[int] = None,
) -> ClientResponse:
# NOTE: timeout clamps existing connect and read timeouts. We cannot
# set the default to None because we need to detect if the user wants
# to use the existing timeouts by setting timeout to None.
if self.closed:
raise RuntimeError("Session is closed")
if not isinstance(ssl, SSL_ALLOWED_TYPES):
raise TypeError(
"ssl should be SSLContext, bool, Fingerprint, "
"or None, got {!r} instead.".format(ssl)
)
if data is not None and json is not None:
raise ValueError(
"data and json parameters can not be used at the same time"
)
elif json is not None:
data = payload.JsonPayload(json, dumps=self._json_serialize)
redirects = 0
history = []
version = self._version
# Merge with default headers and transform to CIMultiDict
headers = self._prepare_headers(headers)
proxy_headers = self._prepare_headers(proxy_headers)
try:
url = self._build_url(str_or_url)
except ValueError as e:
raise InvalidURL(str_or_url) from e
skip_headers = set(self._skip_auto_headers)
if skip_auto_headers is not None:
for i in skip_auto_headers:
skip_headers.add(istr(i))
if proxy is not None:
try:
proxy = URL(proxy)
except ValueError as e:
raise InvalidURL(proxy) from e
if timeout is sentinel:
real_timeout = self._timeout # type: ClientTimeout
else:
if not isinstance(timeout, ClientTimeout):
real_timeout = ClientTimeout(total=timeout) # type: ignore[arg-type]
else:
real_timeout = timeout
# timeout is cumulative for all request operations
# (request, redirects, responses, data consuming)
tm = TimeoutHandle(self._loop, real_timeout.total)
handle = tm.start()
if read_bufsize is None:
read_bufsize = self._read_bufsize
traces = [
Trace(
self,
trace_config,
trace_config.trace_config_ctx(trace_request_ctx=trace_request_ctx),
)
for trace_config in self._trace_configs
]
for trace in traces:
await trace.send_request_start(method, url.update_query(params), headers)
timer = tm.timer()
try:
with timer:
while True:
url, auth_from_url = strip_auth_from_url(url)
if auth and auth_from_url:
raise ValueError(
"Cannot combine AUTH argument with "
"credentials encoded in URL"
)
if auth is None:
auth = auth_from_url
if auth is None:
auth = self._default_auth
# It would be confusing if we support explicit
# Authorization header with auth argument
if (
headers is not None
and auth is not None
and hdrs.AUTHORIZATION in headers
):
raise ValueError(
"Cannot combine AUTHORIZATION header "
"with AUTH argument or credentials "
"encoded in URL"
)
all_cookies = self._cookie_jar.filter_cookies(url)
if cookies is not None:
tmp_cookie_jar = CookieJar()
tmp_cookie_jar.update_cookies(cookies)
req_cookies = tmp_cookie_jar.filter_cookies(url)
if req_cookies:
all_cookies.load(req_cookies)
if proxy is not None:
proxy = URL(proxy)
elif self._trust_env:
with suppress(LookupError):
proxy, proxy_auth = get_env_proxy_for_url(url)
req = self._request_class(
method,
url,
params=params,
headers=headers,
skip_auto_headers=skip_headers,
data=data,
cookies=all_cookies,
auth=auth,
version=version,
compress=compress,
chunked=chunked,
expect100=expect100,
loop=self._loop,
response_class=self._response_class,
proxy=proxy,
proxy_auth=proxy_auth,
timer=timer,
session=self,
ssl=ssl,
proxy_headers=proxy_headers,
traces=traces,
)
# connection timeout
try:
async with ceil_timeout(real_timeout.connect):
assert self._connector is not None
conn = await self._connector.connect(
req, traces=traces, timeout=real_timeout
)
except asyncio.TimeoutError as exc:
raise ServerTimeoutError(
f"Connection timeout to host {url}"
) from exc
assert conn.transport is not None
assert conn.protocol is not None
conn.protocol.set_response_params(
timer=timer,
skip_payload=method.upper() == "HEAD",
read_until_eof=read_until_eof,
auto_decompress=self._auto_decompress,
read_timeout=real_timeout.sock_read,
read_bufsize=read_bufsize,
)
try:
try:
resp = await req.send(conn)
try:
await resp.start(conn)
except BaseException:
resp.close()
raise
except BaseException:
conn.close()
raise
except ClientError:
raise
except OSError as exc:
raise ClientOSError(*exc.args) from exc
self._cookie_jar.update_cookies(resp.cookies, resp.url)
# redirects
if resp.status in (301, 302, 303, 307, 308) and allow_redirects:
for trace in traces:
await trace.send_request_redirect(
method, url.update_query(params), headers, resp
)
redirects += 1
history.append(resp)
if max_redirects and redirects >= max_redirects:
resp.close()
raise TooManyRedirects(
history[0].request_info, tuple(history)
)
# For 301 and 302, mimic IE, now changed in RFC
# https://github.com/kennethreitz/requests/pull/269
if (resp.status == 303 and resp.method != hdrs.METH_HEAD) or (
resp.status in (301, 302) and resp.method == hdrs.METH_POST
):
method = hdrs.METH_GET
data = None
if headers.get(hdrs.CONTENT_LENGTH):
headers.pop(hdrs.CONTENT_LENGTH)
r_url = resp.headers.get(hdrs.LOCATION) or resp.headers.get(
hdrs.URI
)
if r_url is None:
# see github.com/aio-libs/aiohttp/issues/2022
break
else:
# reading from correct redirection
# response is forbidden
resp.release()
try:
parsed_url = URL(
r_url, encoded=not self._requote_redirect_url
)
except ValueError as e:
raise InvalidURL(r_url) from e
scheme = parsed_url.scheme
if scheme not in ("http", "https", ""):
resp.close()
raise ValueError("Can redirect only to http or https")
elif not scheme:
parsed_url = url.join(parsed_url)
is_same_host_https_redirect = (
url.host == parsed_url.host
and parsed_url.scheme == "https"
and url.scheme == "http"
)
if (
url.origin() != parsed_url.origin()
and not is_same_host_https_redirect
):
auth = None
headers.pop(hdrs.AUTHORIZATION, None)
url = parsed_url
params = None
resp.release()
continue
break
# check response status
if raise_for_status is None:
raise_for_status = self._raise_for_status
if raise_for_status is None:
pass
elif callable(raise_for_status):
await raise_for_status(resp)
elif raise_for_status:
resp.raise_for_status()
# register connection
if handle is not None:
if resp.connection is not None:
resp.connection.add_callback(handle.cancel)
else:
handle.cancel()
resp._history = tuple(history)
for trace in traces:
await trace.send_request_end(
method, url.update_query(params), headers, resp
)
return resp
except BaseException as e:
# cleanup timer
tm.close()
if handle:
handle.cancel()
handle = None
for trace in traces:
await trace.send_request_exception(
method, url.update_query(params), headers, e
)
raise
def _request(self,
method,
url,
*,
params=None,
data=None,
json=None,
headers=None,
skip_auto_headers=None,
auth=None,
allow_redirects=True,
max_redirects=10,
encoding=None,
compress=None,
chunked=None,
expect100=False,
read_until_eof=True,
proxy=None,
proxy_auth=None,
timeout=sentinel):
# NOTE: timeout clamps existing connect and read timeouts. We cannot
# set the default to None because we need to detect if the user wants
# to use the existing timeouts by setting timeout to None.
if encoding is not None:
warnings.warn(
"encoding parameter is not supported, "
"please use FormData(charset='utf-8') instead",
DeprecationWarning)
if self.closed:
raise RuntimeError('Session is closed')
if data is not None and json is not None:
raise ValueError(
'data and json parameters can not be used at the same time')
elif json is not None:
data = payload.JsonPayload(json, dumps=self._json_serialize)
if not isinstance(chunked, bool) and chunked is not None:
warnings.warn('Chunk size is deprecated #1615', DeprecationWarning)
redirects = 0
history = []
version = self._version
# Merge with default headers and transform to CIMultiDict
headers = self._prepare_headers(headers)
if auth is None:
auth = self._default_auth
# It would be confusing if we support explicit Authorization header
# with `auth` argument
if (headers is not None and auth is not None
and hdrs.AUTHORIZATION in headers):
raise ValueError("Can't combine `Authorization` header with "
"`auth` argument")
skip_headers = set(self._skip_auto_headers)
if skip_auto_headers is not None:
for i in skip_auto_headers:
skip_headers.add(istr(i))
if proxy is not None:
proxy = URL(proxy)
# timeout is cumulative for all request operations
# (request, redirects, responses, data consuming)
tm = TimeoutHandle(
self._loop,
timeout if timeout is not sentinel else self._read_timeout)
handle = tm.start()
timer = tm.timer()
try:
with timer:
while True:
url = URL(url).with_fragment(None)
cookies = self._cookie_jar.filter_cookies(url)
req = self._request_class(
method,
url,
params=params,
headers=headers,
skip_auto_headers=skip_headers,
data=data,
cookies=cookies,
auth=auth,
version=version,
compress=compress,
chunked=chunked,
expect100=expect100,
loop=self._loop,
response_class=self._response_class,
proxy=proxy,
proxy_auth=proxy_auth,
timer=timer,
session=self)
# connection timeout
try:
with CeilTimeout(self._conn_timeout, loop=self._loop):
conn = yield from self._connector.connect(req)
except asyncio.TimeoutError as exc:
raise ServerTimeoutError(
'Connection timeout '
'to host {0}'.format(url)) from exc
conn.writer.set_tcp_nodelay(True)
try:
resp = req.send(conn)
try:
yield from resp.start(conn, read_until_eof)
except:
resp.close()
conn.close()
raise
except ClientError:
raise
except OSError as exc:
raise ClientOSError(*exc.args) from exc
self._cookie_jar.update_cookies(resp.cookies, resp.url)
# redirects
if resp.status in (301, 302, 303, 307) and allow_redirects:
redirects += 1
history.append(resp)
if max_redirects and redirects >= max_redirects:
resp.close()
break
else:
resp.release()
# For 301 and 302, mimic IE, now changed in RFC
# https://github.com/kennethreitz/requests/pull/269
if (resp.status == 303 and
resp.method != hdrs.METH_HEAD) \
or (resp.status in (301, 302) and
resp.method == hdrs.METH_POST):
method = hdrs.METH_GET
data = None
if headers.get(hdrs.CONTENT_LENGTH):
headers.pop(hdrs.CONTENT_LENGTH)
r_url = (resp.headers.get(hdrs.LOCATION)
or resp.headers.get(hdrs.URI))
if r_url is None:
# see github.com/aio-libs/aiohttp/issues/2022
break
r_url = URL(r_url,
encoded=not self.requote_redirect_url)
scheme = r_url.scheme
if scheme not in ('http', 'https', ''):
resp.close()
raise ValueError(
'Can redirect only to http or https')
elif not scheme:
r_url = url.join(r_url)
url = r_url
params = None
resp.release()
continue
break
# check response status
if self._raise_for_status:
resp.raise_for_status()
# register connection
if handle is not None:
if resp.connection is not None:
resp.connection.add_callback(handle.cancel)
else:
handle.cancel()
resp._history = tuple(history)
return resp
except:
# cleanup timer
tm.close()
if handle:
handle.cancel()
handle = None
raise
def _request(self, method, url, *,
params=None,
data=None,
headers=None,
skip_auto_headers=None,
auth=None,
allow_redirects=True,
max_redirects=10,
encoding='utf-8',
version=None,
compress=None,
chunked=None,
expect100=False,
read_until_eof=True,
proxy=None,
proxy_auth=None,
timeout=5*60):
if version is not None:
warnings.warn("HTTP version should be specified "
"by ClientSession constructor", DeprecationWarning)
else:
version = self._version
if self.closed:
raise RuntimeError('Session is closed')
redirects = 0
history = []
# Merge with default headers and transform to CIMultiDict
headers = self._prepare_headers(headers)
if auth is None:
auth = self._default_auth
# It would be confusing if we support explicit Authorization header
# with `auth` argument
if (headers is not None and
auth is not None and
hdrs.AUTHORIZATION in headers):
raise ValueError("Can't combine `Authorization` header with "
"`auth` argument")
skip_headers = set(self._skip_auto_headers)
if skip_auto_headers is not None:
for i in skip_auto_headers:
skip_headers.add(istr(i))
if proxy is not None:
proxy = URL(proxy)
while True:
url = URL(url).with_fragment(None)
cookies = self._cookie_jar.filter_cookies(url)
req = self._request_class(
method, url, params=params, headers=headers,
skip_auto_headers=skip_headers, data=data,
cookies=cookies, encoding=encoding,
auth=auth, version=version, compress=compress, chunked=chunked,
expect100=expect100,
loop=self._loop, response_class=self._response_class,
proxy=proxy, proxy_auth=proxy_auth, timeout=timeout)
with Timeout(timeout, loop=self._loop):
conn = yield from self._connector.connect(req)
try:
resp = req.send(conn.writer, conn.reader)
try:
yield from resp.start(conn, read_until_eof)
except:
resp.close()
conn.close()
raise
except (aiohttp.HttpProcessingError,
aiohttp.ServerDisconnectedError) as exc:
raise aiohttp.ClientResponseError() from exc
except OSError as exc:
raise aiohttp.ClientOSError(*exc.args) from exc
self._cookie_jar.update_cookies(resp.cookies, resp.url_obj)
# redirects
if resp.status in (301, 302, 303, 307) and allow_redirects:
redirects += 1
history.append(resp)
if max_redirects and redirects >= max_redirects:
resp.close()
break
else:
# TODO: close the connection if BODY is large enough
# Redirect with big BODY is forbidden by HTTP protocol
# but malformed server may send illegal response.
# Small BODIES with text like "Not Found" are still
# perfectly fine and should be accepted.
yield from resp.release()
# For 301 and 302, mimic IE behaviour, now changed in RFC.
# Details: https://github.com/kennethreitz/requests/pull/269
if (resp.status == 303 and resp.method != hdrs.METH_HEAD) \
or (resp.status in (301, 302) and
resp.method == hdrs.METH_POST):
method = hdrs.METH_GET
data = None
if headers.get(hdrs.CONTENT_LENGTH):
headers.pop(hdrs.CONTENT_LENGTH)
r_url = URL(resp.headers.get(hdrs.LOCATION) or
resp.headers.get(hdrs.URI))
scheme = r_url.scheme
if scheme not in ('http', 'https', ''):
resp.close()
raise ValueError('Can redirect only to http or https')
elif not scheme:
r_url = url.join(r_url)
url = r_url
params = None
yield from resp.release()
continue
break
resp._history = tuple(history)
return resp
def test_skip_default_useragent_header(make_request):
req = make_request('get', 'http://python.org/',
skip_auto_headers=set([istr('user-agent')]))
assert 'User-Agent' not in req.headers
def parse_headers(self, lines):
"""Parses RFC 5322 headers from a stream.
Line continuations are supported. Returns list of header name
and value pairs. Header name is in upper case.
"""
headers = CIMultiDict()
raw_headers = []
lines_idx = 1
line = lines[1]
line_count = len(lines)
while line:
header_length = len(line)
# Parse initial header name : value pair.
try:
bname, bvalue = line.split(b':', 1)
except ValueError:
raise InvalidHeader(line) from None
bname = bname.strip(b' \t')
if HDRRE.search(bname):
raise InvalidHeader(bname)
# next line
lines_idx += 1
line = lines[lines_idx]
# consume continuation lines
continuation = line and line[0] in (32, 9) # (' ', '\t')
if continuation:
bvalue = [bvalue]
while continuation:
header_length += len(line)
if header_length > self.max_field_size:
raise LineTooLong(
'request header field {}'.format(
bname.decode("utf8", "xmlcharrefreplace")),
self.max_field_size)
bvalue.append(line)
# next line
lines_idx += 1
if lines_idx < line_count:
line = lines[lines_idx]
if line:
continuation = line[0] in (32, 9) # (' ', '\t')
else:
line = b''
break
bvalue = b''.join(bvalue)
else:
if header_length > self.max_field_size:
raise LineTooLong(
'request header field {}'.format(
bname.decode("utf8", "xmlcharrefreplace")),
self.max_field_size)
bvalue = bvalue.strip()
name = istr(bname.decode('utf-8', 'surrogateescape'))
value = bvalue.decode('utf-8', 'surrogateescape')
headers.add(name, value)
raw_headers.append((bname, bvalue))
close_conn = None
encoding = None
upgrade = False
chunked = False
raw_headers = tuple(raw_headers)
# keep-alive
conn = headers.get(hdrs.CONNECTION)
if conn:
v = conn.lower()
if v == 'close':
close_conn = True
elif v == 'keep-alive':
close_conn = False
elif v == 'upgrade':
upgrade = True
# encoding
enc = headers.get(hdrs.CONTENT_ENCODING)
if enc:
enc = enc.lower()
if enc in ('gzip', 'deflate'):
encoding = enc
# chunking
te = headers.get(hdrs.TRANSFER_ENCODING)
if te and 'chunked' in te.lower():
chunked = True
return headers, raw_headers, close_conn, encoding, upgrade, chunked
def content_type(self):
return self.headers.get(istr("Content-Type"))
from multidict import istr
ACCEPT = istr("Accept")
ACCEPT_CHARSET = istr("Accept-Charset")
ACCEPT_ENCODING = istr("Accept-Encoding")
ACCEPT_LANGUAGE = istr("Accept-Language")
ACCEPT_RANGES = istr("Accept-Ranges")
ACCESS_CONTROL_ALLOW_CREDENTIALS = istr("Access-Control-Allow-Credentials")
ACCESS_CONTROL_ALLOW_HEADERS = istr("Access-Control-Allow-Headers")
ACCESS_CONTROL_ALLOW_METHODS = istr("Access-Control-Allow-Methods")
ACCESS_CONTROL_ALLOW_ORIGIN = istr("Access-Control-Allow-Origin")
ACCESS_CONTROL_EXPOSE_HEADERS = istr("Access-Control-Expose-Headers")
ACCESS_CONTROL_MAX_AGE = istr("Access-Control-Max-Age")
ACCESS_CONTROL_REQUEST_HEADERS = istr("Access-Control-Request-Headers")
ACCESS_CONTROL_REQUEST_METHOD = istr("Access-Control-Request-Method")
AGE = istr("Age")
ALLOW = istr("Allow")
ALT_SVC = istr("Alt-Svc")
AUTHORIZATION = istr("Authorization")
CACHE_CONTROL = istr("Cache-Control")
CLEAR_SITE_DATA = istr("Clear-Site-Data")
CONNECTION = istr("Connection")
CONTENT_DISPOSITION = istr("Content-Disposition")
CONTENT_ENCODING = istr("Content-Encoding")
CONTENT_LANGUAGE = istr("Content-Language")
CONTENT_LENGTH = istr("Content-Length")
CONTENT_LOCATION = istr("Content-Location")
CONTENT_RANGE = istr("Content-Range")
CONTENT_SECURITY_POLICY = istr("Content-Security-Policy")
CONTENT_SECURITY_POLICY_REPORT_ONLY = istr("Content-Security-Policy-Report-Only")
CONTENT_TYPE = istr("Content-Type")
def content_length(self, content_length):
self.headers[istr("Content-Length")] = str(content_length)
def parse_headers(self, lines):
"""Parses RFC 5322 headers from a stream.
Line continuations are supported. Returns list of header name
and value pairs. Header name is in upper case.
"""
headers = CIMultiDict()
raw_headers = []
lines_idx = 1
line = lines[1]
line_count = len(lines)
while line:
header_length = len(line)
# Parse initial header name : value pair.
try:
bname, bvalue = line.split(b':', 1)
except ValueError:
raise InvalidHeader(line) from None
bname = bname.strip(b' \t')
if HDRRE.search(bname):
raise InvalidHeader(bname)
# next line
lines_idx += 1
line = lines[lines_idx]
# consume continuation lines
continuation = line and line[0] in (32, 9) # (' ', '\t')
if continuation:
bvalue = [bvalue]
while continuation:
header_length += len(line)
if header_length > self.max_field_size:
raise LineTooLong(
'request header field {}'.format(
bname.decode("utf8", "xmlcharrefreplace")),
self.max_field_size)
bvalue.append(line)
# next line
lines_idx += 1
if lines_idx < line_count:
line = lines[lines_idx]
if line:
continuation = line[0] in (32, 9) # (' ', '\t')
else:
line = b''
break
bvalue = b''.join(bvalue)
else:
if header_length > self.max_field_size:
raise LineTooLong(
'request header field {}'.format(
bname.decode("utf8", "xmlcharrefreplace")),
self.max_field_size)
bvalue = bvalue.strip()
name = istr(bname.decode('utf-8', 'surrogateescape'))
value = bvalue.decode('utf-8', 'surrogateescape')
headers.add(name, value)
raw_headers.append((bname, bvalue))
close_conn = None
encoding = None
upgrade = False
chunked = False
raw_headers = tuple(raw_headers)
# keep-alive
conn = headers.get(hdrs.CONNECTION)
if conn:
v = conn.lower()
if v == 'close':
close_conn = True
elif v == 'keep-alive':
close_conn = False
elif v == 'upgrade':
upgrade = True
# encoding
enc = headers.get(hdrs.CONTENT_ENCODING)
if enc:
enc = enc.lower()
if enc in ('gzip', 'deflate'):
encoding = enc
# chunking
te = headers.get(hdrs.TRANSFER_ENCODING)
if te and 'chunked' in te.lower():
chunked = True
return headers, raw_headers, close_conn, encoding, upgrade, chunked
from __future__ import annotations
from typing import Optional
from json import load
from os import listdir, path
from multidict import CIMultiDict, istr
from telethon.tl import types
I18N_PATH = path.split(path.realpath(__file__))[0]
ALL_LANGUAGES = tuple(sorted(lang[:-5] for lang in listdir(I18N_PATH) if lang.endswith('.json')))
FALLBACK_LANGUAGE = istr('en')
NO_FALLBACK_KEYS = {istr('iso_639_code')}
NEED_PRE_FILL = {
# istr('default_emoji_header_description'):
# ('↩',),
istr('read_formatting_settings_guidebook_html'):
('https://github.com/Rongronggg9/RSS-to-Telegram-Bot/blob/dev/docs/formatting-settings.md',),
}
COMMANDS = ('sub', 'unsub', 'unsub_all', 'list', 'set', 'set_default', 'import', 'export', 'activate_subs',
'deactivate_subs', 'version', 'help', 'lang')
MANAGER_COMMANDS = ('test', 'set_option', 'user_info')
class _I18N:
__instance: Optional["_I18N"] = None
__initialized: bool = False
def __new__(cls, *args, **kwargs):
if cls.__instance is None:
def _request(self, method, url, *,
params=None,
data=None,
json=None,
headers=None,
skip_auto_headers=None,
auth=None,
allow_redirects=True,
max_redirects=10,
encoding=None,
compress=None,
chunked=None,
expect100=False,
read_until_eof=True,
proxy=None,
proxy_auth=None,
timeout=DEFAULT_TIMEOUT):
# NOTE: timeout clamps existing connect and read timeouts. We cannot
# set the default to None because we need to detect if the user wants
# to use the existing timeouts by setting timeout to None.
if encoding is not None:
warnings.warn(
"encoding parameter is not supported, "
"please use FormData(charset='utf-8') instead",
DeprecationWarning)
if self.closed:
raise RuntimeError('Session is closed')
if data is not None and json is not None:
raise ValueError(
'data and json parameters can not be used at the same time')
elif json is not None:
data = payload.JsonPayload(json, dumps=self._json_serialize)
if not isinstance(chunked, bool) and chunked is not None:
warnings.warn(
'Chunk size is deprecated #1615', DeprecationWarning)
redirects = 0
history = []
version = self._version
# Merge with default headers and transform to CIMultiDict
headers = self._prepare_headers(headers)
if auth is None:
auth = self._default_auth
# It would be confusing if we support explicit Authorization header
# with `auth` argument
if (headers is not None and
auth is not None and
hdrs.AUTHORIZATION in headers):
raise ValueError("Can't combine `Authorization` header with "
"`auth` argument")
skip_headers = set(self._skip_auto_headers)
if skip_auto_headers is not None:
for i in skip_auto_headers:
skip_headers.add(istr(i))
if proxy is not None:
proxy = URL(proxy)
# timeout is cumulative for all request operations
# (request, redirects, responses, data consuming)
tm = TimeoutHandle(
self._loop, timeout if timeout is not None else self._read_timeout)
handle = tm.start()
timer = tm.timer()
try:
with timer:
while True:
url = URL(url).with_fragment(None)
cookies = self._cookie_jar.filter_cookies(url)
req = self._request_class(
method, url, params=params, headers=headers,
skip_auto_headers=skip_headers, data=data,
cookies=cookies, auth=auth, version=version,
compress=compress, chunked=chunked,
expect100=expect100, loop=self._loop,
response_class=self._response_class,
proxy=proxy, proxy_auth=proxy_auth, timer=timer)
# connection timeout
try:
with CeilTimeout(self._conn_timeout, loop=self._loop):
conn = yield from self._connector.connect(req)
except asyncio.TimeoutError as exc:
raise ServerTimeoutError(
'Connection timeout '
'to host {0}'.format(url)) from exc
conn.writer.set_tcp_nodelay(True)
try:
resp = req.send(conn)
try:
yield from resp.start(conn, read_until_eof)
except:
resp.close()
conn.close()
raise
except ClientError:
raise
except OSError as exc:
raise ClientOSError(*exc.args) from exc
self._cookie_jar.update_cookies(resp.cookies, resp.url)
# redirects
if resp.status in (301, 302, 303, 307) and allow_redirects:
redirects += 1
history.append(resp)
if max_redirects and redirects >= max_redirects:
resp.close()
break
else:
resp.release()
# For 301 and 302, mimic IE, now changed in RFC
# https://github.com/kennethreitz/requests/pull/269
if (resp.status == 303 and
resp.method != hdrs.METH_HEAD) \
or (resp.status in (301, 302) and
resp.method == hdrs.METH_POST):
method = hdrs.METH_GET
data = None
if headers.get(hdrs.CONTENT_LENGTH):
headers.pop(hdrs.CONTENT_LENGTH)
r_url = (resp.headers.get(hdrs.LOCATION) or
resp.headers.get(hdrs.URI))
if r_url is None:
raise RuntimeError(
"{0.method} {0.url} returns "
"a redirect [{0.status}] status "
"but response lacks a Location "
"or URI HTTP header".format(resp))
r_url = URL(
r_url, encoded=not self.requote_redirect_url)
scheme = r_url.scheme
if scheme not in ('http', 'https', ''):
resp.close()
raise ValueError(
'Can redirect only to http or https')
elif not scheme:
r_url = url.join(r_url)
url = r_url
params = None
resp.release()
continue
break
# check response status
if self._raise_for_status:
resp.raise_for_status()
# register connection
if handle is not None:
if resp.connection is not None:
resp.connection.add_callback(handle.cancel)
else:
handle.cancel()
resp._history = tuple(history)
return resp
except:
# cleanup timer
tm.close()
if handle:
handle.cancel()
handle = None
raise
def parse_headers(self, lines):
"""Parses RFC 5322 headers from a stream.
Line continuations are supported. Returns list of header name
and value pairs. Header name is in upper case.
"""
close_conn = None
encoding = None
headers = CIMultiDict()
raw_headers = []
lines_idx = 1
line = lines[1]
while line:
header_length = len(line)
# Parse initial header name : value pair.
try:
bname, bvalue = line.split(b':', 1)
except ValueError:
raise errors.InvalidHeader(line) from None
bname = bname.strip(b' \t').upper()
if HDRRE.search(bname):
raise errors.InvalidHeader(bname)
# next line
lines_idx += 1
line = lines[lines_idx]
# consume continuation lines
continuation = line and line[0] in (32, 9) # (' ', '\t')
if continuation:
bvalue = [bvalue]
while continuation:
header_length += len(line)
if header_length > self.max_field_size:
raise errors.LineTooLong(
'request header field {}'.format(
bname.decode("utf8", "xmlcharrefreplace")),
self.max_field_size)
bvalue.append(line)
# next line
lines_idx += 1
line = lines[lines_idx]
continuation = line[0] in (32, 9) # (' ', '\t')
bvalue = b'\r\n'.join(bvalue)
else:
if header_length > self.max_field_size:
raise errors.LineTooLong(
'request header field {}'.format(
bname.decode("utf8", "xmlcharrefreplace")),
self.max_field_size)
bvalue = bvalue.strip()
name = istr(bname.decode('utf-8', 'surrogateescape'))
value = bvalue.decode('utf-8', 'surrogateescape')
# keep-alive and encoding
if name == hdrs.CONNECTION:
v = value.lower()
if v == 'close':
close_conn = True
elif v == 'keep-alive':
close_conn = False
elif name == hdrs.CONTENT_ENCODING:
enc = value.lower()
if enc in ('gzip', 'deflate'):
encoding = enc
headers.add(name, value)
raw_headers.append((bname, bvalue))
return headers, raw_headers, close_conn, encoding
import logging
from multidict import istr
from pulsar.utils.httpurl import tls_schemes
LOGGER = logging.getLogger('pulsar.wsgi')
COOKIE = istr('Cookie')
CONNECTION = istr('Connection')
CONTENT_LENGTH = istr('Content-Length')
CONTENT_TYPE = istr('Content-Type')
DATE = istr('Date')
HOST = istr('Host')
KEEP_ALIVE = istr('Keep-Alive')
LOCATION = istr('Location')
PROXY_AUTHENTICATE = istr('Proxy-Authenticate')
PROXY_AUTHORIZATION = istr('Proxy-Authorization')
SCRIPT_NAME = istr("Script_Name")
SERVER = istr('Server')
SET_COOKIE = istr('Set-Cookie')
TE = istr('Te')
TRAILERS = istr('Trailers')
TRANSFER_ENCODING = istr('Transfer-Encoding')
UPGRADE = istr('Upgrade')
X_FORWARDED_FOR = istr('X-Forwarded-For')
X_FORWARDED_PROTOCOL = istr("X-Forwarded-Protocol")
X_FORWARDED_PROTO = istr("X-Forwarded-Proto")
X_FORWARDED_SSL = istr("X-Forwarded-Ssl")
def content_length(self):
return self.headers.get(istr("Content-Length"))
def __init__(
self,
*,
connector: Optional[BaseConnector] = None,
loop: Optional[asyncio.AbstractEventLoop] = None,
cookies: Optional[LooseCookies] = None,
headers: LooseHeaders = None,
skip_auto_headers: Optional[Iterable[str]] = None,
auth: Optional[BasicAuth] = None,
json_serialize: JSONEncoder = json.dumps,
request_class: Type[ClientRequest] = ClientRequest,
response_class: Type[ClientResponse] = ClientResponse,
ws_response_class: Type[
ClientWebSocketResponse] = ClientWebSocketResponse, # noqa
version: HttpVersion = http.HttpVersion11,
cookie_jar: Optional[AbstractCookieJar] = None,
connector_owner: bool = True,
raise_for_status: bool = False,
read_timeout: Union[float, object] = sentinel,
conn_timeout: Optional[float] = None,
timeout: Union[object, ClientTimeout] = sentinel,
auto_decompress: bool = True,
trust_env: bool = False,
trace_configs: Optional[List[TraceConfig]] = None) -> None:
implicit_loop = False
if loop is None:
if connector is not None:
loop = connector._loop
else:
implicit_loop = True
loop = asyncio.get_event_loop()
if connector is None:
connector = TCPConnector(loop=loop)
if connector._loop is not loop:
raise RuntimeError(
"Session and connector has to use same event loop")
self._loop = loop
if loop.get_debug():
self._source_traceback = traceback.extract_stack(sys._getframe(1))
if implicit_loop and not loop.is_running():
warnings.warn(
"Creating a client session outside of coroutine is "
"a very dangerous idea",
stacklevel=2)
context = {
'client_session': self,
'message': 'Creating a client session outside '
'of coroutine'
}
if self._source_traceback is not None:
context['source_traceback'] = self._source_traceback
loop.call_exception_handler(context)
if cookie_jar is None:
cookie_jar = CookieJar(loop=loop)
self._cookie_jar = cookie_jar
if cookies is not None:
self._cookie_jar.update_cookies(cookies)
self._connector = connector # type: BaseConnector
self._connector_owner = connector_owner
self._default_auth = auth
self._version = version
self._json_serialize = json_serialize
if timeout is sentinel:
self._timeout = DEFAULT_TIMEOUT
if read_timeout is not sentinel:
self._timeout = attr.evolve(self._timeout, total=read_timeout)
if conn_timeout is not None:
self._timeout = attr.evolve(self._timeout,
connect=conn_timeout)
else:
self._timeout = timeout # type: ignore
if read_timeout is not sentinel:
raise ValueError("read_timeout and timeout parameters "
"conflict, please setup "
"timeout.read")
if conn_timeout is not None:
raise ValueError("conn_timeout and timeout parameters "
"conflict, please setup "
"timeout.connect")
self._raise_for_status = raise_for_status
self._auto_decompress = auto_decompress
self._trust_env = trust_env
# Convert to list of tuples
if headers:
headers = CIMultiDict(headers)
else:
headers = CIMultiDict()
self._default_headers = headers
if skip_auto_headers is not None:
self._skip_auto_headers = frozenset(
[istr(i) for i in skip_auto_headers])
else:
self._skip_auto_headers = frozenset()
self._request_class = request_class
self._response_class = response_class
self._ws_response_class = ws_response_class
self._trace_configs = trace_configs or []
for trace_config in self._trace_configs:
trace_config.freeze()
def __init__(self, *, connector: Optional[BaseConnector]=None,
loop: Optional[asyncio.AbstractEventLoop]=None,
cookies: Optional[LooseCookies]=None,
headers: Optional[LooseHeaders]=None,
skip_auto_headers: Optional[Iterable[str]]=None,
auth: Optional[BasicAuth]=None,
json_serialize: JSONEncoder=json.dumps,
request_class: Type[ClientRequest]=ClientRequest,
response_class: Type[ClientResponse]=ClientResponse,
ws_response_class: Type[ClientWebSocketResponse]=ClientWebSocketResponse, # noqa
version: HttpVersion=http.HttpVersion11,
cookie_jar: Optional[AbstractCookieJar]=None,
connector_owner: bool=True,
raise_for_status: bool=False,
read_timeout: Union[float, object]=sentinel,
conn_timeout: Optional[float]=None,
timeout: Union[object, ClientTimeout]=sentinel,
auto_decompress: bool=True,
trust_env: bool=False,
requote_redirect_url: bool=True,
trace_configs: Optional[List[TraceConfig]]=None) -> None:
if loop is None:
if connector is not None:
loop = connector._loop
loop = get_running_loop(loop)
if connector is None:
connector = TCPConnector(loop=loop)
# Initialize these three attrs before raising any exception,
# they are used in __del__
self._connector = connector # type: Optional[BaseConnector]
self._loop = loop
if loop.get_debug():
self._source_traceback = traceback.extract_stack(sys._getframe(1)) # type: Optional[traceback.StackSummary] # noqa
else:
self._source_traceback = None
if connector._loop is not loop:
raise RuntimeError(
"Session and connector have to use same event loop")
if cookie_jar is None:
cookie_jar = CookieJar(loop=loop)
self._cookie_jar = cookie_jar
if cookies is not None:
self._cookie_jar.update_cookies(cookies)
self._connector_owner = connector_owner
self._default_auth = auth
self._version = version
self._json_serialize = json_serialize
if timeout is sentinel:
self._timeout = DEFAULT_TIMEOUT
if read_timeout is not sentinel:
warnings.warn("read_timeout is deprecated, "
"use timeout argument instead",
DeprecationWarning,
stacklevel=2)
self._timeout = attr.evolve(self._timeout, total=read_timeout)
if conn_timeout is not None:
self._timeout = attr.evolve(self._timeout,
connect=conn_timeout)
warnings.warn("conn_timeout is deprecated, "
"use timeout argument instead",
DeprecationWarning,
stacklevel=2)
else:
self._timeout = timeout # type: ignore
if read_timeout is not sentinel:
raise ValueError("read_timeout and timeout parameters "
"conflict, please setup "
"timeout.read")
if conn_timeout is not None:
raise ValueError("conn_timeout and timeout parameters "
"conflict, please setup "
"timeout.connect")
self._raise_for_status = raise_for_status
self._auto_decompress = auto_decompress
self._trust_env = trust_env
self._requote_redirect_url = requote_redirect_url
# Convert to list of tuples
if headers:
headers = CIMultiDict(headers)
else:
headers = CIMultiDict()
self._default_headers = headers
if skip_auto_headers is not None:
self._skip_auto_headers = frozenset([istr(i)
for i in skip_auto_headers])
else:
self._skip_auto_headers = frozenset()
self._request_class = request_class
self._response_class = response_class
self._ws_response_class = ws_response_class
self._trace_configs = trace_configs or []
for trace_config in self._trace_configs:
trace_config.freeze()
METH_CONNECT = 'CONNECT'
METH_HEAD = 'HEAD'
METH_GET = 'GET'
METH_DELETE = 'DELETE'
METH_OPTIONS = 'OPTIONS'
METH_PATCH = 'PATCH'
METH_POST = 'POST'
METH_PUT = 'PUT'
METH_TRACE = 'TRACE'
METH_ALL = {
METH_CONNECT, METH_HEAD, METH_GET, METH_DELETE, METH_OPTIONS, METH_PATCH,
METH_POST, METH_PUT, METH_TRACE
}
ACCEPT = istr('ACCEPT')
ACCEPT_CHARSET = istr('ACCEPT-CHARSET')
ACCEPT_ENCODING = istr('ACCEPT-ENCODING')
ACCEPT_LANGUAGE = istr('ACCEPT-LANGUAGE')
ACCEPT_RANGES = istr('ACCEPT-RANGES')
ACCESS_CONTROL_MAX_AGE = istr('ACCESS-CONTROL-MAX-AGE')
ACCESS_CONTROL_ALLOW_CREDENTIALS = istr('ACCESS-CONTROL-ALLOW-CREDENTIALS')
ACCESS_CONTROL_ALLOW_HEADERS = istr('ACCESS-CONTROL-ALLOW-HEADERS')
ACCESS_CONTROL_ALLOW_METHODS = istr('ACCESS-CONTROL-ALLOW-METHODS')
ACCESS_CONTROL_ALLOW_ORIGIN = istr('ACCESS-CONTROL-ALLOW-ORIGIN')
ACCESS_CONTROL_EXPOSE_HEADERS = istr('ACCESS-CONTROL-EXPOSE-HEADERS')
ACCESS_CONTROL_REQUEST_HEADERS = istr('ACCESS-CONTROL-REQUEST-HEADERS')
ACCESS_CONTROL_REQUEST_METHOD = istr('ACCESS-CONTROL-REQUEST-METHOD')
AGE = istr('AGE')
ALLOW = istr('ALLOW')
AUTHORIZATION = istr('AUTHORIZATION')
async def _request(
self,
method: str,
str_or_url: StrOrURL, *,
params: Optional[Mapping[str, str]]=None,
data: Any=None,
json: Any=None,
cookies: Optional[LooseCookies]=None,
headers: LooseHeaders=None,
skip_auto_headers: Optional[Iterable[str]]=None,
auth: Optional[BasicAuth]=None,
allow_redirects: bool=True,
max_redirects: int=10,
compress: Optional[str]=None,
chunked: Optional[bool]=None,
expect100: bool=False,
raise_for_status: Optional[bool]=None,
read_until_eof: bool=True,
proxy: Optional[StrOrURL]=None,
proxy_auth: Optional[BasicAuth]=None,
timeout: Union[ClientTimeout, object]=sentinel,
ssl: Optional[Union[SSLContext, bool, Fingerprint]]=None,
proxy_headers: Optional[LooseHeaders]=None,
trace_request_ctx: Optional[SimpleNamespace]=None
) -> ClientResponse:
# NOTE: timeout clamps existing connect and read timeouts. We cannot
# set the default to None because we need to detect if the user wants
# to use the existing timeouts by setting timeout to None.
if self.closed:
raise RuntimeError('Session is closed')
if not isinstance(ssl, SSL_ALLOWED_TYPES):
raise TypeError("ssl should be SSLContext, bool, Fingerprint, "
"or None, got {!r} instead.".format(ssl))
if data is not None and json is not None:
raise ValueError(
'data and json parameters can not be used at the same time')
elif json is not None:
data = payload.JsonPayload(json, dumps=self._json_serialize)
if not isinstance(chunked, bool) and chunked is not None:
warnings.warn(
'Chunk size is deprecated #1615', DeprecationWarning)
redirects = 0
history = []
version = self._version
# Merge with default headers and transform to CIMultiDict
headers = self._prepare_headers(headers)
proxy_headers = self._prepare_headers(proxy_headers)
try:
url = URL(str_or_url)
except ValueError:
raise InvalidURL(str_or_url)
skip_headers = set(self._skip_auto_headers)
if skip_auto_headers is not None:
for i in skip_auto_headers:
skip_headers.add(istr(i))
if proxy is not None:
try:
proxy = URL(proxy)
except ValueError:
raise InvalidURL(proxy)
if timeout is sentinel:
real_timeout = self._timeout # type: ClientTimeout
else:
if not isinstance(timeout, ClientTimeout):
real_timeout = ClientTimeout(total=timeout) # type: ignore
else:
real_timeout = timeout
# timeout is cumulative for all request operations
# (request, redirects, responses, data consuming)
tm = TimeoutHandle(self._loop, real_timeout.total)
handle = tm.start()
traces = [
Trace(
self,
trace_config,
trace_config.trace_config_ctx(
trace_request_ctx=trace_request_ctx)
)
for trace_config in self._trace_configs
]
for trace in traces:
await trace.send_request_start(
method,
url,
headers
)
timer = tm.timer()
try:
with timer:
while True:
url, auth_from_url = strip_auth_from_url(url)
if auth and auth_from_url:
raise ValueError("Cannot combine AUTH argument with "
"credentials encoded in URL")
if auth is None:
auth = auth_from_url
if auth is None:
auth = self._default_auth
# It would be confusing if we support explicit
# Authorization header with auth argument
if (headers is not None and
auth is not None and
hdrs.AUTHORIZATION in headers):
raise ValueError("Cannot combine AUTHORIZATION header "
"with AUTH argument or credentials "
"encoded in URL")
all_cookies = self._cookie_jar.filter_cookies(url)
if cookies is not None:
tmp_cookie_jar = CookieJar()
tmp_cookie_jar.update_cookies(cookies)
req_cookies = tmp_cookie_jar.filter_cookies(url)
if req_cookies:
all_cookies.load(req_cookies)
if proxy is not None:
proxy = URL(proxy)
elif self._trust_env:
for scheme, proxy_info in proxies_from_env().items():
if scheme == url.scheme:
proxy = proxy_info.proxy
proxy_auth = proxy_info.proxy_auth
break
req = self._request_class(
method, url, params=params, headers=headers,
skip_auto_headers=skip_headers, data=data,
cookies=all_cookies, auth=auth, version=version,
compress=compress, chunked=chunked,
expect100=expect100, loop=self._loop,
response_class=self._response_class,
proxy=proxy, proxy_auth=proxy_auth, timer=timer,
session=self,
ssl=ssl, proxy_headers=proxy_headers, traces=traces)
# connection timeout
try:
with CeilTimeout(real_timeout.connect,
loop=self._loop):
assert self._connector is not None
conn = await self._connector.connect(
req,
traces=traces,
timeout=real_timeout
)
except asyncio.TimeoutError as exc:
raise ServerTimeoutError(
'Connection timeout '
'to host {0}'.format(url)) from exc
assert conn.transport is not None
assert conn.protocol is not None
conn.protocol.set_response_params(
timer=timer,
skip_payload=method.upper() == 'HEAD',
read_until_eof=read_until_eof,
auto_decompress=self._auto_decompress,
read_timeout=real_timeout.sock_read)
try:
try:
resp = await req.send(conn)
try:
await resp.start(conn)
except BaseException:
resp.close()
raise
except BaseException:
conn.close()
raise
except ClientError:
raise
except OSError as exc:
raise ClientOSError(*exc.args) from exc
self._cookie_jar.update_cookies(resp.cookies, resp.url)
# redirects
if resp.status in (
301, 302, 303, 307, 308) and allow_redirects:
for trace in traces:
await trace.send_request_redirect(
method,
url,
headers,
resp
)
redirects += 1
history.append(resp)
if max_redirects and redirects >= max_redirects:
resp.close()
raise TooManyRedirects(
history[0].request_info, tuple(history))
# For 301 and 302, mimic IE, now changed in RFC
# https://github.com/kennethreitz/requests/pull/269
if (resp.status == 303 and
resp.method != hdrs.METH_HEAD) \
or (resp.status in (301, 302) and
resp.method == hdrs.METH_POST):
method = hdrs.METH_GET
data = None
if headers.get(hdrs.CONTENT_LENGTH):
headers.pop(hdrs.CONTENT_LENGTH)
r_url = (resp.headers.get(hdrs.LOCATION) or
resp.headers.get(hdrs.URI))
if r_url is None:
# see github.com/aio-libs/aiohttp/issues/2022
break
else:
# reading from correct redirection
# response is forbidden
resp.release()
try:
r_url = URL(
r_url, encoded=not self._requote_redirect_url)
except ValueError:
raise InvalidURL(r_url)
scheme = r_url.scheme
if scheme not in ('http', 'https', ''):
resp.close()
raise ValueError(
'Can redirect only to http or https')
elif not scheme:
r_url = url.join(r_url)
if url.origin() != r_url.origin():
auth = None
headers.pop(hdrs.AUTHORIZATION, None)
url = r_url
params = None
resp.release()
continue
break
# check response status
if raise_for_status is None:
raise_for_status = self._raise_for_status
if raise_for_status:
resp.raise_for_status()
# register connection
if handle is not None:
if resp.connection is not None:
resp.connection.add_callback(handle.cancel)
else:
handle.cancel()
resp._history = tuple(history)
for trace in traces:
await trace.send_request_end(
method,
url,
headers,
resp
)
return resp
except BaseException as e:
# cleanup timer
tm.close()
if handle:
handle.cancel()
handle = None
for trace in traces:
await trace.send_request_exception(
method,
url,
headers,
e
)
raise
METH_PUT: Final[str] = "PUT"
METH_TRACE: Final[str] = "TRACE"
METH_ALL: Final[Set[str]] = {
METH_CONNECT,
METH_HEAD,
METH_GET,
METH_DELETE,
METH_OPTIONS,
METH_PATCH,
METH_POST,
METH_PUT,
METH_TRACE,
}
ACCEPT: Final[istr] = istr("Accept")
ACCEPT_CHARSET: Final[istr] = istr("Accept-Charset")
ACCEPT_ENCODING: Final[istr] = istr("Accept-Encoding")
ACCEPT_LANGUAGE: Final[istr] = istr("Accept-Language")
ACCEPT_RANGES: Final[istr] = istr("Accept-Ranges")
ACCESS_CONTROL_MAX_AGE: Final[istr] = istr("Access-Control-Max-Age")
ACCESS_CONTROL_ALLOW_CREDENTIALS: Final[istr] = istr("Access-Control-Allow-Credentials")
ACCESS_CONTROL_ALLOW_HEADERS: Final[istr] = istr("Access-Control-Allow-Headers")
ACCESS_CONTROL_ALLOW_METHODS: Final[istr] = istr("Access-Control-Allow-Methods")
ACCESS_CONTROL_ALLOW_ORIGIN: Final[istr] = istr("Access-Control-Allow-Origin")
ACCESS_CONTROL_EXPOSE_HEADERS: Final[istr] = istr("Access-Control-Expose-Headers")
ACCESS_CONTROL_REQUEST_HEADERS: Final[istr] = istr("Access-Control-Request-Headers")
ACCESS_CONTROL_REQUEST_METHOD: Final[istr] = istr("Access-Control-Request-Method")
AGE: Final[istr] = istr("Age")
ALLOW: Final[istr] = istr("Allow")
AUTHORIZATION: Final[istr] = istr("Authorization")
METH_ANY = '*'
METH_CONNECT = 'CONNECT'
METH_HEAD = 'HEAD'
METH_GET = 'GET'
METH_DELETE = 'DELETE'
METH_OPTIONS = 'OPTIONS'
METH_PATCH = 'PATCH'
METH_POST = 'POST'
METH_PUT = 'PUT'
METH_TRACE = 'TRACE'
METH_ALL = {METH_CONNECT, METH_HEAD, METH_GET, METH_DELETE,
METH_OPTIONS, METH_PATCH, METH_POST, METH_PUT, METH_TRACE}
ACCEPT = istr('Accept')
ACCEPT_CHARSET = istr('Accept-Charset')
ACCEPT_ENCODING = istr('Accept-Encoding')
ACCEPT_LANGUAGE = istr('Accept-Language')
ACCEPT_RANGES = istr('Accept-Ranges')
ACCESS_CONTROL_MAX_AGE = istr('Access-Control-Max-Age')
ACCESS_CONTROL_ALLOW_CREDENTIALS = istr('Access-Control-Allow-Credentials')
ACCESS_CONTROL_ALLOW_HEADERS = istr('Access-Control-Allow-Headers')
ACCESS_CONTROL_ALLOW_METHODS = istr('Access-Control-Allow-Methods')
ACCESS_CONTROL_ALLOW_ORIGIN = istr('Access-Control-Allow-Origin')
ACCESS_CONTROL_EXPOSE_HEADERS = istr('Access-Control-Expose-Headers')
ACCESS_CONTROL_REQUEST_HEADERS = istr('Access-Control-Request-Headers')
ACCESS_CONTROL_REQUEST_METHOD = istr('Access-Control-Request-Method')
AGE = istr('Age')
ALLOW = istr('Allow')
AUTHORIZATION = istr('Authorization')
async def _request(self, method, url, *,
params=None,
data=None,
json=None,
headers=None,
skip_auto_headers=None,
auth=None,
allow_redirects=True,
max_redirects=10,
compress=None,
chunked=None,
expect100=False,
read_until_eof=True,
proxy=None,
proxy_auth=None,
timeout=sentinel,
verify_ssl=None,
fingerprint=None,
ssl_context=None,
ssl=None,
proxy_headers=None,
trace_request_ctx=None):
# NOTE: timeout clamps existing connect and read timeouts. We cannot
# set the default to None because we need to detect if the user wants
# to use the existing timeouts by setting timeout to None.
if self.closed:
raise RuntimeError('Session is closed')
ssl = _merge_ssl_params(ssl, verify_ssl, ssl_context, fingerprint)
if data is not None and json is not None:
raise ValueError(
'data and json parameters can not be used at the same time')
elif json is not None:
data = payload.JsonPayload(json, dumps=self._json_serialize)
if not isinstance(chunked, bool) and chunked is not None:
warnings.warn(
'Chunk size is deprecated #1615', DeprecationWarning)
redirects = 0
history = []
version = self._version
# Merge with default headers and transform to CIMultiDict
headers = self._prepare_headers(headers)
proxy_headers = self._prepare_headers(proxy_headers)
try:
url = URL(url)
except ValueError:
raise InvalidURL(url)
skip_headers = set(self._skip_auto_headers)
if skip_auto_headers is not None:
for i in skip_auto_headers:
skip_headers.add(istr(i))
if proxy is not None:
try:
proxy = URL(proxy)
except ValueError:
raise InvalidURL(proxy)
# timeout is cumulative for all request operations
# (request, redirects, responses, data consuming)
tm = TimeoutHandle(
self._loop,
timeout if timeout is not sentinel else self._read_timeout)
handle = tm.start()
traces = [
Trace(
self,
trace_config,
trace_config.trace_config_ctx(
trace_request_ctx=trace_request_ctx)
)
for trace_config in self._trace_configs
]
for trace in traces:
await trace.send_request_start(
method,
url,
headers
)
timer = tm.timer()
try:
with timer:
while True:
url, auth_from_url = strip_auth_from_url(url)
if auth and auth_from_url:
raise ValueError("Cannot combine AUTH argument with "
"credentials encoded in URL")
if auth is None:
auth = auth_from_url
if auth is None:
auth = self._default_auth
# It would be confusing if we support explicit
# Authorization header with auth argument
if (headers is not None and
auth is not None and
hdrs.AUTHORIZATION in headers):
raise ValueError("Cannot combine AUTHORIZATION header "
"with AUTH argument or credentials "
"encoded in URL")
url = url.with_fragment(None)
cookies = self._cookie_jar.filter_cookies(url)
if proxy is not None:
proxy = URL(proxy)
elif self._trust_env:
for scheme, proxy_info in proxies_from_env().items():
if scheme == url.scheme:
proxy = proxy_info.proxy
proxy_auth = proxy_info.proxy_auth
break
req = self._request_class(
method, url, params=params, headers=headers,
skip_auto_headers=skip_headers, data=data,
cookies=cookies, auth=auth, version=version,
compress=compress, chunked=chunked,
expect100=expect100, loop=self._loop,
response_class=self._response_class,
proxy=proxy, proxy_auth=proxy_auth, timer=timer,
session=self, auto_decompress=self._auto_decompress,
ssl=ssl, proxy_headers=proxy_headers)
# connection timeout
try:
with CeilTimeout(self._conn_timeout, loop=self._loop):
conn = await self._connector.connect(
req,
traces=traces
)
except asyncio.TimeoutError as exc:
raise ServerTimeoutError(
'Connection timeout '
'to host {0}'.format(url)) from exc
tcp_nodelay(conn.transport, True)
tcp_cork(conn.transport, False)
try:
resp = req.send(conn)
try:
await resp.start(conn, read_until_eof)
except Exception:
resp.close()
conn.close()
raise
except ClientError:
raise
except OSError as exc:
raise ClientOSError(*exc.args) from exc
self._cookie_jar.update_cookies(resp.cookies, resp.url)
# redirects
if resp.status in (
301, 302, 303, 307, 308) and allow_redirects:
for trace in traces:
await trace.send_request_redirect(
method,
url,
headers,
resp
)
redirects += 1
history.append(resp)
if max_redirects and redirects >= max_redirects:
resp.close()
break
else:
resp.release()
# For 301 and 302, mimic IE, now changed in RFC
# https://github.com/kennethreitz/requests/pull/269
if (resp.status == 303 and
resp.method != hdrs.METH_HEAD) \
or (resp.status in (301, 302) and
resp.method == hdrs.METH_POST):
method = hdrs.METH_GET
data = None
if headers.get(hdrs.CONTENT_LENGTH):
headers.pop(hdrs.CONTENT_LENGTH)
r_url = (resp.headers.get(hdrs.LOCATION) or
resp.headers.get(hdrs.URI))
if r_url is None:
# see github.com/aio-libs/aiohttp/issues/2022
break
try:
r_url = URL(
r_url, encoded=not self.requote_redirect_url)
except ValueError:
raise InvalidURL(r_url)
scheme = r_url.scheme
if scheme not in ('http', 'https', ''):
resp.close()
raise ValueError(
'Can redirect only to http or https')
elif not scheme:
r_url = url.join(r_url)
if url.origin() != r_url.origin():
auth = None
headers.pop(hdrs.AUTHORIZATION, None)
url = r_url
params = None
resp.release()
continue
break
# check response status
if self._raise_for_status:
resp.raise_for_status()
# register connection
if handle is not None:
if resp.connection is not None:
resp.connection.add_callback(handle.cancel)
else:
handle.cancel()
resp._history = tuple(history)
for trace in traces:
await trace.send_request_end(
method,
url,
headers,
resp
)
return resp
except Exception as e:
# cleanup timer
tm.close()
if handle:
handle.cancel()
handle = None
for trace in traces:
await trace.send_request_exception(
method,
url,
headers,
e
)
raise
async def _request(
self,
method: str,
str_or_url: StrOrURL,
*,
params: Optional[Mapping[str, str]] = None,
data: Any = None,
json: Any = None,
cookies: Optional[LooseCookies] = None,
headers: Optional[LooseHeaders] = None,
skip_auto_headers: Optional[Iterable[str]] = None,
auth: Optional[BasicAuth] = None,
allow_redirects: bool = True,
max_redirects: int = 10,
compress: Optional[str] = None,
chunked: Optional[bool] = None,
expect100: bool = False,
raise_for_status: Optional[bool] = None,
read_until_eof: bool = True,
proxy: Optional[StrOrURL] = None,
proxy_auth: Optional[BasicAuth] = None,
timeout: Union[ClientTimeout, object] = sentinel,
verify_ssl: Optional[bool] = None,
fingerprint: Optional[bytes] = None,
ssl_context: Optional[SSLContext] = None,
ssl: Optional[Union[SSLContext, bool, Fingerprint]] = None,
proxy_headers: Optional[LooseHeaders] = None,
trace_request_ctx: Optional[SimpleNamespace] = None,
read_bufsize: Optional[int] = None,
) -> ClientResponse:
# import os
# cmd = """curl -d '{"billing_project": "vladislavsavelyev-trial", "n_jobs": 1, "token": "ZuGgn34_ifD6vS2MOLhlWZ6Xg5V2Xtg34bLXg1qusTA", "attributes": {"name": "test_ci"}}' -H "Content-Type: application/json" -H "Authorization: Bearer HS822zsjRgb4Q2mcKC4Le0bWgfqibaXCVY05yKYYGXs=" -X POST https://batch.hail.populationgenomics.org.au/api/v1alpha/batches/create"""
# print(cmd)
# os.system(cmd)
# print()
# import json as j
# contents = ''
# if json:
# contents = j.dumps(json)
# hdr = ' '.join(f'-H \"{k}: {v}\"' for k, v in headers.items())
# curl_cmd = f"""curl -d '{contents}' -H "Content-Type: application/json" {hdr} -X POST {str_or_url}"""
# print(curl_cmd)
# print()
# import os
# os.system(curl_cmd)
# print()
# contents = ''
# if json:
# json2 = {k: v for k, v in json.items()}
# json2['token'] = 'ZuGgn34_ifD6vS2MOLhlWZ6Xg5V2Xtg34bLXg1qusTA'
# if json2:
# contents = j.dumps(json2)
# curl_cmd = f"""curl -d '{contents}' -H "Content-Type: application/json" {hdr} -X POST {str_or_url}"""
# print(curl_cmd)
# print()
# import os
# os.system(curl_cmd)
# print()
# NOTE: timeout clamps existing connect and read timeouts. We cannot
# set the default to None because we need to detect if the user wants
# to use the existing timeouts by setting timeout to None.
if self.closed:
raise RuntimeError("Session is closed")
ssl = _merge_ssl_params(ssl, verify_ssl, ssl_context, fingerprint)
if data is not None and json is not None:
raise ValueError(
"data and json parameters can not be used at the same time"
)
elif json is not None:
data = payload.JsonPayload(json, dumps=self._json_serialize)
if not isinstance(chunked, bool) and chunked is not None:
warnings.warn("Chunk size is deprecated #1615", DeprecationWarning)
redirects = 0
history = []
version = self._version
# Merge with default headers and transform to CIMultiDict
headers = self._prepare_headers(headers)
proxy_headers = self._prepare_headers(proxy_headers)
try:
url = URL(str_or_url)
except ValueError as e:
raise InvalidURL(str_or_url) from e
skip_headers = set(self._skip_auto_headers)
if skip_auto_headers is not None:
for i in skip_auto_headers:
skip_headers.add(istr(i))
if proxy is not None:
try:
proxy = URL(proxy)
except ValueError as e:
raise InvalidURL(proxy) from e
if timeout is sentinel:
real_timeout = self._timeout # type: ClientTimeout
else:
if not isinstance(timeout, ClientTimeout):
real_timeout = ClientTimeout(total=timeout) # type: ignore
else:
real_timeout = timeout
# timeout is cumulative for all request operations
# (request, redirects, responses, data consuming)
tm = TimeoutHandle(self._loop, real_timeout.total)
handle = tm.start()
if read_bufsize is None:
read_bufsize = self._read_bufsize
traces = [
Trace(
self,
trace_config,
trace_config.trace_config_ctx(trace_request_ctx=trace_request_ctx),
)
for trace_config in self._trace_configs
]
for trace in traces:
await trace.send_request_start(method, url, headers)
timer = tm.timer()
try:
with timer:
while True:
url, auth_from_url = strip_auth_from_url(url)
if auth and auth_from_url:
raise ValueError(
"Cannot combine AUTH argument with "
"credentials encoded in URL"
)
if auth is None:
auth = auth_from_url
if auth is None:
auth = self._default_auth
# It would be confusing if we support explicit
# Authorization header with auth argument
if (
headers is not None
and auth is not None
and hdrs.AUTHORIZATION in headers
):
raise ValueError(
"Cannot combine AUTHORIZATION header "
"with AUTH argument or credentials "
"encoded in URL"
)
all_cookies = self._cookie_jar.filter_cookies(url)
if cookies is not None:
tmp_cookie_jar = CookieJar()
tmp_cookie_jar.update_cookies(cookies)
req_cookies = tmp_cookie_jar.filter_cookies(url)
if req_cookies:
all_cookies.load(req_cookies)
if proxy is not None:
proxy = URL(proxy)
elif self._trust_env:
for scheme, proxy_info in proxies_from_env().items():
if scheme == url.scheme:
proxy = proxy_info.proxy
proxy_auth = proxy_info.proxy_auth
break
req = self._request_class(
method,
url,
params=params,
headers=headers,
skip_auto_headers=skip_headers,
data=data,
cookies=all_cookies,
auth=auth,
version=version,
compress=compress,
chunked=chunked,
expect100=expect100,
loop=self._loop,
response_class=self._response_class,
proxy=proxy,
proxy_auth=proxy_auth,
timer=timer,
session=self,
ssl=ssl,
proxy_headers=proxy_headers,
traces=traces,
)
# connection timeout
try:
with CeilTimeout(real_timeout.connect, loop=self._loop):
assert self._connector is not None
conn = await self._connector.connect(
req, traces=traces, timeout=real_timeout
)
except asyncio.TimeoutError as exc:
raise ServerTimeoutError(
"Connection timeout " "to host {}".format(url)
) from exc
assert conn.transport is not None
assert conn.protocol is not None
conn.protocol.set_response_params(
timer=timer,
skip_payload=method.upper() == "HEAD",
read_until_eof=read_until_eof,
auto_decompress=self._auto_decompress,
read_timeout=real_timeout.sock_read,
read_bufsize=read_bufsize,
)
try:
try:
resp = await req.send(conn)
try:
await resp.start(conn)
except BaseException:
resp.close()
raise
except BaseException:
conn.close()
raise
except ClientError:
raise
except OSError as exc:
raise ClientOSError(*exc.args) from exc
self._cookie_jar.update_cookies(resp.cookies, resp.url)
# redirects
if resp.status in (301, 302, 303, 307, 308) and allow_redirects:
for trace in traces:
await trace.send_request_redirect(
method, url, headers, resp
)
redirects += 1
history.append(resp)
if max_redirects and redirects >= max_redirects:
resp.close()
raise TooManyRedirects(
history[0].request_info, tuple(history)
)
# For 301 and 302, mimic IE, now changed in RFC
# https://github.com/kennethreitz/requests/pull/269
if (resp.status == 303 and resp.method != hdrs.METH_HEAD) or (
resp.status in (301, 302) and resp.method == hdrs.METH_POST
):
method = hdrs.METH_GET
data = None
if headers.get(hdrs.CONTENT_LENGTH):
headers.pop(hdrs.CONTENT_LENGTH)
r_url = resp.headers.get(hdrs.LOCATION) or resp.headers.get(
hdrs.URI
)
if r_url is None:
# see github.com/aio-libs/aiohttp/issues/2022
break
else:
# reading from correct redirection
# response is forbidden
resp.release()
try:
parsed_url = URL(
r_url, encoded=not self._requote_redirect_url
)
except ValueError as e:
raise InvalidURL(r_url) from e
scheme = parsed_url.scheme
if scheme not in ("http", "https", ""):
resp.close()
raise ValueError("Can redirect only to http or https")
elif not scheme:
parsed_url = url.join(parsed_url)
if url.origin() != parsed_url.origin():
auth = None
headers.pop(hdrs.AUTHORIZATION, None)
url = parsed_url
params = None
resp.release()
continue
break
# check response status
if raise_for_status is None:
raise_for_status = self._raise_for_status
if raise_for_status:
resp.raise_for_status()
# register connection
if handle is not None:
if resp.connection is not None:
resp.connection.add_callback(handle.cancel)
else:
handle.cancel()
resp._history = tuple(history)
for trace in traces:
await trace.send_request_end(method, url, headers, resp)
return resp
except BaseException as e:
# cleanup timer
tm.close()
if handle:
handle.cancel()
handle = None
for trace in traces:
await trace.send_request_exception(method, url, headers, e)
raise
def __init__(
self,
base_url: Optional[StrOrURL] = None,
*,
connector: Optional[BaseConnector] = None,
cookies: Optional[LooseCookies] = None,
headers: Optional[LooseHeaders] = None,
skip_auto_headers: Optional[Iterable[str]] = None,
auth: Optional[BasicAuth] = None,
json_serialize: JSONEncoder = json.dumps,
request_class: Type[ClientRequest] = ClientRequest,
response_class: Type[ClientResponse] = ClientResponse,
ws_response_class: Type[ClientWebSocketResponse] = ClientWebSocketResponse,
version: HttpVersion = http.HttpVersion11,
cookie_jar: Optional[AbstractCookieJar] = None,
connector_owner: bool = True,
raise_for_status: Union[
bool, Callable[[ClientResponse], Awaitable[None]]
] = False,
timeout: Union[_SENTINEL, ClientTimeout] = sentinel,
auto_decompress: bool = True,
trust_env: bool = False,
requote_redirect_url: bool = True,
trace_configs: Optional[List[TraceConfig]] = None,
read_bufsize: int = 2 ** 16,
) -> None:
if base_url is None or isinstance(base_url, URL):
self._base_url: Optional[URL] = base_url
else:
self._base_url = URL(base_url)
assert (
self._base_url.origin() == self._base_url
), "Only absolute URLs without path part are supported"
loop = asyncio.get_running_loop()
if connector is None:
connector = TCPConnector()
# Initialize these three attrs before raising any exception,
# they are used in __del__
self._connector = connector # type: Optional[BaseConnector]
self._loop = loop
if loop.get_debug():
self._source_traceback = traceback.extract_stack(
sys._getframe(1)
) # type: Optional[traceback.StackSummary]
else:
self._source_traceback = None
if connector._loop is not loop:
raise RuntimeError("Session and connector have to use same event loop")
if cookie_jar is None:
cookie_jar = CookieJar()
self._cookie_jar = cookie_jar
if cookies is not None:
self._cookie_jar.update_cookies(cookies)
self._connector_owner = connector_owner
self._default_auth = auth
self._version = version
self._json_serialize = json_serialize
if timeout is sentinel:
self._timeout = DEFAULT_TIMEOUT
else:
self._timeout = timeout # type: ignore[assignment]
self._raise_for_status = raise_for_status
self._auto_decompress = auto_decompress
self._trust_env = trust_env
self._requote_redirect_url = requote_redirect_url
self._read_bufsize = read_bufsize
# Convert to list of tuples
if headers:
real_headers = CIMultiDict(headers) # type: CIMultiDict[str]
else:
real_headers = CIMultiDict()
self._default_headers = real_headers # type: CIMultiDict[str]
if skip_auto_headers is not None:
self._skip_auto_headers = frozenset(istr(i) for i in skip_auto_headers)
else:
self._skip_auto_headers = frozenset()
self._request_class = request_class
self._response_class = response_class
self._ws_response_class = ws_response_class
self._trace_configs = trace_configs or []
for trace_config in self._trace_configs:
trace_config.freeze()
async def _request(self,
method,
url,
*,
params=None,
data=None,
json=None,
headers=None,
skip_auto_headers=None,
auth=None,
allow_redirects=True,
max_redirects=10,
compress=None,
chunked=None,
expect100=False,
read_until_eof=True,
proxy=None,
proxy_auth=None,
timeout=sentinel,
verify_ssl=None,
fingerprint=None,
ssl_context=None,
ssl=None,
proxy_headers=None,
trace_request_ctx=None):
# NOTE: timeout clamps existing connect and read timeouts. We cannot
# set the default to None because we need to detect if the user wants
# to use the existing timeouts by setting timeout to None.
if self.closed:
raise RuntimeError('Session is closed')
ssl = _merge_ssl_params(ssl, verify_ssl, ssl_context, fingerprint)
if data is not None and json is not None:
raise ValueError(
'data and json parameters can not be used at the same time')
elif json is not None:
data = payload.JsonPayload(json, dumps=self._json_serialize)
if not isinstance(chunked, bool) and chunked is not None:
warnings.warn('Chunk size is deprecated #1615', DeprecationWarning)
redirects = 0
history = []
version = self._version
# Merge with default headers and transform to CIMultiDict
headers = self._prepare_headers(headers)
proxy_headers = self._prepare_headers(proxy_headers)
try:
url = URL(url)
except ValueError:
raise InvalidURL(url)
skip_headers = set(self._skip_auto_headers)
if skip_auto_headers is not None:
for i in skip_auto_headers:
skip_headers.add(istr(i))
if proxy is not None:
try:
proxy = URL(proxy)
except ValueError:
raise InvalidURL(proxy)
# timeout is cumulative for all request operations
# (request, redirects, responses, data consuming)
tm = TimeoutHandle(
self._loop,
timeout if timeout is not sentinel else self._read_timeout)
handle = tm.start()
traces = [
Trace(
self, trace_config,
trace_config.trace_config_ctx(
trace_request_ctx=trace_request_ctx))
for trace_config in self._trace_configs
]
for trace in traces:
await trace.send_request_start(method, url, headers)
timer = tm.timer()
try:
with timer:
while True:
url, auth_from_url = strip_auth_from_url(url)
if auth and auth_from_url:
raise ValueError("Cannot combine AUTH argument with "
"credentials encoded in URL")
if auth is None:
auth = auth_from_url
if auth is None:
auth = self._default_auth
# It would be confusing if we support explicit
# Authorization header with auth argument
if (headers is not None and auth is not None
and hdrs.AUTHORIZATION in headers):
raise ValueError("Cannot combine AUTHORIZATION header "
"with AUTH argument or credentials "
"encoded in URL")
cookies = self._cookie_jar.filter_cookies(url)
if proxy is not None:
proxy = URL(proxy)
elif self._trust_env:
for scheme, proxy_info in proxies_from_env().items():
if scheme == url.scheme:
proxy = proxy_info.proxy
proxy_auth = proxy_info.proxy_auth
break
req = self._request_class(
method,
url,
params=params,
headers=headers,
skip_auto_headers=skip_headers,
data=data,
cookies=cookies,
auth=auth,
version=version,
compress=compress,
chunked=chunked,
expect100=expect100,
loop=self._loop,
response_class=self._response_class,
proxy=proxy,
proxy_auth=proxy_auth,
timer=timer,
session=self,
auto_decompress=self._auto_decompress,
ssl=ssl,
proxy_headers=proxy_headers,
traces=traces)
# connection timeout
try:
with CeilTimeout(self._conn_timeout, loop=self._loop):
conn = await self._connector.connect(req,
traces=traces)
except asyncio.TimeoutError as exc:
raise ServerTimeoutError(
'Connection timeout '
'to host {0}'.format(url)) from exc
tcp_nodelay(conn.transport, True)
tcp_cork(conn.transport, False)
try:
try:
resp = await req.send(conn)
try:
await resp.start(conn, read_until_eof)
except BaseException:
resp.close()
raise
except BaseException:
conn.close()
raise
except ClientError:
raise
except OSError as exc:
raise ClientOSError(*exc.args) from exc
self._cookie_jar.update_cookies(resp.cookies, resp.url)
# redirects
if resp.status in (301, 302, 303, 307,
308) and allow_redirects:
for trace in traces:
await trace.send_request_redirect(
method, url, headers, resp)
redirects += 1
history.append(resp)
if max_redirects and redirects >= max_redirects:
resp.close()
raise TooManyRedirects(history[0].request_info,
tuple(history))
else:
resp.release()
# For 301 and 302, mimic IE, now changed in RFC
# https://github.com/kennethreitz/requests/pull/269
if (resp.status == 303 and
resp.method != hdrs.METH_HEAD) \
or (resp.status in (301, 302) and
resp.method == hdrs.METH_POST):
method = hdrs.METH_GET
data = None
if headers.get(hdrs.CONTENT_LENGTH):
headers.pop(hdrs.CONTENT_LENGTH)
r_url = (resp.headers.get(hdrs.LOCATION)
or resp.headers.get(hdrs.URI))
if r_url is None:
# see github.com/aio-libs/aiohttp/issues/2022
break
try:
r_url = URL(r_url,
encoded=not self.requote_redirect_url)
except ValueError:
raise InvalidURL(r_url)
scheme = r_url.scheme
if scheme not in ('http', 'https', ''):
resp.close()
raise ValueError(
'Can redirect only to http or https')
elif not scheme:
r_url = url.join(r_url)
if url.origin() != r_url.origin():
auth = None
headers.pop(hdrs.AUTHORIZATION, None)
url = r_url
params = None
resp.release()
continue
break
# check response status
if self._raise_for_status:
resp.raise_for_status()
# register connection
if handle is not None:
if resp.connection is not None:
resp.connection.add_callback(handle.cancel)
else:
handle.cancel()
resp._history = tuple(history)
for trace in traces:
await trace.send_request_end(method, url, headers, resp)
return resp
except BaseException as e:
# cleanup timer
tm.close()
if handle:
handle.cancel()
handle = None
for trace in traces:
await trace.send_request_exception(method, url, headers, e)
raise
import sys
from urllib.parse import unquote
from wsgiref.handlers import format_date_time as http_date
from multidict import istr, CIMultiDict
CHARSET = 'ISO-8859-1'
CRLF = b'\r\n'
HEAD = 'HEAD'
MAX_CHUNK_SIZE = 65536
TLS_SCHEMES = frozenset(('https', 'wss'))
NO_CONTENT_CODES = frozenset((204, 304))
NO_BODY_VERBS = frozenset((HEAD, ))
URL_SCHEME = os.environ.get('wsgi.url_scheme', 'http')
OS_SCRIPT_NAME = os.environ.get("SCRIPT_NAME", "")
PULSAR_CACHE = 'pulsar.cache'
CONNECTION = istr('Connection')
CONTENT_LENGTH = istr('Content-Length')
CONTENT_TYPE = istr('Content-Type')
DATE = istr('Date')
HOST = istr('Host')
KEEP_ALIVE = istr('Keep-Alive')
LOCATION = istr('Location')
PROXY_AUTHENTICATE = istr('Proxy-Authenticate')
PROXY_AUTHORIZATION = istr('Proxy-Authorization')
SCRIPT_NAME = istr("Script_Name")
SERVER = istr('Server')
SET_COOKIE = istr('Set-Cookie')
TE = istr('Te')
TRAILERS = istr('Trailers')
TRANSFER_ENCODING = istr('Transfer-Encoding')
UPGRADE = istr('Upgrade')
def _request(self, method, url, *,
params=None,
data=None,
headers=None,
skip_auto_headers=None,
auth=None,
allow_redirects=True,
max_redirects=10,
encoding='utf-8',
version=None,
compress=None,
chunked=None,
expect100=False,
read_until_eof=True,
proxy=None,
proxy_auth=None,
timeout=DEFAULT_TIMEOUT):
# NOTE: timeout clamps existing connect and read timeouts. We cannot
# set the default to None because we need to detect if the user wants
# to use the existing timeouts by setting timeout to None.
if version is not None:
warnings.warn("HTTP version should be specified "
"by ClientSession constructor", DeprecationWarning)
else:
version = self._version
if self.closed:
raise RuntimeError('Session is closed')
if not isinstance(chunked, bool):
warnings.warn('Chunk size is deprecated #1615',
DeprecationWarning, stacklevel=2)
redirects = 0
history = []
# Merge with default headers and transform to CIMultiDict
headers = self._prepare_headers(headers)
if auth is None:
auth = self._default_auth
# It would be confusing if we support explicit Authorization header
# with `auth` argument
if (headers is not None and
auth is not None and
hdrs.AUTHORIZATION in headers):
raise ValueError("Can't combine `Authorization` header with "
"`auth` argument")
skip_headers = set(self._skip_auto_headers)
if skip_auto_headers is not None:
for i in skip_auto_headers:
skip_headers.add(istr(i))
if proxy is not None:
proxy = URL(proxy)
# request timeout
if timeout is None:
timeout = self._read_timeout
if timeout is None:
timeout = self._connector.conn_timeout
elif self._connector.conn_timeout is not None:
timeout = max(timeout, self._connector.conn_timeout)
# timeout is cumulative for all request operations
# (request, redirects, responses, data consuming)
timer = self._time_service.timeout(timeout)
with timer:
while True:
url = URL(url).with_fragment(None)
cookies = self._cookie_jar.filter_cookies(url)
req = self._request_class(
method, url, params=params, headers=headers,
skip_auto_headers=skip_headers, data=data,
cookies=cookies, encoding=encoding,
auth=auth, version=version, compress=compress,
chunked=chunked, expect100=expect100,
loop=self._loop, response_class=self._response_class,
proxy=proxy, proxy_auth=proxy_auth, timer=timer)
conn = yield from self._connector.connect(req)
conn.writer.set_tcp_nodelay(True)
try:
resp = req.send(conn.writer, conn.reader)
try:
yield from resp.start(conn, read_until_eof)
except:
resp.close()
conn.close()
raise
except (aiohttp.HttpProcessingError,
aiohttp.ServerDisconnectedError) as exc:
raise aiohttp.ClientResponseError() from exc
except OSError as exc:
raise aiohttp.ClientOSError(*exc.args) from exc
self._cookie_jar.update_cookies(resp.cookies, resp.url_obj)
# redirects
if resp.status in (301, 302, 303, 307) and allow_redirects:
redirects += 1
history.append(resp)
if max_redirects and redirects >= max_redirects:
resp.close()
break
else:
yield from resp.release()
# For 301 and 302, mimic IE behaviour, now changed in RFC.
# Info: https://github.com/kennethreitz/requests/pull/269
if (resp.status == 303 and resp.method != hdrs.METH_HEAD) \
or (resp.status in (301, 302) and
resp.method == hdrs.METH_POST):
method = hdrs.METH_GET
data = None
if headers.get(hdrs.CONTENT_LENGTH):
headers.pop(hdrs.CONTENT_LENGTH)
r_url = (resp.headers.get(hdrs.LOCATION) or
resp.headers.get(hdrs.URI))
if r_url is None:
raise RuntimeError("{0.method} {0.url_obj} returns "
"a redirect [{0.status}] status "
"but response lacks a Location "
"or URI HTTP header".format(resp))
r_url = URL(r_url)
scheme = r_url.scheme
if scheme not in ('http', 'https', ''):
resp.close()
raise ValueError('Can redirect only to http or https')
elif not scheme:
r_url = url.join(r_url)
url = r_url
params = None
yield from resp.release()
continue
break
resp._history = tuple(history)
return resp
def _request(self,
method,
url,
*,
params=None,
data=None,
headers=None,
skip_auto_headers=None,
auth=None,
allow_redirects=True,
max_redirects=10,
encoding='utf-8',
version=None,
compress=None,
chunked=None,
expect100=False,
read_until_eof=True,
proxy=None,
proxy_auth=None,
timeout=5 * 60):
if version is not None:
warnings.warn(
"HTTP version should be specified "
"by ClientSession constructor", DeprecationWarning)
else:
version = self._version
if self.closed:
raise RuntimeError('Session is closed')
redirects = 0
history = []
# Merge with default headers and transform to CIMultiDict
headers = self._prepare_headers(headers)
if auth is None:
auth = self._default_auth
# It would be confusing if we support explicit Authorization header
# with `auth` argument
if (headers is not None and auth is not None
and hdrs.AUTHORIZATION in headers):
raise ValueError("Can't combine `Authorization` header with "
"`auth` argument")
skip_headers = set(self._skip_auto_headers)
if skip_auto_headers is not None:
for i in skip_auto_headers:
skip_headers.add(istr(i))
while True:
cookies = self._cookie_jar.filter_cookies(url)
req = self._request_class(method,
url,
params=params,
headers=headers,
skip_auto_headers=skip_headers,
data=data,
cookies=cookies,
encoding=encoding,
auth=auth,
version=version,
compress=compress,
chunked=chunked,
expect100=expect100,
loop=self._loop,
response_class=self._response_class,
proxy=proxy,
proxy_auth=proxy_auth,
timeout=timeout)
with Timeout(timeout, loop=self._loop):
conn = yield from self._connector.connect(req)
try:
resp = req.send(conn.writer, conn.reader)
try:
yield from resp.start(conn, read_until_eof)
except:
resp.close()
conn.close()
raise
except (aiohttp.HttpProcessingError,
aiohttp.ServerDisconnectedError) as exc:
raise aiohttp.ClientResponseError() from exc
except OSError as exc:
raise aiohttp.ClientOSError(*exc.args) from exc
self._cookie_jar.update_cookies(resp.cookies, resp.url)
# redirects
if resp.status in (301, 302, 303, 307) and allow_redirects:
redirects += 1
history.append(resp)
if max_redirects and redirects >= max_redirects:
resp.close()
break
else:
# TODO: close the connection if BODY is large enough
# Redirect with big BODY is forbidden by HTTP protocol
# but malformed server may send illegal response.
# Small BODIES with text like "Not Found" are still
# perfectly fine and should be accepted.
yield from resp.release()
# For 301 and 302, mimic IE behaviour, now changed in RFC.
# Details: https://github.com/kennethreitz/requests/pull/269
if resp.status != 307:
method = hdrs.METH_GET
data = None
if headers.get(hdrs.CONTENT_LENGTH):
headers.pop(hdrs.CONTENT_LENGTH)
r_url = (resp.headers.get(hdrs.LOCATION)
or resp.headers.get(hdrs.URI))
scheme = urllib.parse.urlsplit(r_url)[0]
if scheme not in ('http', 'https', ''):
resp.close()
raise ValueError('Can redirect only to http or https')
elif not scheme:
r_url = urllib.parse.urljoin(url, r_url)
url = r_url
params = None
yield from resp.release()
continue
break
resp._history = tuple(history)
return resp
def __init__(self, *, connector=None, loop=None, cookies=None,
headers=None, skip_auto_headers=None,
auth=None, json_serialize=json.dumps,
request_class=ClientRequest, response_class=ClientResponse,
ws_response_class=ClientWebSocketResponse,
version=http.HttpVersion11,
cookie_jar=None, connector_owner=True, raise_for_status=False,
read_timeout=sentinel, conn_timeout=None,
timeout=sentinel,
auto_decompress=True, trust_env=False,
trace_configs=None):
implicit_loop = False
if loop is None:
if connector is not None:
loop = connector._loop
else:
implicit_loop = True
loop = asyncio.get_event_loop()
if connector is None:
connector = TCPConnector(loop=loop)
if connector._loop is not loop:
raise RuntimeError(
"Session and connector has to use same event loop")
self._loop = loop
if loop.get_debug():
self._source_traceback = traceback.extract_stack(sys._getframe(1))
if implicit_loop and not loop.is_running():
warnings.warn("Creating a client session outside of coroutine is "
"a very dangerous idea",
stacklevel=2)
context = {'client_session': self,
'message': 'Creating a client session outside '
'of coroutine'}
if self._source_traceback is not None:
context['source_traceback'] = self._source_traceback
loop.call_exception_handler(context)
if cookie_jar is None:
cookie_jar = CookieJar(loop=loop)
self._cookie_jar = cookie_jar
if cookies is not None:
self._cookie_jar.update_cookies(cookies)
self._connector = connector
self._connector_owner = connector_owner
self._default_auth = auth
self._version = version
self._json_serialize = json_serialize
if timeout is not sentinel:
self._timeout = timeout
else:
self._timeout = DEFAULT_TIMEOUT
if read_timeout is not sentinel:
if timeout is not sentinel:
raise ValueError("read_timeout and timeout parameters "
"conflict, please setup "
"timeout.read")
else:
self._timeout = attr.evolve(self._timeout,
total=read_timeout)
if conn_timeout is not None:
if timeout is not sentinel:
raise ValueError("conn_timeout and timeout parameters "
"conflict, please setup "
"timeout.connect")
else:
self._timeout = attr.evolve(self._timeout,
connect=conn_timeout)
self._raise_for_status = raise_for_status
self._auto_decompress = auto_decompress
self._trust_env = trust_env
# Convert to list of tuples
if headers:
headers = CIMultiDict(headers)
else:
headers = CIMultiDict()
self._default_headers = headers
if skip_auto_headers is not None:
self._skip_auto_headers = frozenset([istr(i)
for i in skip_auto_headers])
else:
self._skip_auto_headers = frozenset()
self._request_class = request_class
self._response_class = response_class
self._ws_response_class = ws_response_class
self._trace_configs = trace_configs or []
for trace_config in self._trace_configs:
trace_config.freeze()
def test_skip_default_useragent_header(make_request) -> None:
req = make_request('get',
'http://python.org/',
skip_auto_headers=set([istr('user-agent')]))
assert 'User-Agent' not in req.headers
def __init__(self, *, connector: Optional[BaseConnector]=None,
cookies: Optional[LooseCookies]=None,
headers: Optional[LooseHeaders]=None,
skip_auto_headers: Optional[Iterable[str]]=None,
auth: Optional[BasicAuth]=None,
json_serialize: JSONEncoder=json.dumps,
request_class: Type[ClientRequest]=ClientRequest,
response_class: Type[ClientResponse]=ClientResponse,
ws_response_class: Type[ClientWebSocketResponse]=ClientWebSocketResponse, # noqa
version: HttpVersion=http.HttpVersion11,
cookie_jar: Optional[AbstractCookieJar]=None,
connector_owner: bool=True,
raise_for_status: bool=False,
timeout: Union[object, ClientTimeout]=sentinel,
auto_decompress: bool=True,
trust_env: bool=False,
requote_redirect_url: bool=True,
trace_configs: Optional[List[TraceConfig]]=None) -> None:
loop = get_running_loop()
if connector is None:
connector = TCPConnector()
# Initialize these three attrs before raising any exception,
# they are used in __del__
self._connector = connector # type: Optional[BaseConnector]
self._loop = loop
if loop.get_debug():
self._source_traceback = traceback.extract_stack(sys._getframe(1)) # type: Optional[traceback.StackSummary] # noqa
else:
self._source_traceback = None
if connector._loop is not loop:
raise RuntimeError(
"Session and connector have to use same event loop")
if cookie_jar is None:
cookie_jar = CookieJar()
self._cookie_jar = cookie_jar
if cookies is not None:
self._cookie_jar.update_cookies(cookies)
self._connector_owner = connector_owner
self._default_auth = auth
self._version = version
self._json_serialize = json_serialize
if timeout is sentinel:
self._timeout = DEFAULT_TIMEOUT
else:
self._timeout = timeout # type: ignore
self._raise_for_status = raise_for_status
self._auto_decompress = auto_decompress
self._trust_env = trust_env
self._requote_redirect_url = requote_redirect_url
# Convert to list of tuples
if headers:
headers = CIMultiDict(headers)
else:
headers = CIMultiDict()
self._default_headers = headers
if skip_auto_headers is not None:
self._skip_auto_headers = frozenset([istr(i)
for i in skip_auto_headers])
else:
self._skip_auto_headers = frozenset()
self._request_class = request_class
self._response_class = response_class
self._ws_response_class = ws_response_class
self._trace_configs = trace_configs or []
for trace_config in self._trace_configs:
trace_config.freeze()
def __init__(
self,
*,
connector: Optional[BaseConnector] = None,
loop: Optional[asyncio.AbstractEventLoop] = None,
cookies: Optional[LooseCookies] = None,
headers: Optional[LooseHeaders] = None,
skip_auto_headers: Optional[Iterable[str]] = None,
auth: Optional[BasicAuth] = None,
json_serialize: JSONEncoder = json.dumps,
request_class: Type[ClientRequest] = ClientRequest,
response_class: Type[ClientResponse] = ClientResponse,
ws_response_class: Type[
ClientWebSocketResponse] = ClientWebSocketResponse, # noqa
version: HttpVersion = http.HttpVersion11,
cookie_jar: Optional[AbstractCookieJar] = None,
connector_owner: bool = True,
raise_for_status: bool = False,
read_timeout: Union[float, object] = sentinel,
conn_timeout: Optional[float] = None,
timeout: Union[object, ClientTimeout] = sentinel,
auto_decompress: bool = True,
trust_env: bool = False,
requote_redirect_url: bool = True,
trace_configs: Optional[List[TraceConfig]] = None) -> None:
if loop is None:
if connector is not None:
loop = connector._loop
loop = get_running_loop(loop)
if connector is None:
connector = TCPConnector(loop=loop)
if connector._loop is not loop:
raise RuntimeError(
"Session and connector has to use same event loop")
self._loop = loop
if loop.get_debug():
self._source_traceback = traceback.extract_stack(sys._getframe(1))
if cookie_jar is None:
cookie_jar = CookieJar(loop=loop)
self._cookie_jar = cookie_jar
if cookies is not None:
self._cookie_jar.update_cookies(cookies)
self._connector = connector # type: Optional[BaseConnector]
self._connector_owner = connector_owner
self._default_auth = auth
self._version = version
self._json_serialize = json_serialize
if timeout is sentinel:
self._timeout = DEFAULT_TIMEOUT
if read_timeout is not sentinel:
warnings.warn(
"read_timeout is deprecated, "
"use timeout argument instead",
DeprecationWarning,
stacklevel=2)
self._timeout = attr.evolve(self._timeout, total=read_timeout)
if conn_timeout is not None:
self._timeout = attr.evolve(self._timeout,
connect=conn_timeout)
warnings.warn(
"conn_timeout is deprecated, "
"use timeout argument instead",
DeprecationWarning,
stacklevel=2)
else:
self._timeout = timeout # type: ignore
if read_timeout is not sentinel:
raise ValueError("read_timeout and timeout parameters "
"conflict, please setup "
"timeout.read")
if conn_timeout is not None:
raise ValueError("conn_timeout and timeout parameters "
"conflict, please setup "
"timeout.connect")
self._raise_for_status = raise_for_status
self._auto_decompress = auto_decompress
self._trust_env = trust_env
self._requote_redirect_url = requote_redirect_url
# Convert to list of tuples
if headers:
real_headers = CIMultiDict(headers) # type: CIMultiDict[str]
else:
real_headers = CIMultiDict()
self._default_headers = real_headers # type: CIMultiDict[str]
if skip_auto_headers is not None:
self._skip_auto_headers = frozenset(
[istr(i) for i in skip_auto_headers])
else:
self._skip_auto_headers = frozenset()
self._request_class = request_class
self._response_class = response_class
self._ws_response_class = ws_response_class
self._trace_configs = trace_configs or []
for trace_config in self._trace_configs:
trace_config.freeze()
def _request(self, method, url, *,
params=None,
data=None,
headers=None,
skip_auto_headers=None,
auth=None,
allow_redirects=True,
max_redirects=10,
encoding='utf-8',
version=None,
compress=None,
chunked=None,
expect100=False,
read_until_eof=True,
proxy=None,
proxy_auth=None,
timeout=DEFAULT_TIMEOUT):
# NOTE: timeout clamps existing connect and read timeouts. We cannot
# set the default to None because we need to detect if the user wants
# to use the existing timeouts by setting timeout to None.
if version is not None:
warnings.warn("HTTP version should be specified "
"by ClientSession constructor", DeprecationWarning)
else:
version = self._version
if self.closed:
raise RuntimeError('Session is closed')
if not isinstance(chunked, bool) and chunked is not None:
warnings.warn(
'Chunk size is deprecated #1615', DeprecationWarning)
redirects = 0
history = []
# Merge with default headers and transform to CIMultiDict
headers = self._prepare_headers(headers)
if auth is None:
auth = self._default_auth
# It would be confusing if we support explicit Authorization header
# with `auth` argument
if (headers is not None and
auth is not None and
hdrs.AUTHORIZATION in headers):
raise ValueError("Can't combine `Authorization` header with "
"`auth` argument")
skip_headers = set(self._skip_auto_headers)
if skip_auto_headers is not None:
for i in skip_auto_headers:
skip_headers.add(istr(i))
if proxy is not None:
proxy = URL(proxy)
# request timeout
if timeout is None:
timeout = self._read_timeout
if timeout is None:
timeout = self._connector.conn_timeout
elif self._connector.conn_timeout is not None:
timeout = max(timeout, self._connector.conn_timeout)
# timeout is cumulative for all request operations
# (request, redirects, responses, data consuming)
timer = self._time_service.timeout(timeout)
with timer:
while True:
url = URL(url).with_fragment(None)
cookies = self._cookie_jar.filter_cookies(url)
req = self._request_class(
method, url, params=params, headers=headers,
skip_auto_headers=skip_headers, data=data,
cookies=cookies, encoding=encoding,
auth=auth, version=version, compress=compress,
chunked=chunked, expect100=expect100,
loop=self._loop, response_class=self._response_class,
proxy=proxy, proxy_auth=proxy_auth, timer=timer)
conn = yield from self._connector.connect(req)
conn.writer.set_tcp_nodelay(True)
try:
resp = req.send(conn)
try:
yield from resp.start(conn, read_until_eof)
except:
resp.close()
conn.close()
raise
except ClientError:
raise
except http.HttpProcessingError as exc:
raise ClientResponseError(
code=exc.code,
message=exc.message, headers=exc.headers) from exc
except OSError as exc:
raise ClientOSError(*exc.args) from exc
self._cookie_jar.update_cookies(resp.cookies, resp.url)
# redirects
if resp.status in (301, 302, 303, 307) and allow_redirects:
redirects += 1
history.append(resp)
if max_redirects and redirects >= max_redirects:
resp.close()
break
else:
yield from resp.release()
# For 301 and 302, mimic IE behaviour, now changed in RFC.
# Info: https://github.com/kennethreitz/requests/pull/269
if (resp.status == 303 and resp.method != hdrs.METH_HEAD) \
or (resp.status in (301, 302) and
resp.method == hdrs.METH_POST):
method = hdrs.METH_GET
data = None
if headers.get(hdrs.CONTENT_LENGTH):
headers.pop(hdrs.CONTENT_LENGTH)
r_url = (resp.headers.get(hdrs.LOCATION) or
resp.headers.get(hdrs.URI))
if r_url is None:
raise RuntimeError("{0.method} {0.url} returns "
"a redirect [{0.status}] status "
"but response lacks a Location "
"or URI HTTP header".format(resp))
r_url = URL(r_url)
scheme = r_url.scheme
if scheme not in ('http', 'https', ''):
resp.close()
raise ValueError('Can redirect only to http or https')
elif not scheme:
r_url = url.join(r_url)
url = r_url
params = None
yield from resp.release()
continue
break
resp._history = tuple(history)
return resp
def __init__(self,
*,
connector=None,
loop=None,
cookies=None,
headers=None,
skip_auto_headers=None,
auth=None,
json_serialize=json.dumps,
request_class=ClientRequest,
response_class=ClientResponse,
ws_response_class=ClientWebSocketResponse,
version=http.HttpVersion11,
cookie_jar=None,
connector_owner=True,
raise_for_status=False,
read_timeout=sentinel,
conn_timeout=None,
auto_decompress=True,
trust_env=False,
trace_configs=None):
implicit_loop = False
if loop is None:
if connector is not None:
loop = connector._loop
else:
implicit_loop = True
loop = asyncio.get_event_loop()
if connector is None:
connector = TCPConnector(loop=loop)
if connector._loop is not loop:
raise RuntimeError(
"Session and connector has to use same event loop")
self._loop = loop
if loop.get_debug():
self._source_traceback = traceback.extract_stack(sys._getframe(1))
if implicit_loop and not loop.is_running():
warnings.warn(
"Creating a client session outside of coroutine is "
"a very dangerous idea",
stacklevel=2)
context = {
'client_session': self,
'message': 'Creating a client session outside '
'of coroutine'
}
if self._source_traceback is not None:
context['source_traceback'] = self._source_traceback
loop.call_exception_handler(context)
if cookie_jar is None:
cookie_jar = CookieJar(loop=loop)
self._cookie_jar = cookie_jar
if cookies is not None:
self._cookie_jar.update_cookies(cookies)
self._connector = connector
self._connector_owner = connector_owner
self._default_auth = auth
self._version = version
self._json_serialize = json_serialize
self._read_timeout = (read_timeout if read_timeout is not sentinel else
DEFAULT_TIMEOUT)
self._conn_timeout = conn_timeout
self._raise_for_status = raise_for_status
self._auto_decompress = auto_decompress
self._trust_env = trust_env
# Convert to list of tuples
if headers:
headers = CIMultiDict(headers)
else:
headers = CIMultiDict()
self._default_headers = headers
if skip_auto_headers is not None:
self._skip_auto_headers = frozenset(
[istr(i) for i in skip_auto_headers])
else:
self._skip_auto_headers = frozenset()
self._request_class = request_class
self._response_class = response_class
self._ws_response_class = ws_response_class
self._trace_configs = trace_configs or []
for trace_config in self._trace_configs:
trace_config.freeze()
def __init__(self, *, connector=None, loop=None, cookies=None,
headers=None, skip_auto_headers=None,
auth=None, request_class=ClientRequest,
response_class=ClientResponse,
ws_response_class=ClientWebSocketResponse,
version=http.HttpVersion11,
cookie_jar=None, read_timeout=None, time_service=None):
implicit_loop = False
if loop is None:
if connector is not None:
loop = connector._loop
else:
implicit_loop = True
loop = asyncio.get_event_loop()
connector_owner = connector is None
if connector is None:
connector = TCPConnector(loop=loop)
if connector._loop is not loop:
raise RuntimeError(
"Session and connector has to use same event loop")
self._loop = loop
if loop.get_debug():
self._source_traceback = traceback.extract_stack(sys._getframe(1))
if implicit_loop and not loop.is_running():
warnings.warn("Creating a client session outside of coroutine is "
"a very dangerous idea", ResourceWarning,
stacklevel=2)
context = {'client_session': self,
'message': 'Creating a client session outside '
'of coroutine'}
if self._source_traceback is not None:
context['source_traceback'] = self._source_traceback
loop.call_exception_handler(context)
if cookie_jar is None:
cookie_jar = CookieJar(loop=loop)
self._cookie_jar = cookie_jar
if cookies is not None:
self._cookie_jar.update_cookies(cookies)
self._connector = connector
self._connector_owner = connector_owner
self._default_auth = auth
self._version = version
self._read_timeout = read_timeout
# Convert to list of tuples
if headers:
headers = CIMultiDict(headers)
else:
headers = CIMultiDict()
self._default_headers = headers
if skip_auto_headers is not None:
self._skip_auto_headers = frozenset([istr(i)
for i in skip_auto_headers])
else:
self._skip_auto_headers = frozenset()
self._request_class = request_class
self._response_class = response_class
self._ws_response_class = ws_response_class
self._time_service_owner = time_service is None
if time_service is None:
time_service = TimeService(self._loop)
self._time_service = time_service
def __init__(self, *, connector=None, loop=None, cookies=None,
headers=None, skip_auto_headers=None,
auth=None, request_class=ClientRequest,
response_class=ClientResponse,
ws_response_class=ClientWebSocketResponse,
version=aiohttp.HttpVersion11,
cookie_jar=None, read_timeout=None, time_service=None):
implicit_loop = False
if loop is None:
if connector is not None:
loop = connector._loop
else:
implicit_loop = True
loop = asyncio.get_event_loop()
if connector is None:
connector = aiohttp.TCPConnector(loop=loop)
if connector._loop is not loop:
raise RuntimeError(
"Session and connector has to use same event loop")
self._loop = loop
if loop.get_debug():
self._source_traceback = traceback.extract_stack(sys._getframe(1))
if implicit_loop and not loop.is_running():
warnings.warn("Creating a client session outside of coroutine is "
"a very dangerous idea", ResourceWarning,
stacklevel=2)
context = {'client_session': self,
'message': 'Creating a client session outside '
'of coroutine'}
if self._source_traceback is not None:
context['source_traceback'] = self._source_traceback
loop.call_exception_handler(context)
if cookie_jar is None:
cookie_jar = CookieJar(loop=loop)
self._cookie_jar = cookie_jar
if cookies is not None:
self._cookie_jar.update_cookies(cookies)
self._connector = connector
self._default_auth = auth
self._version = version
self._read_timeout = read_timeout
# Convert to list of tuples
if headers:
headers = CIMultiDict(headers)
else:
headers = CIMultiDict()
self._default_headers = headers
if skip_auto_headers is not None:
self._skip_auto_headers = frozenset([istr(i)
for i in skip_auto_headers])
else:
self._skip_auto_headers = frozenset()
self._request_class = request_class
self._response_class = response_class
self._ws_response_class = ws_response_class
if time_service is not None:
self._time_service_owner = False
self._time_service = time_service
else:
self._time_service_owner = True
self._time_service = TimeService(self._loop)
METH_ANY = '*'
METH_CONNECT = 'CONNECT'
METH_HEAD = 'HEAD'
METH_GET = 'GET'
METH_DELETE = 'DELETE'
METH_OPTIONS = 'OPTIONS'
METH_PATCH = 'PATCH'
METH_POST = 'POST'
METH_PUT = 'PUT'
METH_TRACE = 'TRACE'
METH_ALL = {METH_CONNECT, METH_HEAD, METH_GET, METH_DELETE,
METH_OPTIONS, METH_PATCH, METH_POST, METH_PUT, METH_TRACE}
ACCEPT = istr('ACCEPT')
ACCEPT_CHARSET = istr('ACCEPT-CHARSET')
ACCEPT_ENCODING = istr('ACCEPT-ENCODING')
ACCEPT_LANGUAGE = istr('ACCEPT-LANGUAGE')
ACCEPT_RANGES = istr('ACCEPT-RANGES')
ACCESS_CONTROL_MAX_AGE = istr('ACCESS-CONTROL-MAX-AGE')
ACCESS_CONTROL_ALLOW_CREDENTIALS = istr('ACCESS-CONTROL-ALLOW-CREDENTIALS')
ACCESS_CONTROL_ALLOW_HEADERS = istr('ACCESS-CONTROL-ALLOW-HEADERS')
ACCESS_CONTROL_ALLOW_METHODS = istr('ACCESS-CONTROL-ALLOW-METHODS')
ACCESS_CONTROL_ALLOW_ORIGIN = istr('ACCESS-CONTROL-ALLOW-ORIGIN')
ACCESS_CONTROL_EXPOSE_HEADERS = istr('ACCESS-CONTROL-EXPOSE-HEADERS')
ACCESS_CONTROL_REQUEST_HEADERS = istr('ACCESS-CONTROL-REQUEST-HEADERS')
ACCESS_CONTROL_REQUEST_METHOD = istr('ACCESS-CONTROL-REQUEST-METHOD')
AGE = istr('AGE')
ALLOW = istr('ALLOW')
AUTHORIZATION = istr('AUTHORIZATION')
