class HttpRequest(RequestBase):
'''An :class:`HttpClient` request for an HTTP resource.
This class has a similar interface to :class:`urllib.request.Request`.
:param files: optional dictionary of name, file-like-objects.
:param allow_redirects: allow the response to follow redirects.
.. attribute:: method
The request method
.. attribute:: version
HTTP version for this request, usually ``HTTP/1.1``
.. attribute:: encode_multipart
If ``True`` (default), defaults POST data as ``multipart/form-data``.
Pass ``encode_multipart=False`` to default to
``application/x-www-form-urlencoded``. In any case, this parameter is
overwritten by passing the correct content-type header.
.. attribute:: history
List of past :class:`.HttpResponse` (collected during redirects).
.. attribute:: wait_continue
if ``True``, the :class:`HttpRequest` includes the
``Expect: 100-Continue`` header.
'''
CONNECT = 'CONNECT'
_proxy = None
_ssl = None
_tunnel = None
def __init__(self,
client,
url,
method,
inp_params=None,
headers=None,
data=None,
files=None,
history=None,
charset=None,
encode_multipart=True,
multipart_boundary=None,
source_address=None,
allow_redirects=False,
max_redirects=10,
decompress=True,
version=None,
wait_continue=False,
websocket_handler=None,
cookies=None,
urlparams=None,
**ignored):
self.client = client
self._data = None
self.files = files
self.urlparams = urlparams
self.inp_params = inp_params or {}
self.unredirected_headers = Headers(kind='client')
self.method = method.upper()
self.full_url = url
if urlparams:
self._encode_url(urlparams)
self.set_proxy(None)
self.history = history
self.wait_continue = wait_continue
self.max_redirects = max_redirects
self.allow_redirects = allow_redirects
self.charset = charset or 'utf-8'
self.version = version
self.decompress = decompress
self.encode_multipart = encode_multipart
self.multipart_boundary = multipart_boundary
self.websocket_handler = websocket_handler
self.source_address = source_address
self.new_parser()
if self._scheme in tls_schemes:
self._ssl = client.ssl_context(**ignored)
self.headers = client.get_headers(self, headers)
cookies = cookiejar_from_dict(client.cookies, cookies)
if cookies:
cookies.add_cookie_header(self)
self.unredirected_headers['host'] = host_no_default_port(
self._scheme, self._netloc)
client.set_proxy(self)
self.data = data
@property
def address(self):
'''``(host, port)`` tuple of the HTTP resource'''
return self._tunnel.address if self._tunnel else (self.host, self.port)
@property
def target_address(self):
return (self.host, int(self.port))
@property
def ssl(self):
'''Context for TLS connections.
If this is a tunneled request and the tunnel connection is not yet
established, it returns ``None``.
'''
if not self._tunnel:
return self._ssl
@property
def key(self):
return (self.scheme, self.host, self.port)
@property
def proxy(self):
'''Proxy server for this request.'''
return self._proxy
@property
def netloc(self):
if self._proxy:
return self._proxy.netloc
else:
return self._netloc
def __repr__(self):
return self.first_line()
__str__ = __repr__
@property
def full_url(self):
'''Full url of endpoint'''
return urlunparse((self._scheme, self._netloc, self.path, self.params,
self.query, self.fragment))
@full_url.setter
def full_url(self, url):
self._scheme, self._netloc, self.path, self.params,\
self.query, self.fragment = urlparse(url)
if not self._netloc and self.method == 'CONNECT':
self._scheme, self._netloc, self.path, self.params,\
self.query, self.fragment = urlparse('http://%s' % url)
@property
def data(self):
'''Body of request'''
return self._data
@data.setter
def data(self, data):
self._data = self._encode_data(data)
def first_line(self):
if not self._proxy and self.method != self.CONNECT:
url = urlunparse(('', '', self.path
or '/', self.params, self.query, self.fragment))
else:
url = self.full_url
return '%s %s %s' % (self.method, url, self.version)
def new_parser(self):
self.parser = self.client.http_parser(kind=1,
decompress=self.decompress,
method=self.method)
def set_proxy(self, scheme, *host):
if not host and scheme is None:
self.scheme = self._scheme
self._set_hostport(self._scheme, self._netloc)
else:
le = 2 + len(host)
if not le == 3:
raise TypeError(
'set_proxy() takes exactly three arguments (%s given)' %
le)
if not self._ssl:
self.scheme = scheme
self._set_hostport(scheme, host[0])
self._proxy = scheme_host(scheme, host[0])
else:
self._tunnel = HttpTunnel(self, scheme, host[0])
def _set_hostport(self, scheme, host):
self._tunnel = None
self._proxy = None
self.host, self.port = get_hostport(scheme, host)
def encode(self):
'''The bytes representation of this :class:`HttpRequest`.
Called by :class:`HttpResponse` when it needs to encode this
:class:`HttpRequest` before sending it to the HTTP resource.
'''
# Call body before fist_line in case the query is changes.
first_line = self.first_line()
body = self.data
if body and self.wait_continue:
self.headers['expect'] = '100-continue'
body = None
headers = self.headers
if self.unredirected_headers:
headers = self.unredirected_headers.copy()
headers.update(self.headers)
buffer = [first_line.encode('ascii'), b'\r\n', bytes(headers)]
if body:
buffer.append(body)
return b''.join(buffer)
def add_header(self, key, value):
self.headers[key] = value
def has_header(self, header_name):
'''Check ``header_name`` is in this request headers.
'''
return (header_name in self.headers
or header_name in self.unredirected_headers)
def get_header(self, header_name, default=None):
'''Retrieve ``header_name`` from this request headers.
'''
return self.headers.get(
header_name, self.unredirected_headers.get(header_name, default))
def remove_header(self, header_name):
'''Remove ``header_name`` from this request.
'''
self.headers.pop(header_name, None)
self.unredirected_headers.pop(header_name, None)
def add_unredirected_header(self, header_name, header_value):
self.unredirected_headers[header_name] = header_value
# INTERNAL ENCODING METHODS
def _encode_data(self, data):
body = None
if self.method in ENCODE_URL_METHODS:
self.files = None
self._encode_url(data)
elif isinstance(data, bytes):
assert self.files is None, ('data cannot be bytes when files are '
'present')
body = data
elif isinstance(data, str):
assert self.files is None, ('data cannot be string when files are '
'present')
body = to_bytes(data, self.charset)
elif data or self.files:
if self.files:
body, content_type = self._encode_files(data)
else:
body, content_type = self._encode_params(data)
# set files to None, Important!
self.files = None
self.headers['Content-Type'] = content_type
if body:
self.headers['content-length'] = str(len(body))
elif 'expect' not in self.headers:
self.headers.pop('content-length', None)
self.headers.pop('content-type', None)
return body
def _encode_url(self, data):
query = self.query
if data:
data = native_str(data)
if isinstance(data, str):
data = parse_qsl(data)
else:
data = mapping_iterator(data)
query = parse_qsl(query)
query.extend(data)
query = urlencode(query)
self.query = query
def _encode_files(self, data):
fields = []
for field, val in mapping_iterator(data or ()):
if (isinstance(val, str) or isinstance(val, bytes)
or not hasattr(val, '__iter__')):
val = [val]
for v in val:
if v is not None:
if not isinstance(v, bytes):
v = str(v)
fields.append(
(field.decode('utf-8')
if isinstance(field, bytes) else field,
v.encode('utf-8') if isinstance(v, str) else v))
for (k, v) in mapping_iterator(self.files):
# support for explicit filename
ft = None
if isinstance(v, (tuple, list)):
if len(v) == 2:
fn, fp = v
else:
fn, fp, ft = v
else:
fn = guess_filename(v) or k
fp = v
if isinstance(fp, bytes):
fp = BytesIO(fp)
elif isinstance(fp, str):
fp = StringIO(fp)
if ft:
new_v = (fn, fp.read(), ft)
else:
new_v = (fn, fp.read())
fields.append((k, new_v))
#
return encode_multipart_formdata(fields, charset=self.charset)
def _encode_params(self, data):
content_type = self.headers.get('content-type')
# No content type given, chose one
if not content_type:
if self.encode_multipart:
content_type = MULTIPART_FORM_DATA
else:
content_type = FORM_URL_ENCODED
if content_type in JSON_CONTENT_TYPES:
body = json.dumps(data).encode(self.charset)
elif content_type == FORM_URL_ENCODED:
body = urlencode(data).encode(self.charset)
elif content_type == MULTIPART_FORM_DATA:
body, content_type = encode_multipart_formdata(
data, boundary=self.multipart_boundary, charset=self.charset)
else:
raise ValueError("Don't know how to encode body for %s" %
content_type)
return body, content_type
class HttpRequest(RequestBase):
"""An :class:`HttpClient` request for an HTTP resource.
This class has a similar interface to :class:`urllib.request.Request`.
:param files: optional dictionary of name, file-like-objects.
:param allow_redirects: allow the response to follow redirects.
.. attribute:: method
The request method
.. attribute:: version
HTTP version for this request, usually ``HTTP/1.1``
.. attribute:: history
List of past :class:`.HttpResponse` (collected during redirects).
.. attribute:: wait_continue
if ``True``, the :class:`HttpRequest` includes the
``Expect: 100-Continue`` header.
.. attribute:: stream
Allow for streaming body
"""
_proxy = None
_ssl = None
_tunnel = None
_write_done = False
def __init__(self, client, url, method, inp_params=None, headers=None,
data=None, files=None, json=None, history=None, auth=None,
charset=None, max_redirects=10, source_address=None,
allow_redirects=False, decompress=True, version=None,
wait_continue=False, websocket_handler=None, cookies=None,
params=None, stream=False, proxies=None, verify=True,
**ignored):
self.client = client
self.method = method.upper()
self.inp_params = inp_params or {}
self.unredirected_headers = Headers()
self.history = history
self.wait_continue = wait_continue
self.max_redirects = max_redirects
self.allow_redirects = allow_redirects
self.charset = charset or 'utf-8'
self.version = version
self.decompress = decompress
self.websocket_handler = websocket_handler
self.source_address = source_address
self.stream = stream
self.verify = verify
self.new_parser()
if auth and not isinstance(auth, Auth):
auth = HTTPBasicAuth(*auth)
self.auth = auth
self.headers = client._get_headers(headers)
self.url = full_url(url, params, method=self.method)
self.body = self._encode_body(data, files, json)
self._set_proxy(proxies, ignored)
cookies = cookiejar_from_dict(client.cookies, cookies)
if cookies:
cookies.add_cookie_header(self)
@property
def _loop(self):
return self.client._loop
@property
def address(self):
"""``(host, port)`` tuple of the HTTP resource
"""
return self._tunnel.address if self._tunnel else super().address
@property
def ssl(self):
"""Context for TLS connections.
If this is a tunneled request and the tunnel connection is not yet
established, it returns ``None``.
"""
if not self._tunnel:
return self._ssl
@property
def key(self):
tunnel = self._tunnel.url if self._tunnel else None
scheme, host, port = scheme_host_port(self._proxy or self.url)
return scheme, host, port, tunnel, self.verify
@property
def proxy(self):
"""Proxy server for this request.
"""
return self._proxy
@property
def tunnel(self):
"""Tunnel for this request.
"""
return self._tunnel
def __repr__(self):
return self.first_line()
__str__ = __repr__
def first_line(self):
p = urlparse(self.url)
if self._proxy:
if self.method == 'CONNECT':
url = p.netloc
else:
url = self.url
else:
url = urlunparse(('', '', p.path or '/', p.params,
p.query, p.fragment))
return '%s %s %s' % (self.method, url, self.version)
def new_parser(self):
self.parser = self.client.http_parser(
kind=1, decompress=self.decompress
)
return self.parser
def is_chunked(self):
return self.body and 'content-length' not in self.headers
def encode(self):
"""The bytes representation of this :class:`HttpRequest`.
Called by :class:`HttpResponse` when it needs to encode this
:class:`HttpRequest` before sending it to the HTTP resource.
"""
# Call body before fist_line in case the query is changes.
first_line = self.first_line()
if self.body and self.wait_continue:
self.headers['expect'] = '100-continue'
headers = self.headers
if self.unredirected_headers:
headers = self.unredirected_headers.copy()
headers.update(self.headers)
buffer = [first_line.encode('ascii'), b'\r\n', bytes(headers)]
return b''.join(buffer)
def add_header(self, key, value):
self.headers[key] = value
def has_header(self, header_name):
"""Check ``header_name`` is in this request headers.
"""
return (header_name in self.headers or
header_name in self.unredirected_headers)
def get_header(self, header_name, default=None):
"""Retrieve ``header_name`` from this request headers.
"""
return self.headers.get(
header_name, self.unredirected_headers.get(header_name, default))
def remove_header(self, header_name):
"""Remove ``header_name`` from this request.
"""
val1 = self.headers.pop(header_name, None)
val2 = self.unredirected_headers.pop(header_name, None)
return val1 or val2
def add_unredirected_header(self, header_name, header_value):
self.unredirected_headers[header_name] = header_value
def write_body(self, transport):
assert not self._write_done, 'Body already sent'
self._write_done = True
if not self.body:
return
if is_streamed(self.body):
ensure_future(self._write_streamed_data(transport),
loop=self._loop)
else:
self._write_body_data(transport, self.body, True)
# INTERNAL ENCODING METHODS
def _encode_body(self, data, files, json):
body = None
if isinstance(data, (str, bytes)):
if files:
raise ValueError('data cannot be a string or bytes when '
'files are present')
body = to_bytes(data, self.charset)
elif data and is_streamed(data):
if files:
raise ValueError('data cannot be an iterator when '
'files are present')
if 'content-length' not in self.headers:
self.headers['transfer-encoding'] = 'chunked'
return data
elif data or files:
if files:
body, content_type = self._encode_files(data, files)
else:
body, content_type = self._encode_params(data)
self.headers['Content-Type'] = content_type
elif json:
body = _json.dumps(json).encode(self.charset)
self.headers['Content-Type'] = 'application/json'
if body:
self.headers['content-length'] = str(len(body))
return body
def _encode_files(self, data, files):
fields = []
for field, val in mapping_iterator(data or ()):
if (isinstance(val, str) or isinstance(val, bytes) or
not hasattr(val, '__iter__')):
val = [val]
for v in val:
if v is not None:
if not isinstance(v, bytes):
v = str(v)
fields.append((field.decode('utf-8') if
isinstance(field, bytes) else field,
v.encode('utf-8') if isinstance(v, str)
else v))
for (k, v) in mapping_iterator(files):
# support for explicit filename
ft = None
if isinstance(v, (tuple, list)):
if len(v) == 2:
fn, fp = v
else:
fn, fp, ft = v
else:
fn = guess_filename(v) or k
fp = v
if isinstance(fp, bytes):
fp = BytesIO(fp)
elif isinstance(fp, str):
fp = StringIO(fp)
if ft:
new_v = (fn, fp.read(), ft)
else:
new_v = (fn, fp.read())
fields.append((k, new_v))
#
return encode_multipart_formdata(fields, charset=self.charset)
def _encode_params(self, params):
content_type = self.headers.get('content-type')
# No content type given, chose one
if not content_type:
content_type = FORM_URL_ENCODED
if hasattr(params, 'read'):
params = params.read()
if content_type in JSON_CONTENT_TYPES:
body = _json.dumps(params)
elif content_type == FORM_URL_ENCODED:
body = urlencode(tuple(split_url_params(params)))
elif content_type == MULTIPART_FORM_DATA:
body, content_type = encode_multipart_formdata(
params, charset=self.charset)
else:
body = params
return to_bytes(body, self.charset), content_type
def _write_body_data(self, transport, data, finish=False):
if self.is_chunked():
data = http_chunks(data, finish)
elif data:
data = (data,)
else:
return
for chunk in data:
transport.write(chunk)
async def _write_streamed_data(self, transport):
for data in self.body:
if isawaitable(data):
data = await data
self._write_body_data(transport, data)
self._write_body_data(transport, b'', True)
# PROXY INTERNALS
def _set_proxy(self, proxies, ignored):
url = urlparse(self.url)
self.unredirected_headers['host'] = host_no_default_port(url.scheme,
url.netloc)
if url.scheme in tls_schemes:
self._ssl = self.client._ssl_context(verify=self.verify, **ignored)
request_proxies = self.client.proxies.copy()
if proxies:
request_proxies.update(proxies)
self.proxies = request_proxies
#
if url.scheme in request_proxies:
host, port = get_hostport(url.scheme, url.netloc)
no_proxy = [n for n in request_proxies.get('no', '').split(',')
if n]
if not any(map(host.endswith, no_proxy)):
url = request_proxies[url.scheme]
if not self._ssl:
self._proxy = url
else:
self._tunnel = HttpTunnel(self, url)
class HttpRequest(RequestBase):
'''An :class:`HttpClient` request for an HTTP resource.
This class has a similar interface to :class:`urllib.request.Request`.
:param files: optional dictionary of name, file-like-objects.
:param allow_redirects: allow the response to follow redirects.
.. attribute:: method
The request method
.. attribute:: version
HTTP version for this request, usually ``HTTP/1.1``
.. attribute:: encode_multipart
If ``True`` (default), defaults POST data as ``multipart/form-data``.
Pass ``encode_multipart=False`` to default to
``application/x-www-form-urlencoded``. In any case, this parameter is
overwritten by passing the correct content-type header.
.. attribute:: history
List of past :class:`.HttpResponse` (collected during redirects).
.. attribute:: wait_continue
if ``True``, the :class:`HttpRequest` includes the
``Expect: 100-Continue`` header.
'''
CONNECT = 'CONNECT'
_proxy = None
_ssl = None
_tunnel = None
def __init__(self, client, url, method, inp_params=None, headers=None,
data=None, files=None, history=None,
charset=None, encode_multipart=True, multipart_boundary=None,
source_address=None, allow_redirects=False, max_redirects=10,
decompress=True, version=None, wait_continue=False,
websocket_handler=None, cookies=None, urlparams=None,
**ignored):
self.client = client
self._data = None
self.files = files
self.urlparams = urlparams
self.inp_params = inp_params or {}
self.unredirected_headers = Headers(kind='client')
self.method = method.upper()
self.full_url = url
if urlparams:
self._encode_url(urlparams)
self.set_proxy(None)
self.history = history
self.wait_continue = wait_continue
self.max_redirects = max_redirects
self.allow_redirects = allow_redirects
self.charset = charset or 'utf-8'
self.version = version
self.decompress = decompress
self.encode_multipart = encode_multipart
self.multipart_boundary = multipart_boundary
self.websocket_handler = websocket_handler
self.source_address = source_address
self.new_parser()
if self._scheme in tls_schemes:
self._ssl = client.ssl_context(**ignored)
self.headers = client.get_headers(self, headers)
cookies = cookiejar_from_dict(client.cookies, cookies)
if cookies:
cookies.add_cookie_header(self)
self.unredirected_headers['host'] = host_no_default_port(self._scheme,
self._netloc)
client.set_proxy(self)
self.data = data
@property
def address(self):
'''``(host, port)`` tuple of the HTTP resource'''
return self._tunnel.address if self._tunnel else (self.host, self.port)
@property
def target_address(self):
return (self.host, int(self.port))
@property
def ssl(self):
'''Context for TLS connections.
If this is a tunneled request and the tunnel connection is not yet
established, it returns ``None``.
'''
if not self._tunnel:
return self._ssl
@property
def key(self):
return (self.scheme, self.host, self.port)
@property
def proxy(self):
'''Proxy server for this request.'''
return self._proxy
@property
def netloc(self):
if self._proxy:
return self._proxy.netloc
else:
return self._netloc
def __repr__(self):
return self.first_line()
__str__ = __repr__
@property
def full_url(self):
'''Full url of endpoint'''
return urlunparse((self._scheme, self._netloc, self.path,
self.params, self.query, self.fragment))
@full_url.setter
def full_url(self, url):
self._scheme, self._netloc, self.path, self.params,\
self.query, self.fragment = urlparse(url)
if not self._netloc and self.method == 'CONNECT':
self._scheme, self._netloc, self.path, self.params,\
self.query, self.fragment = urlparse('http://%s' % url)
@property
def data(self):
'''Body of request'''
return self._data
@data.setter
def data(self, data):
self._data = self._encode_data(data)
def first_line(self):
if not self._proxy and self.method != self.CONNECT:
url = urlunparse(('', '', self.path or '/', self.params,
self.query, self.fragment))
else:
url = self.full_url
return '%s %s %s' % (self.method, url, self.version)
def new_parser(self):
self.parser = self.client.http_parser(kind=1,
decompress=self.decompress,
method=self.method)
def set_proxy(self, scheme, *host):
if not host and scheme is None:
self.scheme = self._scheme
self._set_hostport(self._scheme, self._netloc)
else:
le = 2 + len(host)
if not le == 3:
raise TypeError(
'set_proxy() takes exactly three arguments (%s given)'
% le)
if not self._ssl:
self.scheme = scheme
self._set_hostport(scheme, host[0])
self._proxy = scheme_host(scheme, host[0])
else:
self._tunnel = HttpTunnel(self, scheme, host[0])
def _set_hostport(self, scheme, host):
self._tunnel = None
self._proxy = None
self.host, self.port = get_hostport(scheme, host)
def encode(self):
'''The bytes representation of this :class:`HttpRequest`.
Called by :class:`HttpResponse` when it needs to encode this
:class:`HttpRequest` before sending it to the HTTP resource.
'''
# Call body before fist_line in case the query is changes.
first_line = self.first_line()
body = self.data
if body and self.wait_continue:
self.headers['expect'] = '100-continue'
body = None
headers = self.headers
if self.unredirected_headers:
headers = self.unredirected_headers.copy()
headers.update(self.headers)
buffer = [first_line.encode('ascii'), b'\r\n', bytes(headers)]
if body:
buffer.append(body)
return b''.join(buffer)
def add_header(self, key, value):
self.headers[key] = value
def has_header(self, header_name):
'''Check ``header_name`` is in this request headers.
'''
return (header_name in self.headers or
header_name in self.unredirected_headers)
def get_header(self, header_name, default=None):
'''Retrieve ``header_name`` from this request headers.
'''
return self.headers.get(
header_name, self.unredirected_headers.get(header_name, default))
def remove_header(self, header_name):
'''Remove ``header_name`` from this request.
'''
self.headers.pop(header_name, None)
self.unredirected_headers.pop(header_name, None)
def add_unredirected_header(self, header_name, header_value):
self.unredirected_headers[header_name] = header_value
# INTERNAL ENCODING METHODS
def _encode_data(self, data):
body = None
if self.method in ENCODE_URL_METHODS:
self.files = None
self._encode_url(data)
elif isinstance(data, bytes):
assert self.files is None, ('data cannot be bytes when files are '
'present')
body = data
elif isinstance(data, str):
assert self.files is None, ('data cannot be string when files are '
'present')
body = to_bytes(data, self.charset)
elif data or self.files:
if self.files:
body, content_type = self._encode_files(data)
else:
body, content_type = self._encode_params(data)
# set files to None, Important!
self.files = None
self.headers['Content-Type'] = content_type
if body:
self.headers['content-length'] = str(len(body))
elif 'expect' not in self.headers:
self.headers.pop('content-length', None)
self.headers.pop('content-type', None)
return body
def _encode_url(self, data):
query = self.query
if data:
data = native_str(data)
if isinstance(data, str):
data = parse_qsl(data)
else:
data = mapping_iterator(data)
query = parse_qsl(query)
query.extend(data)
query = urlencode(query)
self.query = query
def _encode_files(self, data):
fields = []
for field, val in mapping_iterator(data or ()):
if (isinstance(val, str) or isinstance(val, bytes) or
not hasattr(val, '__iter__')):
val = [val]
for v in val:
if v is not None:
if not isinstance(v, bytes):
v = str(v)
fields.append((field.decode('utf-8') if
isinstance(field, bytes) else field,
v.encode('utf-8') if isinstance(v, str)
else v))
for (k, v) in mapping_iterator(self.files):
# support for explicit filename
ft = None
if isinstance(v, (tuple, list)):
if len(v) == 2:
fn, fp = v
else:
fn, fp, ft = v
else:
fn = guess_filename(v) or k
fp = v
if isinstance(fp, bytes):
fp = BytesIO(fp)
elif isinstance(fp, str):
fp = StringIO(fp)
if ft:
new_v = (fn, fp.read(), ft)
else:
new_v = (fn, fp.read())
fields.append((k, new_v))
#
return encode_multipart_formdata(fields, charset=self.charset)
def _encode_params(self, data):
content_type = self.headers.get('content-type')
# No content type given, chose one
if not content_type:
if self.encode_multipart:
content_type = MULTIPART_FORM_DATA
else:
content_type = FORM_URL_ENCODED
if content_type in JSON_CONTENT_TYPES:
body = json.dumps(data).encode(self.charset)
elif content_type == FORM_URL_ENCODED:
body = urlencode(data).encode(self.charset)
elif content_type == MULTIPART_FORM_DATA:
body, content_type = encode_multipart_formdata(
data, boundary=self.multipart_boundary, charset=self.charset)
else:
raise ValueError("Don't know how to encode body for %s" %
content_type)
return body, content_type
class HttpRequest(RequestBase):
"""An :class:`HttpClient` request for an HTTP resource.
This class has a similar interface to :class:`urllib.request.Request`.
:param files: optional dictionary of name, file-like-objects.
:param allow_redirects: allow the response to follow redirects.
.. attribute:: method
The request method
.. attribute:: version
HTTP version for this request, usually ``HTTP/1.1``
.. attribute:: encode_multipart
If ``True`` (default), defaults POST data as ``multipart/form-data``.
Pass ``encode_multipart=False`` to default to
``application/x-www-form-urlencoded``. In any case, this parameter is
overwritten by passing the correct content-type header.
.. attribute:: history
List of past :class:`.HttpResponse` (collected during redirects).
.. attribute:: wait_continue
if ``True``, the :class:`HttpRequest` includes the
``Expect: 100-Continue`` header.
.. attribute:: stream
Allow for streaming body
"""
_proxy = None
_ssl = None
_tunnel = None
_write_done = False
def __init__(self, client, url, method, inp_params=None, headers=None,
data=None, files=None, history=None, auth=None,
charset=None, encode_multipart=True, multipart_boundary=None,
source_address=None, allow_redirects=False, max_redirects=10,
decompress=True, version=None, wait_continue=False,
websocket_handler=None, cookies=None, urlparams=None,
stream=False, proxies=None, verify=True, **ignored):
self.client = client
self._data = None
self.files = files
self.urlparams = urlparams
self.inp_params = inp_params or {}
self.unredirected_headers = Headers(kind='client')
self.method = method.upper()
self.full_url = url
if urlparams:
self._encode_url(urlparams)
self.history = history
self.wait_continue = wait_continue
self.max_redirects = max_redirects
self.allow_redirects = allow_redirects
self.charset = charset or 'utf-8'
self.version = version
self.decompress = decompress
self.encode_multipart = encode_multipart
self.multipart_boundary = multipart_boundary
self.websocket_handler = websocket_handler
self.source_address = source_address
self.stream = stream
self.verify = verify
self.new_parser()
if self._scheme in tls_schemes:
self._ssl = client.ssl_context(verify=self.verify, **ignored)
if auth and not isinstance(auth, Auth):
auth = HTTPBasicAuth(*auth)
self.auth = auth
self.headers = client.get_headers(self, headers)
cookies = cookiejar_from_dict(client.cookies, cookies)
if cookies:
cookies.add_cookie_header(self)
self.unredirected_headers['host'] = host_no_default_port(self._scheme,
self._netloc)
self.data = data
self._set_proxy(proxies)
@property
def address(self):
"""``(host, port)`` tuple of the HTTP resource
"""
return self._tunnel.address if self._tunnel else (self.host, self.port)
@property
def target_address(self):
return (self.host, int(self.port))
@property
def ssl(self):
"""Context for TLS connections.
If this is a tunneled request and the tunnel connection is not yet
established, it returns ``None``.
"""
if not self._tunnel:
return self._ssl
@property
def key(self):
tunnel = self._tunnel.full_url if self._tunnel else None
return (self.scheme, self.host, self.port, tunnel, self.verify)
@property
def proxy(self):
"""Proxy server for this request.
"""
return self._proxy
@property
def tunnel(self):
"""Tunnel for this request.
"""
return self._tunnel
@property
def netloc(self):
if self._proxy:
return self._proxy.netloc
else:
return self._netloc
def __repr__(self):
return self.first_line()
__str__ = __repr__
@property
def full_url(self):
"""Full url of endpoint
"""
return urlunparse((self._scheme, self._netloc, self.path,
self.params, self.query, self.fragment))
@full_url.setter
def full_url(self, url):
self._scheme, self._netloc, self.path, self.params,\
self.query, self.fragment = urlparse(url)
if not self._netloc and self.method == 'CONNECT':
self._scheme, self._netloc, self.path, self.params,\
self.query, self.fragment = urlparse('http://%s' % url)
@property
def data(self):
"""Body of request
"""
return self._data
@data.setter
def data(self, data):
self._data = self._encode_data(data)
def first_line(self):
if self._proxy:
if self.method == 'CONNECT':
url = self._netloc
else:
url = self.full_url
else:
url = urlunparse(('', '', self.path or '/', self.params,
self.query, self.fragment))
return '%s %s %s' % (self.method, url, self.version)
def new_parser(self):
self.parser = self.client.http_parser(kind=1,
decompress=self.decompress,
method=self.method)
def is_chunked(self):
return self.data and 'content-length' not in self.headers
def encode(self):
"""The bytes representation of this :class:`HttpRequest`.
Called by :class:`HttpResponse` when it needs to encode this
:class:`HttpRequest` before sending it to the HTTP resource.
"""
# Call body before fist_line in case the query is changes.
first_line = self.first_line()
if self.data and self.wait_continue:
self.headers['expect'] = '100-continue'
headers = self.headers
if self.unredirected_headers:
headers = self.unredirected_headers.copy()
headers.update(self.headers)
buffer = [first_line.encode('ascii'), b'\r\n', bytes(headers)]
return b''.join(buffer)
def add_header(self, key, value):
self.headers[key] = value
def has_header(self, header_name):
"""Check ``header_name`` is in this request headers.
"""
return (header_name in self.headers or
header_name in self.unredirected_headers)
def get_header(self, header_name, default=None):
"""Retrieve ``header_name`` from this request headers.
"""
return self.headers.get(
header_name, self.unredirected_headers.get(header_name, default))
def remove_header(self, header_name):
"""Remove ``header_name`` from this request.
"""
val1 = self.headers.pop(header_name, None)
val2 = self.unredirected_headers.pop(header_name, None)
return val1 or val2
def add_unredirected_header(self, header_name, header_value):
self.unredirected_headers[header_name] = header_value
def write_body(self, transport):
assert not self._write_done, 'Body already sent'
self._write_done = True
if not self.data:
return
if is_streamed(self.data):
ensure_future(self._write_streamed_data(transport),
loop=transport._loop)
else:
self._write_body_data(transport, self.data, True)
# INTERNAL ENCODING METHODS
def _encode_data(self, data):
body = None
if self.method in ENCODE_URL_METHODS:
self.files = None
self._encode_url(data)
elif isinstance(data, bytes):
assert self.files is None, ('data cannot be bytes when files are '
'present')
body = data
elif isinstance(data, str):
assert self.files is None, ('data cannot be string when files are '
'present')
body = to_bytes(data, self.charset)
elif data and is_streamed(data):
assert self.files is None, ('data cannot be an iterator when '
'files are present')
if 'content-type' not in self.headers:
self.headers['content-type'] = 'application/octet-stream'
if 'content-length' not in self.headers:
self.headers['transfer-encoding'] = 'chunked'
return data
elif data or self.files:
if self.files:
body, content_type = self._encode_files(data)
else:
body, content_type = self._encode_params(data)
# set files to None, Important!
self.files = None
self.headers['Content-Type'] = content_type
if body:
self.headers['content-length'] = str(len(body))
elif 'expect' not in self.headers:
self.headers.pop('content-length', None)
self.headers.pop('content-type', None)
return body
def _encode_url(self, data):
query = self.query
if data:
data = native_str(data)
if isinstance(data, str):
data = parse_qsl(data)
else:
data = mapping_iterator(data)
query = parse_qsl(query)
query.extend(data)
query = urlencode(query)
self.query = query
def _encode_files(self, data):
fields = []
for field, val in mapping_iterator(data or ()):
if (isinstance(val, str) or isinstance(val, bytes) or
not hasattr(val, '__iter__')):
val = [val]
for v in val:
if v is not None:
if not isinstance(v, bytes):
v = str(v)
fields.append((field.decode('utf-8') if
isinstance(field, bytes) else field,
v.encode('utf-8') if isinstance(v, str)
else v))
for (k, v) in mapping_iterator(self.files):
# support for explicit filename
ft = None
if isinstance(v, (tuple, list)):
if len(v) == 2:
fn, fp = v
else:
fn, fp, ft = v
else:
fn = guess_filename(v) or k
fp = v
if isinstance(fp, bytes):
fp = BytesIO(fp)
elif isinstance(fp, str):
fp = StringIO(fp)
if ft:
new_v = (fn, fp.read(), ft)
else:
new_v = (fn, fp.read())
fields.append((k, new_v))
#
return encode_multipart_formdata(fields, charset=self.charset)
def _encode_params(self, data):
content_type = self.headers.get('content-type')
# No content type given, chose one
if not content_type:
if self.encode_multipart:
content_type = MULTIPART_FORM_DATA
else:
content_type = FORM_URL_ENCODED
if content_type in JSON_CONTENT_TYPES:
body = json.dumps(data).encode(self.charset)
elif content_type == FORM_URL_ENCODED:
body = urlencode(data).encode(self.charset)
elif content_type == MULTIPART_FORM_DATA:
body, content_type = encode_multipart_formdata(
data, boundary=self.multipart_boundary, charset=self.charset)
else:
raise ValueError("Don't know how to encode body for %s" %
content_type)
return body, content_type
def _write_body_data(self, transport, data, finish=False):
if self.is_chunked():
data = http_chunks(data, finish)
elif data:
data = (data,)
else:
return
for chunk in data:
transport.write(chunk)
@asyncio.coroutine
def _write_streamed_data(self, transport):
for data in self.data:
if isawaitable(data):
data = yield from data
self._write_body_data(transport, data)
self._write_body_data(transport, b'', True)
# PROXY INTERNALS
def _set_proxy(self, proxies):
request_proxies = self.client.proxies.copy()
if proxies:
request_proxies.update(proxies)
self.proxies = request_proxies
self.scheme = self._scheme
self._set_hostport(self._scheme, self._netloc)
#
if self.scheme in request_proxies:
hostonly = self.host
no_proxy = [n for n in request_proxies.get('no', '').split(',')
if n]
if not any(map(hostonly.endswith, no_proxy)):
url = request_proxies[self.scheme]
p = urlparse(url)
if not p.scheme:
raise ValueError('Could not understand proxy %s' % url)
scheme = p.scheme
host = p.netloc
if not self._ssl:
self.scheme = scheme
self._set_hostport(scheme, host)
self._proxy = scheme_host(scheme, host)
else:
self._tunnel = HttpTunnel(self, scheme, host)
def _set_hostport(self, scheme, host):
self._tunnel = None
self._proxy = None
self.host, self.port = get_hostport(scheme, host)