def fetch(
url,
method='GET',
headers=None,
body='',
timeout=DEFAULT_TIMEOUT,
connect_timeout=DEFAULT_CONNECT_TIMEOUT,
tcp_nodelay=False,
):
"""
Make an HTTP request.
:param url: the URL to fetch.
:param method: the HTTP method.
:param headers: a dictionary mapping from string keys to lists of string
values. For example::
{
'X-Foo': ['Bar'],
'X-Baz': ['Quux'],
}
:param body: the request body (must be of bytes type).
:param timeout: maximum allowed request time in seconds.
:param connect_timeout: maximum time allowed to establish a connection
in seconds.
:param tcp_nodelay: flag to enable tcp_nodelay for request
:returns: a crochet EventualResult object which behaves as a future,
.wait() can be called on it to retrieve the fido.fido.Response object.
.wait() throws any exception occurred while performing the request.
Eventual additional failures information is stored in the crochet
EventualResult object as stated in the official documentation
"""
# Twisted requires the method, url, headers to be bytes
url = to_bytes(url)
method = to_bytes(method)
# Make a copy to avoid mutating the original value
headers = dict(headers or {})
if not any(header.lower() == 'user-agent' for header in headers):
headers['User-Agent'] = [DEFAULT_USER_AGENT]
# initializes twisted reactor in a different thread
crochet.setup()
return fetch_inner(
url,
method,
headers,
body,
timeout,
connect_timeout,
tcp_nodelay,
)
def test_fetch_basic(server_url):
response = fido.fetch(server_url + ECHO_URL).wait(timeout=1)
assert response.headers.get(b'User-Agent') == [
to_bytes(DEFAULT_USER_AGENT),
]
assert response.reason == b'OK'
assert response.code == 200
def _decode(string, encoding='internet'):
if string is None:
return string
string = to_bytes(string)
string = _unquote_bytes(string)
string = string.decode(encoding)
return string
def _decode(string, encoding='internet'):
if string is None:
return string
string = to_bytes(string)
string = _unquote_bytes(string)
string = string.decode(encoding)
return string
def prepare_request_for_twisted(request_params):
# type: (typing.MutableMapping[str, typing.Any]) -> typing.Mapping[str, typing.Any]
"""
Uses the python package 'requests' to prepare the data as per twisted
needs. requests.PreparedRequest.prepare is able to compute the body and
the headers for the http call based on the input request_params. This
contains any query parameters, files, body and headers to include.
:return: dictionary in the form
{
'body': string, # (can represent any content-type i.e. json,
file, multipart..),
'headers': dictionary, # headers->values
'method': string, # can be 'GET', 'POST' etc.
'url': string,
'timeout': float, # optional
'connect_timeout': float, # optional
}
"""
prepared_request = requests.PreparedRequest()
# Ensure that all the headers are converted to strings.
# This is need to workaround https://github.com/requests/requests/issues/3491
request_params['headers'] = {
k: v if isinstance(v, six.binary_type) else str(v)
for k, v in six.iteritems(request_params.get('headers', {}))
}
prepared_request.prepare(headers=request_params.get('headers'),
data=request_params.get('data'),
params=request_params.get('params'),
files=request_params.get('files'),
url=request_params.get('url'),
method=request_params.get('method'))
# content-length was computed by 'requests' based on the current body
# but body will be processed by fido using twisted FileBodyProducer
# causing content-length to lose meaning and break the client.
prepared_request.headers.pop('Content-Length', None)
request_for_twisted = {
# converting to string for `requests` method is necessary when
# using requests < 2.8.1 due to a bug while handling unicode values
# See changelog 2.8.1 at https://pypi.python.org/pypi/requests
'method':
str(prepared_request.method or 'GET'),
'body': (to_bytes(prepared_request.body)
if prepared_request.body is not None else None),
'headers':
prepared_request.headers,
'url':
prepared_request.url,
}
for fetch_kwarg in ('connect_timeout', 'timeout', 'tcp_nodelay'):
if fetch_kwarg in request_params:
request_for_twisted[fetch_kwarg] = request_params[fetch_kwarg]
return request_for_twisted
def fetch(
url,
method='GET',
headers=None,
body='',
timeout=DEFAULT_TIMEOUT,
connect_timeout=DEFAULT_CONNECT_TIMEOUT,
):
"""
Make an HTTP request.
:param url: the URL to fetch.
:param method: the HTTP method.
:param headers: a dictionary mapping from string keys to lists of string
values. For example::
{
'X-Foo': ['Bar'],
'X-Baz': ['Quux'],
}
:param body: the request body (must be of bytes type).
:param timeout: maximum allowed request time in seconds.
:param connect_timeout: maximum time allowed to establish a connection
in seconds.
:returns: a crochet EventualResult object which behaves as a future,
.wait() can be called on it to retrieve the fido.fido.Response object.
.wait() throws any exception occurred while performing the request.
Eventual additional failures information is stored in the crochet
EventualResult object as stated in the official documentation
"""
# Twisted requires the method, url, headers to be bytes
url = to_bytes(url)
method = to_bytes(method)
# Make a copy to avoid mutating the original value
headers = dict(headers or {})
if not any(header.lower() == 'user-agent' for header in headers):
headers['User-Agent'] = [DEFAULT_USER_AGENT]
# initializes twisted reactor in a different thread
crochet.setup()
return fetch_inner(url, method, headers, body, timeout, connect_timeout)
def content_length(self):
"""Send back the content-length number as the response."""
self.send_response(200)
response = to_bytes(self.headers.get('Content-Length'))
content_length = len(response)
self.send_header('Content-Length', content_length)
self.end_headers()
self.wfile.write(response)
def content_length(self):
"""Send back the content-length number as the response."""
self.send_response(200)
response = to_bytes(self.headers.get('Content-Length'))
content_length = len(response)
self.send_header('Content-Length', content_length)
self.end_headers()
self.wfile.write(response)
def _decode_hostname(hostname):
# IDNA decoding is not idempotent (sadface) so we need a special case here.
if hostname is None:
return hostname
hostname = to_bytes(hostname)
hostname = _unquote_bytes(hostname)
if _is_punycoded(hostname):
return hostname.decode('IDNA')
else:
return hostname.decode('internet')
def _decode_hostname(hostname):
# IDNA decoding is not idempotent (sadface) so we need a special case here.
if hostname is None:
return hostname
hostname = to_bytes(hostname)
hostname = _unquote_bytes(hostname)
if _is_punycoded(hostname):
return hostname.decode('IDNA')
else:
return hostname.decode('internet')
def prepare_request_for_twisted(request_params):
"""
Uses the python package 'requests' to prepare the data as per twisted
needs. requests.PreparedRequest.prepare is able to compute the body and
the headers for the http call based on the input request_params. This
contains any query parameters, files, body and headers to include.
:return: dictionary in the form
{
'body': string, # (can represent any content-type i.e. json,
file, multipart..),
'headers': dictionary, # headers->values
'method': string, # can be 'GET', 'POST' etc.
'url': string,
'timeout': float, # optional
'connect_timeout': float, # optional
}
"""
prepared_request = requests.PreparedRequest()
prepared_request.prepare(
headers=request_params.get('headers'),
data=request_params.get('data'),
params=request_params.get('params'),
files=request_params.get('files'),
url=request_params.get('url'),
method=request_params.get('method')
)
# content-length was computed by 'requests' based on the current body
# but body will be processed by fido using twisted FileBodyProducer
# causing content-length to lose meaning and break the client.
prepared_request.headers.pop('Content-Length', None)
request_for_twisted = {
# converting to string for `requests` method is necessary when
# using requests < 2.8.1 due to a bug while handling unicode values
# See changelog 2.8.1 at https://pypi.python.org/pypi/requests
'method': str(prepared_request.method or 'GET'),
'body': (
to_bytes(prepared_request.body)
if prepared_request.body is not None else None
),
'headers': prepared_request.headers,
'url': prepared_request.url,
}
for fetch_kwarg in ('connect_timeout', 'timeout'):
if fetch_kwarg in request_params:
request_for_twisted[fetch_kwarg] = request_params[fetch_kwarg]
return request_for_twisted
def test_to_bytes_is_like_str_encode(value):
# pylint:disable=bare-except,broad-except
try:
bytes_result = str(value) if PY2 else str(value).encode('US-ASCII')
except:
bytes_result = '(error)'
try:
to_bytes_result = to_bytes(value, 'US-ASCII')
except:
to_bytes_result = '(error)'
assert bytes_result == to_bytes_result
def test_to_bytes_is_like_str_encode(value):
# pylint:disable=bare-except,broad-except,redefined-variable-type
try:
bytes_result = str(value) if PY2 else str(value).encode('US-ASCII')
except:
bytes_result = '(error)'
try:
to_bytes_result = to_bytes(value, 'US-ASCII')
except:
to_bytes_result = '(error)'
assert bytes_result == to_bytes_result
def test_utf8_roundtrip(value):
assert from_bytes(value, 'utf8') == to_bytes(value, 'utf8').decode('utf8')
def test_windows_roundtrip(value):
assert from_bytes(value, 'windows-1252') == to_bytes(
value, 'windows-1252').decode('windows-1252')
def test_utf8_roundtrip(value):
assert from_bytes(value, 'utf8') == to_bytes(value, 'utf8').decode('utf8')
def test_internet_roundtrip(value):
assert from_bytes(value) == to_bytes(value).decode('internet')
def test_windows_roundtrip(value):
assert from_bytes(value, 'windows-1252') == to_bytes(value, 'windows-1252').decode('windows-1252')
def test_internet_roundtrip(value):
assert from_bytes(value) == to_bytes(value).decode('internet')
def test_fetch_basic(server_url):
response = fido.fetch(server_url + ECHO_URL).wait()
assert response.headers.get(b'User-Agent') == \
[to_bytes(DEFAULT_USER_AGENT)]
assert response.reason == b'OK'
assert response.code == 200
def listify_headers(headers):
"""Twisted agent requires header values as lists"""
for key, val in six.iteritems(headers):
if not isinstance(val, list):
headers[key] = [to_bytes(val)]
else:
headers[key] = [to_bytes(x) for x in val]
return Headers(headers)
