def test_fido_request_decompress_gzip(server_url):
expected_body = b'hello world'
# Ensure invalid gzipped responses raise a
# GzipDecompressionError exception.
with pytest.raises(GzipDecompressionError):
# Here we trick the client into decompressing the text
# response by echoing a gzip content-encoding response
# header. The client should then fail to decompress the
# text response.
response = fido.fetch(
server_url + ECHO_URL,
headers={
'Content-Encoding': 'gzip'
},
body=expected_body,
decompress_gzip=True,
).wait()
# Ensure valid gzipped responses are decompressed
# when gzip_enabled is True.
response = fido.fetch(
server_url + GZIP_URL,
# Ensure that fido successfully appends gzip to accept-encoding.
headers={
'Content-Encoding': 'gzip',
'Accept-Encoding': 'deflate, br, identity'
},
body=expected_body,
decompress_gzip=True,
).wait()
actual_body = response.body
assert response.code == 200
assert expected_body == actual_body
def test_fido_request_throws_no_timeout_when_header_value_not_list():
fido.fetch(
'http://www.yelp.com',
headers={
'Accept-Charset': 'utf-8',
'Accept-Language': ['en-US']
},
).wait(timeout=5)
def test_unicode_url(mock_future, mock_inner, _):
mock_future.configure_mock(
**{'set_running_or_notify_cancel.return_value': True})
fido.fetch(u'繁')
mock_inner.assert_called_once_with('\xe7\xb9\x81',
mock.ANY, mock.ANY,
mock.ANY, mock.ANY,
mock.ANY, mock.ANY)
def test_fetch_basic(server_url):
response = fido.fetch(server_url + ECHO_URL).wait(timeout=1)
assert response.headers.get(b'User-Agent') == [
to_bytes(DEFAULT_USER_AGENT),
]
assert response.reason == b'OK'
assert response.code == 200
def request(self, request_params, response_callback=None,
also_return_response=False):
"""Sets up the request params as per Twisted Agent needs.
Sets up crochet and triggers the API request in background
:param request_params: request parameters for API call
:type request_params: dict
:param response_callback: Function to be called after
receiving the response
:type response_callback: method
:param also_return_response: Consult the constructor documentation for
:class:`bravado.http_future.HttpFuture`.
:rtype: :class: `bravado_core.http_future.HttpFuture`
"""
url = '%s?%s' % (request_params['url'], urllib_utf8.urlencode(
request_params.get('params', []), True))
fetch_kwargs = {
'method': str(request_params.get('method', 'GET')),
'body': stringify_body(request_params),
'headers': request_params.get('headers', {}),
}
for fetch_kwarg in ('connect_timeout', 'timeout'):
if fetch_kwarg in request_params:
fetch_kwargs[fetch_kwarg] = request_params[fetch_kwarg]
concurrent_future = fido.fetch(url, **fetch_kwargs)
return HttpFuture(concurrent_future,
FidoResponseAdapter,
response_callback,
also_return_response)
def test_agent_timeout(server_url, tcp_nodelay):
"""
Testing that we don't wait forever on the server sending back a response
"""
eventual_result = fido.fetch(
server_url + ECHO_URL + '/slow',
timeout=TIMEOUT_TEST,
tcp_nodelay=tcp_nodelay,
)
# wait for fido to estinguish the timeout and abort before test-assertions
time.sleep(2 * TIMEOUT_TEST)
# timeout errors were thrown and handled in the reactor thread.
# EventualResult stores them and re-raises on result retrieval
assert eventual_result.original_failure() is not None
with pytest.raises(HTTPTimeoutError) as excinfo:
eventual_result.wait(timeout=1)
assert ("Connection was closed by fido because the server took "
"more than timeout={timeout} seconds to "
"send the response".format(timeout=TIMEOUT_TEST)
in str(excinfo.value))
def test_agent_connect_timeout(tcp_nodelay):
"""
Testing that we don't wait more than connect_timeout to establish a http
connection
"""
# google drops TCP SYN packets
eventual_result = fido.fetch(
"http://www.google.com:81",
connect_timeout=TIMEOUT_TEST,
tcp_nodelay=tcp_nodelay,
)
# wait enough for the connection to be dropped by Twisted Agent
time.sleep(3 * TIMEOUT_TEST)
# timeout errors were thrown and handled in the reactor thread.
# EventualResult stores them and re-raises on result retrieval
assert eventual_result.original_failure() is not None
with pytest.raises(TCPConnectionError) as e:
eventual_result.wait()
assert ("Connection was closed by Twisted Agent because there was "
"a problem establishing the connection or the "
"connect_timeout={connect_timeout} was reached.".format(
connect_timeout=TIMEOUT_TEST) in str(e))
def request(self, request_params, operation=None, response_callbacks=None,
also_return_response=False):
"""Sets up the request params as per Twisted Agent needs.
Sets up crochet and triggers the API request in background
:param request_params: request parameters for the http request.
:type request_params: dict
:param operation: operation that this http request is for. Defaults
to None - in which case, we're obviously just retrieving a Swagger
Spec.
:type operation: :class:`bravado_core.operation.Operation`
:param response_callbacks: List of callables to post-process the
incoming response. Expects args incoming_response and operation.
:param also_return_response: Consult the constructor documentation for
:class:`bravado.http_future.HttpFuture`.
:rtype: :class: `bravado_core.http_future.HttpFuture`
"""
request_for_twisted = self.prepare_request_for_twisted(request_params)
future_adapter = FidoFutureAdapter(fido.fetch(**request_for_twisted))
return HttpFuture(future_adapter,
FidoResponseAdapter,
operation,
response_callbacks,
also_return_response)
def request(self, request_params, response_callback=None):
"""Sets up the request params as per Twisted Agent needs.
Sets up crochet and triggers the API request in background
:param request_params: request parameters for API call
:type request_params: dict
:param response_callback: Function to be called after
receiving the response
:type response_callback: method
:rtype: :class: `bravado_core.http_future.HttpFuture`
"""
url = '%s?%s' % (request_params['url'],
urllib_utf8.urlencode(
request_params.get('params', []), True))
fetch_kwargs = {
'method': str(request_params.get('method', 'GET')),
'body': stringify_body(request_params),
'headers': request_params.get('headers', {}),
}
for fetch_kwarg in ('connect_timeout', 'timeout'):
if fetch_kwarg in request_params:
fetch_kwargs[fetch_kwarg] = request_params[fetch_kwarg]
concurrent_future = fido.fetch(url, **fetch_kwargs)
return HttpFuture(concurrent_future, FidoResponseAdapter,
response_callback)
def request(self, request_params, operation=None, request_config=None):
request_for_twisted = self.prepare_request_for_twisted(request_params)
request_for_twisted["headers"].update(self.headers)
future_adapter = self.future_adapter_class(
fido.fetch(**request_for_twisted))
return HttpFuture(future_adapter, self.response_adapter_class,
operation, request_config)
def _make_request(
self, method, path, headers, query_params, payload,
region=None, bucket=None,
):
"""Authenticate and actually make the HTTP request to S3."""
headers = headers or {}
query_params = query_params or {}
region = region or self.region
bucket = bucket or self.bucket
if not region or not bucket:
raise ValueError(
"Region and bucket must be either set at the client level "
"or passed in at call time. "
"Region={region}; Bucket={bucket}".format(
region=region,
bucket=bucket,
),
)
hashed_payload = auth.compute_hashed_payload(payload)
host = ""
if region == "us-east-1":
host = "{bucket}.s3.amazonaws.com".format(
bucket=bucket,
)
else:
host = "{bucket}.s3-{region}.amazonaws.com".format(
bucket=bucket,
region=region,
)
query_string = auth.create_canonical_query_string(query_params)
if not path.startswith("/"):
path = "/" + path
now = datetime.datetime.utcnow()
# Add headers necessary for auth computation
headers["host"] = host
headers["x-amz-content-sha256"] = hashed_payload
headers["x-amz-date"] = now.strftime(auth.ISO8601_FMT)
headers["Authorization"] = auth.compute_auth_header(
headers,
method,
now,
region,
bucket,
path,
query_string,
hashed_payload,
self.access_key,
self.secret_key,
)
url = "http://{host}{path}".format(host=host, path=path)
if query_string:
url += "?" + query_string
return fido.fetch(url, method=method, body=payload, headers=headers)
def test_agent_connect_timeout():
"""
Testing that we don't wait more than connect_timeout to establish a http
connection
"""
# google drops TCP SYN packets
eventual_result = fido.fetch(
"http://www.google.com:81",
connect_timeout=TIMEOUT_TEST
)
# wait enough for the connection to be dropped by Twisted Agent
time.sleep(2 * TIMEOUT_TEST)
# timeout errors were thrown and handled in the reactor thread.
# EventualResult stores them and re-raises on result retrieval
assert eventual_result.original_failure() is not None
with pytest.raises(TCPConnectionError) as e:
eventual_result.wait()
assert (
"Connection was closed by Twisted Agent because there was "
"a problem establishing the connection or the "
"connect_timeout={connect_timeout} was reached."
.format(connect_timeout=TIMEOUT_TEST)
in str(e)
)
def test_fetch_headers(server_url, tcp_nodelay):
headers = {'foo': ['bar']}
eventual_result = fido.fetch(server_url + ECHO_URL,
headers=headers,
tcp_nodelay=tcp_nodelay)
actual_headers = eventual_result.wait().headers
assert actual_headers.get(b'Foo') == [b'bar']
def test_agent_timeout(server_url):
"""
Testing that we don't wait forever on the server sending back a response
"""
eventual_result = fido.fetch(
server_url + ECHO_URL + '/slow',
timeout=TIMEOUT_TEST
)
# wait for fido to estinguish the timeout and abort before test-assertions
time.sleep(2 * TIMEOUT_TEST)
# timeout errors were thrown and handled in the reactor thread.
# EventualResult stores them and re-raises on result retrieval
assert eventual_result.original_failure() is not None
with pytest.raises(HTTPTimeoutError) as e:
eventual_result.wait()
assert (
"Connection was closed by fido because the server took "
"more than timeout={timeout} seconds to "
"send the response".format(timeout=TIMEOUT_TEST)
in str(e)
)
def request(self, request_params, response_callback=None):
"""Sets up the request params as per Twisted Agent needs.
Sets up crochet and triggers the API request in background
:param request_params: request parameters for API call
:type request_params: dict
:param response_callback: Function to be called after
receiving the response
:type response_callback: method
:rtype: :class: `bravado_core.http_future.HttpFuture`
"""
url = "%s?%s" % (request_params["url"], urllib_utf8.urlencode(request_params.get("params", []), True))
fetch_kwargs = {
"method": str(request_params.get("method", "GET")),
"body": stringify_body(request_params),
"headers": request_params.get("headers", {}),
}
for fetch_kwarg in ("connect_timeout", "timeout"):
if fetch_kwarg in request_params:
fetch_kwargs[fetch_kwarg] = request_params[fetch_kwarg]
concurrent_future = fido.fetch(url, **fetch_kwargs)
return HttpFuture(concurrent_future, FidoResponseAdapter, response_callback)
def request(
self,
request_params, # type: typing.MutableMapping[str, typing.Any]
operation=None, # type: typing.Optional[Operation]
request_config=None, # type: typing.Optional[RequestConfig]
):
# type: (...) -> HttpFuture[T]
"""Sets up the request params as per Twisted Agent needs.
Sets up crochet and triggers the API request in background
:param request_params: request parameters for the http request.
:type request_params: dict
:param operation: operation that this http request is for. Defaults
to None - in which case, we're obviously just retrieving a Swagger
Spec.
:type operation: :class:`bravado_core.operation.Operation`
:param RequestConfig request_config: per-request configuration
:rtype: :class: `bravado_core.http_future.HttpFuture`
"""
request_for_twisted = self.prepare_request_for_twisted(request_params)
future_adapter = FidoFutureAdapter(fido.fetch(**request_for_twisted)) # type: FidoFutureAdapter[T]
return HttpFuture(future_adapter,
FidoResponseAdapter,
operation,
request_config)
def test_future_wait(server_url):
futures = [fido.fetch(server_url) for _ in xrange(10)]
done, not_done = concurrent.futures.wait(futures)
assert len(done) == 10
assert len(not_done) == 0
for future in done:
assert future.done()
def test_json_body(server_url):
body = b'{"some_json_data": 30}'
eventual_result = fido.fetch(
server_url + ECHO_URL,
method='POST',
body=body
)
assert eventual_result.wait().json()['some_json_data'] == 30
def test_fetch_body(server_url):
expected_body = b'corpus'
eventual_result = fido.fetch(
server_url + ECHO_URL,
body=expected_body
)
actual_body = eventual_result.wait().body
assert expected_body == actual_body
def test_fetch_body(server_url, tcp_nodelay):
expected_body = b'corpus'
eventual_result = fido.fetch(
server_url + ECHO_URL,
body=expected_body,
tcp_nodelay=tcp_nodelay,
)
actual_body = eventual_result.wait().body
assert expected_body == actual_body
def test_json_body(server_url, tcp_nodelay):
body = b'{"some_json_data": 30}'
eventual_result = fido.fetch(
server_url + ECHO_URL,
method='POST',
body=body,
tcp_nodelay=tcp_nodelay,
)
assert eventual_result.wait().json()['some_json_data'] == 30
def test_fetch_content_type(server_url):
expected_content_type = b'text/html'
eventual_result = fido.fetch(
server_url + ECHO_URL,
headers={'Content-Type': expected_content_type}
)
actual_content_type = eventual_result.wait().headers.\
get(b'Content-Type')
assert [expected_content_type] == actual_content_type
def test_future_callback(server_url):
condition = threading.Condition()
done_callback = mock.Mock()
with condition:
future = fido.fetch(server_url)
future.add_done_callback(done_callback)
condition.wait(1)
done_callback.assert_called_once_with(future)
def test_fetch_user_agent(server_url, header_name):
expected_user_agent = [b'skynet']
headers = {header_name: expected_user_agent}
eventual_result = fido.fetch(
server_url + ECHO_URL,
headers=headers,
)
actual_user_agent = eventual_result.wait().headers.get(b'User-Agent')
assert expected_user_agent == actual_user_agent
def test_content_length_readded_by_twisted(server_url):
headers = {'Content-Length': '250'}
body = b'{"some_json_data": 30}'
eventual_result = fido.fetch(server_url + '/content_length',
method='POST',
headers=headers,
body=body)
content_length = int(eventual_result.wait().body)
assert content_length == 22
def test_fetch_content_type(server_url, tcp_nodelay):
expected_content_type = b'text/html'
eventual_result = fido.fetch(
server_url + ECHO_URL,
headers={'Content-Type': expected_content_type},
tcp_nodelay=tcp_nodelay,
)
actual_content_type = eventual_result.wait(
timeout=1, ).headers.get(b'Content-Type')
assert [expected_content_type] == actual_content_type
def test_content_length_readded_by_twisted(server_url):
headers = {'Content-Length': '250'}
body = b'{"some_json_data": 30}'
eventual_result = fido.fetch(
server_url + '/content_length',
method='POST',
headers=headers,
body=body
)
content_length = int(eventual_result.wait().body)
assert content_length == 22
def test_eventual_result_timeout(server_url):
"""
Testing timeout on result retrieval
"""
# fetch without setting timeouts -> we could potentially wait forever
eventual_result = fido.fetch(server_url + ECHO_URL + '/slow')
# make sure no timeout error is thrown here but only on result retrieval
assert eventual_result.original_failure() is None
with pytest.raises(crochet.TimeoutError):
eventual_result.wait(timeout=TIMEOUT_TEST)
assert eventual_result.original_failure() is None
def test_fido_request_gzip_disabled(server_url):
expected_body = b'hello world'
# Ensure that gzipped responses with decompress_gzip set
# to false remain compressed.
response = fido.fetch(
server_url + GZIP_URL,
body=expected_body,
headers={
'Accept-Encoding': 'gzip'
},
decompress_gzip=False,
).wait()
actual_body = response.body
assert response.code == 200
assert _compress_gzip(expected_body) == actual_body
def request(self, request_params, operation=None, response_callbacks=None,
also_return_response=False):
"""Sets up the request params as per Twisted Agent needs.
Sets up crochet and triggers the API request in background
:param request_params: request parameters for the http request.
:type request_params: dict
:param operation: operation that this http request is for. Defaults
to None - in which case, we're obviously just retrieving a Swagger
Spec.
:type operation: :class:`bravado_core.operation.Operation`
:param response_callbacks: List of callables to post-process the
incoming response. Expects args incoming_response and operation.
:param also_return_response: Consult the constructor documentation for
:class:`bravado.http_future.HttpFuture`.
:rtype: :class: `bravado_core.http_future.HttpFuture`
"""
url = '%s?%s' % (request_params['url'], urllib_utf8.urlencode(
request_params.get('params', []), True))
fetch_kwargs = {
'method': str(request_params.get('method', 'GET')),
'body': stringify_body(request_params),
'headers': request_params.get('headers', {}),
}
for fetch_kwarg in ('connect_timeout', 'timeout'):
if fetch_kwarg in request_params:
fetch_kwargs[fetch_kwarg] = request_params[fetch_kwarg]
concurrent_future = fido.fetch(url, **fetch_kwargs)
return HttpFuture(concurrent_future,
FidoResponseAdapter,
operation,
response_callbacks,
also_return_response)
def request(self, request_params, operation=None, request_config=None):
"""Sets up the request params as per Twisted Agent needs.
Sets up crochet and triggers the API request in background
:param request_params: request parameters for the http request.
:type request_params: dict
:param operation: operation that this http request is for. Defaults
to None - in which case, we're obviously just retrieving a Swagger
Spec.
:type operation: :class:`bravado_core.operation.Operation`
:param RequestConfig request_config: per-request configuration
:rtype: :class: `bravado_core.http_future.HttpFuture`
"""
request_for_twisted = self.prepare_request_for_twisted(request_params)
future_adapter = FidoFutureAdapter(fido.fetch(**request_for_twisted))
return HttpFuture(future_adapter,
FidoResponseAdapter,
operation,
request_config)
def request(self, request_params, response_callback=None):
"""Sets up the request params as per Twisted Agent needs.
Sets up crochet and triggers the API request in background
:param request_params: request parameters for API call
:type request_params: dict
:param response_callback: Function to be called after
receiving the response
:type response_callback: method
:rtype: :class: `bravado_core.http_future.HttpFuture`
"""
url = '%s?%s' % (request_params['url'], urllib_utf8.urlencode(
request_params.get('params', []), True))
request_params = {
'method': str(request_params.get('method', 'GET')),
'body': stringify_body(request_params),
'headers': request_params.get('headers', {}),
}
return HttpFuture(fido.fetch(url, **request_params),
FidoResponseAdapter,
response_callback)
def test_fetch_headers(server_url):
headers = {'foo': ['bar']}
future = fido.fetch(server_url, headers=headers)
actual_headers = future.result().json()['headers']
assert actual_headers.get('foo') == 'bar'
def test_fetch_method(server_url):
expected_method = 'POST'
future = fido.fetch(server_url, method=expected_method)
actual_method = future.result().json()['method']
assert expected_method == actual_method
def test_fetch_stress(server_url):
futures = [fido.fetch(server_url, timeout=8) for _ in xrange(1000)]
for future in concurrent.futures.as_completed(futures):
future.result()
def test_fetch_timeout(server_url):
with pytest.raises(twisted.web._newclient.ResponseNeverReceived):
fido.fetch(server_url + 'slow', timeout=0.5).result()
def test_fetch_basic(server_url):
response = fido.fetch(server_url).result()
assert 'Content-Type' in response.headers
assert response.code == 200
assert response.json()['method'] == 'GET'
def test_future_as_completed(server_url):
futures = [fido.fetch(server_url) for _ in xrange(10)]
for future in concurrent.futures.as_completed(futures):
assert future.done()
def test_future_exception(server_url):
future = fido.fetch(server_url + 'slow', timeout=0.5)
assert future.exception() is not None
def test_future_connection_refused():
with pytest.raises(twisted.internet.error.ConnectionRefusedError):
fido.fetch('http://localhost:0').result()
def test_future_timeout(server_url):
with pytest.raises(concurrent.futures.TimeoutError):
fido.fetch(server_url + 'slow').result(timeout=0.5)
def test_future_done(server_url):
future = fido.fetch(server_url)
future.result()
assert future.done()
def test_future_cancel(server_url):
# This is usually a no-op because we cannot cancel requests once they are
# being processed, which happens almost instantaneously.
future = fido.fetch(server_url)
future.cancel()
