def test_auth_header_preserved_from_s3_redirects(self):
request = AWSRequest()
request.url = 'https://bucket.s3.amazonaws.com/'
request.method = 'GET'
request.headers['Authorization'] = 'original auth header'
prepared_request = request.prepare()
fake_response = Mock()
fake_response.headers = {
'location': 'https://bucket.s3-us-west-2.amazonaws.com'}
fake_response.url = request.url
fake_response.status_code = 307
fake_response.is_permanent_redirect = False
# This line is needed to disable the cookie handling
# code in requests.
fake_response.raw._original_response = None
success_response = Mock()
success_response.raw._original_response = None
success_response.is_redirect = False
success_response.status_code = 200
session = BotocoreHTTPSession()
session.send = Mock(return_value=success_response)
list(session.resolve_redirects(
fake_response, prepared_request, stream=False))
redirected_request = session.send.call_args[0][0]
# The Authorization header for the newly sent request should
# still have our original Authorization header.
self.assertEqual(
redirected_request.headers['Authorization'],
'original auth header')
def test_can_prepare_url_params_with_existing_query(self):
request = AWSRequest(
url='http://example.com/?bar=foo', params={'foo': 'bar'}
)
prepared_request = request.prepare()
self.assertEqual(
prepared_request.url, 'http://example.com/?bar=foo&foo=bar'
)
def test_can_prepare_empty_body(self):
request = AWSRequest(url='http://example.com/', data=b'')
prepared_request = request.prepare()
self.assertEqual(prepared_request.body, None)
content_length = prepared_request.headers.get('content-length')
self.assertEqual(content_length, '0')
def test_can_prepare_dict_body_unicode_keys(self):
body = {u'\u30c6\u30b9\u30c8': 'string'}
expected_body = '%E3%83%86%E3%82%B9%E3%83%88=string'
request = AWSRequest(url='http://example.com/', data=body)
prepared_request = request.prepare()
self.assertEqual(prepared_request.body, expected_body)
def test_can_prepare_dict_body(self):
body = {'dead': 'beef'}
request = AWSRequest(url='http://example.com/', data=body)
prepared_request = request.prepare()
self.assertEqual(prepared_request.body, 'dead=beef')
def test_can_prepare_url_params(self):
request = AWSRequest(url='http://example.com/', params={'foo': 'bar'})
prepared_request = request.prepare()
self.assertEqual(prepared_request.url, 'http://example.com/?foo=bar')
class TestAWSRequest(unittest.TestCase):
def setUp(self):
self.tempdir = tempfile.mkdtemp()
self.filename = os.path.join(self.tempdir, 'foo')
self.request = AWSRequest(method='GET', url='http://example.com')
self.prepared_request = self.request.prepare()
def tearDown(self):
shutil.rmtree(self.tempdir)
def test_prepared_request_repr(self):
expected_repr = (
'<AWSPreparedRequest stream_output=False, method=GET, '
'url=http://example.com, headers={}>')
request_repr = repr(self.prepared_request)
self.assertEqual(request_repr, expected_repr)
def test_can_prepare_url_params(self):
request = AWSRequest(url='http://example.com/', params={'foo': 'bar'})
prepared_request = request.prepare()
self.assertEqual(prepared_request.url, 'http://example.com/?foo=bar')
def test_can_prepare_dict_body(self):
body = {'dead': 'beef'}
request = AWSRequest(url='http://example.com/', data=body)
prepared_request = request.prepare()
self.assertEqual(prepared_request.body, 'dead=beef')
def test_can_prepare_dict_body_unicode_values(self):
body = {'Text': u'\u30c6\u30b9\u30c8 string'}
expected_body = 'Text=%E3%83%86%E3%82%B9%E3%83%88+string'
request = AWSRequest(url='http://example.com/', data=body)
prepared_request = request.prepare()
self.assertEqual(prepared_request.body, expected_body)
def test_can_prepare_dict_body_unicode_keys(self):
body = {u'\u30c6\u30b9\u30c8': 'string'}
expected_body = '%E3%83%86%E3%82%B9%E3%83%88=string'
request = AWSRequest(url='http://example.com/', data=body)
prepared_request = request.prepare()
self.assertEqual(prepared_request.body, expected_body)
def test_can_prepare_empty_body(self):
request = AWSRequest(url='http://example.com/', data=b'')
prepared_request = request.prepare()
self.assertEqual(prepared_request.body, None)
content_length = prepared_request.headers.get('content-length')
self.assertEqual(content_length, '0')
def test_request_body_is_prepared(self):
request = AWSRequest(url='http://example.com/', data='body')
self.assertEqual(request.body, b'body')
def test_prepare_body_content_adds_content_length(self):
content = b'foobarbaz'
expected_len = str(len(content))
with open(self.filename, 'wb') as f:
f.write(content)
with open(self.filename, 'rb') as f:
data = Seekable(f)
self.request.data = data
self.request.method = 'POST'
prepared_request = self.request.prepare()
calculated_len = prepared_request.headers['Content-Length']
self.assertEqual(calculated_len, expected_len)
def test_prepare_body_doesnt_override_content_length(self):
self.request.method = 'PUT'
self.request.headers['Content-Length'] = '20'
self.request.data = b'asdf'
prepared_request = self.request.prepare()
self.assertEqual(prepared_request.headers['Content-Length'], '20')
def test_prepare_body_doesnt_set_content_length_head(self):
self.request.method = 'HEAD'
self.request.data = b'thisshouldntbehere'
prepared_request = self.request.prepare()
self.assertEqual(prepared_request.headers.get('Content-Length'), None)
def test_prepare_body_doesnt_set_content_length_get(self):
self.request.method = 'GET'
self.request.data = b'thisshouldntbehere'
prepared_request = self.request.prepare()
self.assertEqual(prepared_request.headers.get('Content-Length'), None)
def test_prepare_body_doesnt_set_content_length_options(self):
self.request.method = 'OPTIONS'
self.request.data = b'thisshouldntbehere'
prepared_request = self.request.prepare()
self.assertEqual(prepared_request.headers.get('Content-Length'), None)
def test_can_reset_stream_handles_binary(self):
contents = b'notastream'
self.prepared_request.body = contents
self.prepared_request.reset_stream()
# assert the request body doesn't change after reset_stream is called
self.assertEqual(self.prepared_request.body, contents)
def test_can_reset_stream_handles_bytearray(self):
contents = bytearray(b'notastream')
self.prepared_request.body = contents
self.prepared_request.reset_stream()
# assert the request body doesn't change after reset_stream is called
self.assertEqual(self.prepared_request.body, contents)
def test_can_reset_stream(self):
contents = b'foobarbaz'
with open(self.filename, 'wb') as f:
f.write(contents)
with open(self.filename, 'rb') as body:
self.prepared_request.body = body
# pretend the request body was partially sent
body.read()
self.assertNotEqual(body.tell(), 0)
# have the prepared request reset its stream
self.prepared_request.reset_stream()
# the stream should be reset
self.assertEqual(body.tell(), 0)
def test_cannot_reset_stream_raises_error(self):
contents = b'foobarbaz'
with open(self.filename, 'wb') as f:
f.write(contents)
with open(self.filename, 'rb') as body:
self.prepared_request.body = Unseekable(body)
# pretend the request body was partially sent
body.read()
self.assertNotEqual(body.tell(), 0)
# reset stream should fail
with self.assertRaises(UnseekableStreamError):
self.prepared_request.reset_stream()
def test_duck_type_for_file_check(self):
# As part of determining whether or not we can rewind a stream
# we first need to determine if the thing is a file like object.
# We should not be using an isinstance check. Instead, we should
# be using duck type checks.
class LooksLikeFile(object):
def __init__(self):
self.seek_called = False
def read(self, amount=None):
pass
def seek(self, where):
self.seek_called = True
looks_like_file = LooksLikeFile()
self.prepared_request.body = looks_like_file
self.prepared_request.reset_stream()
# The stream should now be reset.
self.assertTrue(looks_like_file.seek_called)
class TestAWSRequest(unittest.TestCase):
def setUp(self):
self.tempdir = tempfile.mkdtemp()
self.request = AWSRequest(url='http://example.com')
self.prepared_request = self.request.prepare()
self.filename = os.path.join(self.tempdir, 'foo')
def tearDown(self):
shutil.rmtree(self.tempdir)
def test_should_reset_stream(self):
with open(self.filename, 'wb') as f:
f.write(b'foobarbaz')
with open(self.filename, 'rb') as body:
self.prepared_request.body = body
# Now pretend we try to send the request.
# This means that we read the body:
body.read()
# And create a response object that indicates
# a redirect.
fake_response = Mock()
fake_response.status_code = 307
# Then requests calls our reset_stream hook.
self.prepared_request.reset_stream_on_redirect(fake_response)
# The stream should now be reset.
self.assertEqual(body.tell(), 0)
def test_cannot_reset_stream_raises_error(self):
with open(self.filename, 'wb') as f:
f.write(b'foobarbaz')
with open(self.filename, 'rb') as body:
self.prepared_request.body = Unseekable(body)
# Now pretend we try to send the request.
# This means that we read the body:
body.read()
# And create a response object that indicates
# a redirect
fake_response = Mock()
fake_response.status_code = 307
# Then requests calls our reset_stream hook.
with self.assertRaises(UnseekableStreamError):
self.prepared_request.reset_stream_on_redirect(fake_response)
def test_duck_type_for_file_check(self):
# As part of determining whether or not we can rewind a stream
# we first need to determine if the thing is a file like object.
# We should not be using an isinstance check. Instead, we should
# be using duck type checks.
class LooksLikeFile(object):
def __init__(self):
self.seek_called = False
def read(self, amount=None):
pass
def seek(self, where):
self.seek_called = True
looks_like_file = LooksLikeFile()
self.prepared_request.body = looks_like_file
fake_response = Mock()
fake_response.status_code = 307
# Then requests calls our reset_stream hook.
self.prepared_request.reset_stream_on_redirect(fake_response)
# The stream should now be reset.
self.assertTrue(looks_like_file.seek_called)
class TestURLLib3Session(unittest.TestCase):
def setUp(self):
self.request = AWSRequest(
method='GET',
url='http://example.com/',
headers={},
data=b'',
)
self.response = Mock()
self.response.headers = {}
self.response.stream.return_value = b''
self.pool_manager = Mock()
self.connection = Mock()
self.connection.urlopen.return_value = self.response
self.pool_manager.connection_from_url.return_value = self.connection
self.pool_patch = patch('ibm_botocore.httpsession.PoolManager')
self.proxy_patch = patch('ibm_botocore.httpsession.proxy_from_url')
self.pool_manager_cls = self.pool_patch.start()
self.proxy_manager_fun = self.proxy_patch.start()
self.pool_manager_cls.return_value = self.pool_manager
self.proxy_manager_fun.return_value = self.pool_manager
def tearDown(self):
self.pool_patch.stop()
self.proxy_patch.stop()
def assert_request_sent(self,
headers=None,
body=None,
url='/',
chunked=False):
if headers is None:
headers = {}
self.connection.urlopen.assert_called_once_with(
method=self.request.method,
url=url,
body=body,
headers=headers,
retries=ANY,
assert_same_host=False,
preload_content=False,
decode_content=False,
chunked=chunked,
)
def _assert_manager_call(self, manager, *assert_args, **assert_kwargs):
call_kwargs = {
'strict': True,
'maxsize': ANY,
'timeout': ANY,
'ssl_context': ANY,
'socket_options': [],
'cert_file': None,
'key_file': None,
}
call_kwargs.update(assert_kwargs)
manager.assert_called_with(*assert_args, **call_kwargs)
def assert_pool_manager_call(self, *args, **kwargs):
self._assert_manager_call(self.pool_manager_cls, *args, **kwargs)
def assert_proxy_manager_call(self, *args, **kwargs):
self._assert_manager_call(self.proxy_manager_fun, *args, **kwargs)
def test_forwards_max_pool_size(self):
URLLib3Session(max_pool_connections=22)
self.assert_pool_manager_call(maxsize=22)
def test_forwards_client_cert(self):
URLLib3Session(client_cert='/some/cert')
self.assert_pool_manager_call(cert_file='/some/cert', key_file=None)
def test_forwards_client_cert_and_key_tuple(self):
cert = ('/some/cert', '/some/key')
URLLib3Session(client_cert=cert)
self.assert_pool_manager_call(cert_file=cert[0], key_file=cert[1])
def test_basic_https_proxy_with_client_cert(self):
proxies = {'https': 'http://proxy.com'}
session = URLLib3Session(proxies=proxies, client_cert='/some/cert')
self.request.url = 'https://example.com/'
session.send(self.request.prepare())
self.assert_proxy_manager_call(
proxies['https'],
proxy_headers={},
cert_file='/some/cert',
key_file=None,
)
self.assert_request_sent()
def test_basic_https_proxy_with_client_cert_and_key(self):
cert = ('/some/cert', '/some/key')
proxies = {'https': 'http://proxy.com'}
session = URLLib3Session(proxies=proxies, client_cert=cert)
self.request.url = 'https://example.com/'
session.send(self.request.prepare())
self.assert_proxy_manager_call(
proxies['https'],
proxy_headers={},
cert_file=cert[0],
key_file=cert[1],
)
self.assert_request_sent()
def test_basic_request(self):
session = URLLib3Session()
session.send(self.request.prepare())
self.assert_request_sent()
self.response.stream.assert_called_once_with()
def test_basic_streaming_request(self):
session = URLLib3Session()
self.request.stream_output = True
session.send(self.request.prepare())
self.assert_request_sent()
self.response.stream.assert_not_called()
def test_basic_https_request(self):
session = URLLib3Session()
self.request.url = 'https://example.com/'
session.send(self.request.prepare())
self.assert_request_sent()
def test_basic_https_proxy_request(self):
proxies = {'https': 'http://proxy.com'}
session = URLLib3Session(proxies=proxies)
self.request.url = 'https://example.com/'
session.send(self.request.prepare())
self.assert_proxy_manager_call(proxies['https'], proxy_headers={})
self.assert_request_sent()
def test_basic_proxy_request_caches_manager(self):
proxies = {'https': 'http://proxy.com'}
session = URLLib3Session(proxies=proxies)
self.request.url = 'https://example.com/'
session.send(self.request.prepare())
# assert we created the proxy manager
self.assert_proxy_manager_call(proxies['https'], proxy_headers={})
session.send(self.request.prepare())
# assert that we did not create another proxy manager
self.assertEqual(self.proxy_manager_fun.call_count, 1)
def test_basic_http_proxy_request(self):
proxies = {'http': 'http://proxy.com'}
session = URLLib3Session(proxies=proxies)
session.send(self.request.prepare())
self.assert_proxy_manager_call(proxies['http'], proxy_headers={})
self.assert_request_sent(url=self.request.url)
def test_ssl_context_is_explicit(self):
session = URLLib3Session()
session.send(self.request.prepare())
_, manager_kwargs = self.pool_manager_cls.call_args
self.assertIsNotNone(manager_kwargs.get('ssl_context'))
def test_proxy_request_ssl_context_is_explicit(self):
proxies = {'http': 'http://proxy.com'}
session = URLLib3Session(proxies=proxies)
session.send(self.request.prepare())
_, proxy_kwargs = self.proxy_manager_fun.call_args
self.assertIsNotNone(proxy_kwargs.get('ssl_context'))
def test_session_forwards_socket_options_to_pool_manager(self):
socket_options = [(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)]
URLLib3Session(socket_options=socket_options)
self.assert_pool_manager_call(socket_options=socket_options)
def test_session_forwards_socket_options_to_proxy_manager(self):
proxies = {'http': 'http://proxy.com'}
socket_options = [(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)]
session = URLLib3Session(
proxies=proxies,
socket_options=socket_options,
)
session.send(self.request.prepare())
self.assert_proxy_manager_call(
proxies['http'],
proxy_headers={},
socket_options=socket_options,
)
def make_request_with_error(self, error):
self.connection.urlopen.side_effect = error
session = URLLib3Session()
session.send(self.request.prepare())
@raises(EndpointConnectionError)
def test_catches_new_connection_error(self):
error = NewConnectionError(None, None)
self.make_request_with_error(error)
@raises(ConnectionClosedError)
def test_catches_bad_status_line(self):
error = ProtocolError(None)
self.make_request_with_error(error)
def test_aws_connection_classes_are_used(self):
session = URLLib3Session()
# ensure the pool manager is using the correct classes
http_class = self.pool_manager.pool_classes_by_scheme.get('http')
self.assertIs(http_class, AWSHTTPConnectionPool)
https_class = self.pool_manager.pool_classes_by_scheme.get('https')
self.assertIs(https_class, AWSHTTPSConnectionPool)
def test_chunked_encoding_is_set_with_header(self):
session = URLLib3Session()
self.request.headers['Transfer-Encoding'] = 'chunked'
session.send(self.request.prepare())
self.assert_request_sent(
chunked=True,
headers={'Transfer-Encoding': 'chunked'},
)
def test_chunked_encoding_is_not_set_without_header(self):
session = URLLib3Session()
session.send(self.request.prepare())
self.assert_request_sent(chunked=False)
