def test_presign_with_spaces_in_param(self):
request = AWSRequest()
request.url = 'https://ec2.us-east-1.amazonaws.com/'
request.data = {'Action': 'MyOperation', 'Description': 'With Spaces'}
self.auth.add_auth(request)
# Verify we encode spaces as '%20, and we don't use '+'.
self.assertIn('Description=With%20Spaces', request.url)
def test_destination_region_always_changed(self):
# If the user provides a destination region, we will still
# override the DesinationRegion with the region_name from
# the endpoint object.
actual_region = 'us-west-1'
v4query_auth = mock.Mock()
def add_auth(request):
request.url += '?PRESIGNED_STUFF'
v4query_auth.add_auth = add_auth
request_signer = mock.Mock()
request_signer._region_name = actual_region
request_signer.get_auth.return_value = v4query_auth
endpoint = mock.Mock()
request = AWSRequest()
request.method = 'POST'
request.url = 'https://ec2.us-east-1.amazonaws.com'
request = request.prepare()
endpoint.create_request.return_value = request
# The user provides us-east-1, but we will override this to
# endpoint.region_name, of 'us-west-1' in this case.
params = {'SourceRegion': 'us-west-2', 'DestinationRegion': 'us-east-1'}
handlers.copy_snapshot_encrypted({'body': params},
request_signer,
endpoint)
# Always use the DestinationRegion from the endpoint, regardless of
# whatever value the user provides.
self.assertEqual(params['DestinationRegion'], actual_region)
def test_copy_snapshot_encrypted(self):
v4query_auth = mock.Mock()
def add_auth(request):
request.url += '?PRESIGNED_STUFF'
v4query_auth.add_auth = add_auth
request_signer = mock.Mock()
request_signer._region_name = 'us-east-1'
request_signer.get_auth.return_value = v4query_auth
params = {'SourceRegion': 'us-west-2'}
endpoint = mock.Mock()
request = AWSRequest()
request.method = 'POST'
request.url = 'https://ec2.us-east-1.amazonaws.com'
request = request.prepare()
endpoint.create_request.return_value = request
handlers.copy_snapshot_encrypted({'body': params},
request_signer,
endpoint)
self.assertEqual(params['PresignedUrl'],
'https://ec2.us-west-2.amazonaws.com?PRESIGNED_STUFF')
# We should also populate the DestinationRegion with the
# region_name of the endpoint object.
self.assertEqual(params['DestinationRegion'], 'us-east-1')
def test_auth_header_preserved_from_s3_redirects(self):
request = AWSRequest()
request.url = 'https://bucket.s3.amazonaws.com/'
request.method = 'GET'
request.headers['Authorization'] = 'original auth header'
prepared_request = request.prepare()
fake_response = Mock()
fake_response.headers = {
'location': 'https://bucket.s3-us-west-2.amazonaws.com'}
fake_response.url = request.url
fake_response.status_code = 307
fake_response.is_permanent_redirect = False
# This line is needed to disable the cookie handling
# code in requests.
fake_response.raw._original_response = None
success_response = Mock()
success_response.raw._original_response = None
success_response.is_redirect = False
success_response.status_code = 200
session = PreserveAuthSession()
session.send = Mock(return_value=success_response)
list(session.resolve_redirects(
fake_response, prepared_request, stream=False))
redirected_request = session.send.call_args[0][0]
# The Authorization header for the newly sent request should
# still have our original Authorization header.
self.assertEqual(
redirected_request.headers['Authorization'],
'original auth header')
def request(url, method, credentials, service_name, region=None, headers=None, data=None):
if not region:
region = os.environ["AWS_REGION"]
aws_request = AWSRequest(url=url, method=method, headers=headers, data=data)
SigV4Auth(credentials, service_name, region).add_auth(aws_request)
return PreserveAuthSession().send(aws_request.prepare())
def test_payload_not_signed_if_disabled_in_context(self):
request = AWSRequest()
request.data = u'\u2713'.encode('utf-8')
request.url = 'https://amazonaws.com'
request.context['payload_signing_enabled'] = False
auth = self.create_signer()
payload = auth.payload(request)
self.assertEqual(payload, 'UNSIGNED-PAYLOAD')
def test_strips_default_port_and_http_auth(self):
request = AWSRequest()
request.url = 'http://*****:*****@s3.us-west-2.amazonaws.com:80/'
request.method = 'GET'
auth = self.create_signer('s3', 'us-west-2')
actual = auth.headers_to_sign(request)['host']
expected = 's3.us-west-2.amazonaws.com'
self.assertEqual(actual, expected)
def test_payload_is_bytes_type(self):
request = AWSRequest()
request.data = u'\u2713'.encode('utf-8')
auth = self.create_signer()
payload = auth.payload(request)
self.assertEqual(
payload,
'1dabba21cdad44541f6b15796f8d22978fc7ea10c46aeceeeeb66c23b3ac7604')
def test_operation_params_before_auth_params(self):
# The spec is picky about this.
request = AWSRequest()
request.url = 'https://ec2.us-east-1.amazonaws.com/?Action=MyOperation'
self.auth.add_auth(request)
# Verify auth params come after the existing params.
self.assertIn(
'?Action=MyOperation&X-Amz', request.url)
def test_presign_with_empty_param_value(self):
request = AWSRequest()
request.method = 'POST'
# actual URL format for creating a multipart upload
request.url = 'https://s3.amazonaws.com/mybucket/mykey?uploads'
self.auth.add_auth(request)
# verify that uploads param is still in URL
self.assertIn('uploads', request.url)
def test_operation_params_before_auth_params_in_body(self):
request = AWSRequest()
request.url = 'https://ec2.us-east-1.amazonaws.com/'
request.data = {'Action': 'MyOperation'}
self.auth.add_auth(request)
# Same situation, the params from request.data come before the auth
# params in the query string.
self.assertIn(
'?Action=MyOperation&X-Amz', request.url)
def test_switch_host_with_param(self):
request = AWSRequest()
url = 'https://machinelearning.us-east-1.amazonaws.com'
new_endpoint = 'https://my-custom-endpoint.amazonaws.com'
data = '{"PredictEndpoint":"%s"}' % new_endpoint
request.data = data.encode('utf-8')
request.url = url
handlers.switch_host_with_param(request, 'PredictEndpoint')
self.assertEqual(request.url, new_endpoint)
def test_signature_with_date_headers(self):
request = AWSRequest()
request.headers = {'Date': 'Thu, 17 Nov 2005 18:49:58 GMT'}
request.url = 'https://route53.amazonaws.com'
self.auth.add_auth(request)
self.assertEqual(
request.headers['X-Amzn-Authorization'],
('AWS3-HTTPS AWSAccessKeyId=access_key,Algorithm=HmacSHA256,'
'Signature=M245fo86nVKI8rLpH4HgWs841sBTUKuwciiTpjMDgPs='))
def test_payload_is_binary_file(self):
request = AWSRequest()
request.data = six.BytesIO(u'\u2713'.encode('utf-8'))
request.url = 'https://amazonaws.com'
auth = self.create_signer()
payload = auth.payload(request)
self.assertEqual(
payload,
'1dabba21cdad44541f6b15796f8d22978fc7ea10c46aeceeeeb66c23b3ac7604')
def test_signature_is_not_normalized(self):
request = AWSRequest()
request.url = 'https://s3.amazonaws.com/bucket/foo/./bar/../bar'
request.method = 'GET'
credentials = botocore.credentials.Credentials('access_key',
'secret_key')
auth = botocore.auth.S3SigV4Auth(credentials, 's3', 'us-east-1')
auth.add_auth(request)
self.assertTrue(
request.headers['Authorization'].startswith('AWS4-HMAC-SHA256'))
def test_presign_with_security_token(self):
self.credentials.token = 'security-token'
auth = botocore.auth.S3SigV4QueryAuth(
self.credentials, self.service_name, self.region_name, expires=60)
request = AWSRequest()
request.url = 'https://ec2.us-east-1.amazonaws.com/'
self.auth.add_auth(request)
query_string = self.get_parsed_query_string(request)
self.assertEqual(
query_string['X-Amz-Security-Token'], 'security-token')
def test_content_sha256_set_if_payload_signing_disabled(self):
request = AWSRequest()
request.data = six.BytesIO(u'\u2713'.encode('utf-8'))
request.url = 'https://amazonaws.com'
request.context['payload_signing_enabled'] = False
request.method = 'PUT'
auth = self.create_signer()
auth.add_auth(request)
sha_header = request.headers['X-Amz-Content-SHA256']
self.assertEqual(sha_header, 'UNSIGNED-PAYLOAD')
def _test_blacklist_header(self, header, value):
request = AWSRequest()
request.url = 'https://s3.amazonaws.com/bucket/foo'
request.method = 'PUT'
request.headers[header] = value
credentials = botocore.credentials.Credentials('access_key',
'secret_key')
auth = botocore.auth.S3SigV4Auth(credentials, 's3', 'us-east-1')
auth.add_auth(request)
self.assertNotIn(header, request.headers['Authorization'])
def test_discover_endpoint_no_protocol(self):
request = AWSRequest()
request.context = {
'discovery': {'identifiers': {}}
}
self.manager.describe_endpoint.return_value = 'new.foo'
self.handler.discover_endpoint(request, 'TestOperation')
self.assertEqual(request.url, 'https://new.foo')
self.manager.describe_endpoint.assert_called_with(
Operation='TestOperation', Identifiers={}
)
def test_presign_content_type_form_encoded_not_signed(self):
request = AWSRequest()
request.method = 'GET'
request.url = 'https://myservice.us-east-1.amazonaws.com/'
request.headers['Content-Type'] = (
'application/x-www-form-urlencoded; charset=utf-8'
)
self.auth.add_auth(request)
query_string = self.get_parsed_query_string(request)
signed_headers = query_string.get('X-Amz-SignedHeaders')
self.assertNotIn('content-type', signed_headers)
def test_discover_endpoint_fails(self):
request = AWSRequest()
request.context = {
'discovery': {'identifiers': {}}
}
request.url = 'old.com'
self.manager.describe_endpoint.return_value = None
self.handler.discover_endpoint(request, 'TestOperation')
self.assertEqual(request.url, 'old.com')
self.manager.describe_endpoint.assert_called_with(
Operation='TestOperation', Identifiers={}
)
def test_fields(self):
request = AWSRequest()
request.url = '/'
request.method = 'POST'
request.data = {'Foo': u'\u2713'}
self.signer.add_auth(request)
self.assertEqual(request.data['AWSAccessKeyId'], 'foo')
self.assertEqual(request.data['Foo'], u'\u2713')
self.assertEqual(request.data['Timestamp'], '2014-06-20T08:40:23Z')
self.assertEqual(request.data['Signature'],
u'Tiecw+t51tok4dTT8B4bg47zxHEM/KcD55f2/x6K22o=')
self.assertEqual(request.data['SignatureMethod'], 'HmacSHA256')
self.assertEqual(request.data['SignatureVersion'], '2')
def test_resign(self):
# Make sure that resigning after e.g. retries works
request = AWSRequest()
request.url = '/'
request.method = 'POST'
params = {
'Foo': u'\u2713',
'Signature': u'VCtWuwaOL0yMffAT8W4y0AFW3W4KUykBqah9S40rB+Q='
}
result = self.signer.calc_signature(request, params)
self.assertEqual(
result, ('Foo=%E2%9C%93',
u'VCtWuwaOL0yMffAT8W4y0AFW3W4KUykBqah9S40rB+Q='))
def test_get(self):
request = AWSRequest()
request.url = '/'
request.method = 'GET'
request.params = {'Foo': u'\u2713'}
self.signer.add_auth(request)
self.assertEqual(request.params['AWSAccessKeyId'], 'foo')
self.assertEqual(request.params['Foo'], u'\u2713')
self.assertEqual(request.params['Timestamp'], '2014-06-20T08:40:23Z')
self.assertEqual(request.params['Signature'],
u'Un97klqZCONP65bA1+Iv4H3AcB2I40I4DBvw5ZERFPw=')
self.assertEqual(request.params['SignatureMethod'], 'HmacSHA256')
self.assertEqual(request.params['SignatureVersion'], '2')
def test_presign_no_params(self):
request = AWSRequest()
request.url = 'https://ec2.us-east-1.amazonaws.com/'
self.auth.add_auth(request)
query_string = self.get_parsed_query_string(request)
self.assertEqual(
query_string,
{'X-Amz-Algorithm': 'AWS4-HMAC-SHA256',
'X-Amz-Credential': 'access_key/20140101/myregion/myservice/aws4_request',
'X-Amz-Date': '20140101T000000Z',
'X-Amz-Expires': '60',
'X-Amz-Signature': 'c70e0bcdb4cd3ee324f71c78195445b8788315af0800bbbdbbb6d05a616fb84c',
'X-Amz-SignedHeaders': 'host'})
def create_request_from_raw_request(raw_request):
raw_request = raw_request.replace('http/1.1', 'HTTP/1.1')
request = AWSRequest()
raw = RawHTTPRequest(raw_request)
if raw.error_code is not None:
raise Exception(raw.error_message)
request.method = raw.command
datetime_now = datetime.datetime(2011, 9, 9, 23, 36)
request.context['timestamp'] = datetime_now.strftime('%Y%m%dT%H%M%SZ')
for key, val in raw.headers.items():
request.headers[key] = val
request.data = raw.rfile.read()
host = raw.headers.get('host', '')
# For whatever reason, the BaseHTTPRequestHandler encodes
# the first line of the response as 'iso-8859-1',
# so we need decode this into utf-8.
if isinstance(raw.path, six.text_type):
raw.path = raw.path.encode('iso-8859-1').decode('utf-8')
url = 'https://%s%s' % (host, raw.path)
if '?' in url:
split_url = urlsplit(url)
params = dict(parse_qsl(split_url.query))
request.url = split_url.path
request.params = params
else:
request.url = url
return request
def test_sign_with_token(self):
credentials = botocore.credentials.Credentials(
access_key='foo', secret_key='bar', token='baz')
auth = botocore.auth.HmacV1Auth(credentials)
request = AWSRequest()
request.headers['Date'] = 'Thu, 17 Nov 2005 18:49:58 GMT'
request.headers['Content-Type'] = 'text/html'
request.method = 'PUT'
request.url = 'https://s3.amazonaws.com/bucket/key'
auth.add_auth(request)
self.assertIn('Authorization', request.headers)
# We're not actually checking the signature here, we're
# just making sure the auth header has the right format.
self.assertTrue(request.headers['Authorization'].startswith('AWS '))
def test_query_string_params_in_urls(self):
request = AWSRequest()
request.url = (
'https://s3.amazonaws.com/bucket?'
'marker=%C3%A4%C3%B6%C3%BC-01.txt&prefix'
)
request.data = {'Action': 'MyOperation'}
request.method = 'GET'
# Check that the canonical query string is correct formatting
# by ensuring that query string paramters that are added to the
# canonical query string are correctly formatted.
cqs = self.auth.canonical_query_string(request)
self.assertEqual('marker=%C3%A4%C3%B6%C3%BC-01.txt&prefix=', cqs)
def test_resign_with_token(self):
credentials = botocore.credentials.Credentials(
access_key='foo', secret_key='bar', token='baz')
auth = botocore.auth.SigV3Auth(credentials)
request = AWSRequest()
request.headers['Date'] = 'Thu, 17 Nov 2005 18:49:58 GMT'
request.method = 'PUT'
request.url = 'https://route53.amazonaws.com/'
auth.add_auth(request)
original_auth = request.headers['X-Amzn-Authorization']
# Resigning the request shouldn't change the authorization
# header.
auth.add_auth(request)
self.assertEqual(request.headers.get_all('X-Amzn-Authorization'),
[original_auth])
def test_canonical_query_string(self):
request = AWSRequest()
request.url = (
'https://search-testdomain1-j67dwxlet67gf7ghwfmik2c67i.us-west-2.'
'cloudsearch.amazonaws.com/'
'2013-01-01/search?format=sdk&pretty=true&'
'q.options=%7B%22defaultOperator%22%3A%20%22and%22%2C%20%22'
'fields%22%3A%5B%22directors%5E10%22%5D%7D&q=George%20Lucas'
)
request.method = 'GET'
auth = self.create_signer('cloudsearchdomain', 'us-west-2')
actual = auth.canonical_query_string(request)
# Here 'q' should come before 'q.options'.
expected = ("format=sdk&pretty=true&q=George%20Lucas&q.options=%7B%22"
"defaultOperator%22%3A%20%22and%22%2C%20%22fields%22%3A%5B"
"%22directors%5E10%22%5D%7D")
self.assertEqual(actual, expected)
def test_empty_fields_and_policy(self):
self.request = AWSRequest()
self.request.url = 'https://s3.amazonaws.com/%s' % self.bucket
self.request.method = 'POST'
self.auth.add_auth(self.request)
result_fields = self.request.context['s3-presign-post-fields']
self.assertEqual(result_fields['x-amz-algorithm'], 'AWS4-HMAC-SHA256')
self.assertEqual(
result_fields['x-amz-credential'],
'access_key/20140101/myregion/myservice/aws4_request')
self.assertEqual(
result_fields['x-amz-date'],
'20140101T000000Z')
result_policy = json.loads(base64.b64decode(
result_fields['policy']).decode('utf-8'))
self.assertEqual(
result_policy['conditions'],
[{"x-amz-algorithm": "AWS4-HMAC-SHA256"},
{"x-amz-credential":
"access_key/20140101/myregion/myservice/aws4_request"},
{"x-amz-date": "20140101T000000Z"}])
self.assertIn('x-amz-signature', result_fields)
def test_resign_with_token(self):
credentials = botocore.credentials.Credentials(
access_key='foo', secret_key='bar', token='baz')
auth = botocore.auth.HmacV1Auth(credentials)
request = AWSRequest()
request.headers['Date'] = 'Thu, 17 Nov 2005 18:49:58 GMT'
request.headers['Content-Type'] = 'text/html'
request.method = 'PUT'
request.url = 'https://s3.amazonaws.com/bucket/key'
auth.add_auth(request)
original_auth = request.headers['Authorization']
# Resigning the request shouldn't change the authorization
# header. We are also ensuring that the date stays the same
# because we're mocking out the formatdate() call. There's
# another unit test that verifies we use the latest time
# when we sign the request.
auth.add_auth(request)
self.assertEqual(request.headers.get_all('Authorization'),
[original_auth])
class TestAWSRequest(unittest.TestCase):
def setUp(self):
self.tempdir = tempfile.mkdtemp()
self.request = AWSRequest(url='http://example.com')
self.prepared_request = self.request.prepare()
self.filename = os.path.join(self.tempdir, 'foo')
def tearDown(self):
shutil.rmtree(self.tempdir)
def test_should_reset_stream(self):
with open(self.filename, 'wb') as f:
f.write(b'foobarbaz')
with open(self.filename, 'rb') as body:
self.prepared_request.body = body
# Now pretend we try to send the request.
# This means that we read the body:
body.read()
# And create a response object that indicates
# a redirect.
fake_response = Mock()
fake_response.status_code = 307
# Then requests calls our reset_stream hook.
self.prepared_request.reset_stream_on_redirect(fake_response)
# The stream should now be reset.
self.assertEqual(body.tell(), 0)
def test_cannot_reset_stream_raises_error(self):
with open(self.filename, 'wb') as f:
f.write(b'foobarbaz')
with open(self.filename, 'rb') as body:
self.prepared_request.body = Unseekable(body)
# Now pretend we try to send the request.
# This means that we read the body:
body.read()
# And create a response object that indicates
# a redirect
fake_response = Mock()
fake_response.status_code = 307
# Then requests calls our reset_stream hook.
with self.assertRaises(UnseekableStreamError):
self.prepared_request.reset_stream_on_redirect(fake_response)
def test_duck_type_for_file_check(self):
# As part of determining whether or not we can rewind a stream
# we first need to determine if the thing is a file like object.
# We should not be using an isinstance check. Instead, we should
# be using duck type checks.
class LooksLikeFile(object):
def __init__(self):
self.seek_called = False
def read(self, amount=None):
pass
def seek(self, where):
self.seek_called = True
looks_like_file = LooksLikeFile()
self.prepared_request.body = looks_like_file
fake_response = Mock()
fake_response.status_code = 307
# Then requests calls our reset_stream hook.
self.prepared_request.reset_stream_on_redirect(fake_response)
# The stream should now be reset.
self.assertTrue(looks_like_file.seek_called)
def test_can_prepare_empty_body(self):
request = AWSRequest(url='http://example.com/', data=b'')
prepared_request = request.prepare()
self.assertEqual(prepared_request.body, None)
content_length = prepared_request.headers.get('content-length')
self.assertEqual(content_length, '0')
def test_can_prepare_dict_body_unicode_keys(self):
body = {u'\u30c6\u30b9\u30c8': 'string'}
expected_body = '%E3%83%86%E3%82%B9%E3%83%88=string'
request = AWSRequest(url='http://example.com/', data=body)
prepared_request = request.prepare()
self.assertEqual(prepared_request.body, expected_body)
def test_request_body_is_prepared(self):
request = AWSRequest(url='http://example.com/', data='body')
self.assertEqual(request.body, b'body')
def send_to_es(self,
path="/",
method=None,
payload=None,
extra_headers=None):
"""Low-level POST data to Amazon Elasticsearch Service generating a Sigv4 signed request
Args:
path (str): path to send to ES
method (str, optional): HTTP method default:GET
payload (dict, optional): additional payload used during POST or PUT
Returns:
dict: json answer converted in dict
Raises:
#: Error during ES communication
ESException: Description
"""
# resolve default kwargs
payload = payload or {}
extra_headers = extra_headers or {}
if not path.startswith("/"):
path = "/" + path
method = method if method else self.method
es_region = self.cfg["es_endpoint"].split(".")[1]
# send to ES with exponential backoff
retries = 0
while retries < int(self.cfg["es_max_retry"]):
if retries > 0:
seconds = (2**retries) * .1
# print('Waiting for %.1f seconds', seconds)
time.sleep(seconds)
extra_headers.update({"Host": self.cfg["es_endpoint"]})
req = AWSRequest(
method=method,
url=('https://%s%s?pretty&format=json' %
(self.cfg['es_endpoint'], urllib.parse.quote(path))),
data=payload,
headers=extra_headers)
credential_resolver = create_credential_resolver(get_session())
credentials = credential_resolver.load_credentials()
SigV4Auth(credentials, 'es', es_region).add_auth(req)
try:
preq = req.prepare()
session = Session()
res = session.send(preq)
if res.status_code >= 200 and res.status_code <= 299:
LOGGER.debug("%s %s", res.status_code, res.content)
return json.loads(res.content)
else:
LOGGER.debug("%s %s", res.status_code, res.content)
raise ESException(res.status_code, res.content)
except ESException as err:
if (err.status_code >= 500) and (err.status_code <= 599):
retries += 1 # Candidate for retry
else:
raise # Stop retrying, re-raise exception
class TestAWSRequest(unittest.TestCase):
def setUp(self):
self.tempdir = tempfile.mkdtemp()
self.filename = os.path.join(self.tempdir, 'foo')
self.request = AWSRequest(method='GET', url='http://example.com')
self.prepared_request = self.request.prepare()
self.prepared_request.prepare_headers(self.request.headers)
def tearDown(self):
shutil.rmtree(self.tempdir)
def test_prepared_request_repr(self):
expected_repr = (
'<AWSPreparedRequest stream_output=False, method=GET, '
'url=http://example.com, headers={}>'
)
request_repr = repr(self.prepared_request)
self.assertEqual(request_repr, expected_repr)
def test_can_prepare_url_params(self):
request = AWSRequest(url='http://example.com/', params={'foo': 'bar'})
prepared_request = request.prepare()
self.assertEqual(prepared_request.url, 'http://example.com/?foo=bar')
def test_can_prepare_dict_body(self):
body = {'dead': 'beef'}
request = AWSRequest(url='http://example.com/', data=body)
prepared_request = request.prepare()
self.assertEqual(prepared_request.body, 'dead=beef')
def test_can_prepare_empty_body(self):
request = AWSRequest(url='http://example.com/', data=b'')
prepared_request = request.prepare()
self.assertEqual(prepared_request.body, None)
content_length = prepared_request.headers.get('content-length')
self.assertEqual(content_length, '0')
def test_request_body_is_prepared(self):
request = AWSRequest(url='http://example.com/', data='body')
self.assertEqual(request.body, b'body')
def test_prepare_body_content_adds_content_length(self):
content = b'foobarbaz'
with open(self.filename, 'wb') as f:
f.write(content)
with open(self.filename, 'rb') as f:
data = Seekable(f)
self.prepared_request.prepare_body(data)
self.assertEqual(
self.prepared_request.headers['Content-Length'],
str(len(content)))
def test_can_reset_stream_handles_binary(self):
contents = b'notastream'
self.prepared_request.body = contents
self.prepared_request.reset_stream()
# assert the request body doesn't change after reset_stream is called
self.assertEqual(self.prepared_request.body, contents)
def test_can_reset_stream(self):
contents = b'foobarbaz'
with open(self.filename, 'wb') as f:
f.write(contents)
with open(self.filename, 'rb') as body:
self.prepared_request.body = body
# pretend the request body was partially sent
body.read()
self.assertNotEqual(body.tell(), 0)
# have the prepared request reset its stream
self.prepared_request.reset_stream()
# the stream should be reset
self.assertEqual(body.tell(), 0)
def test_cannot_reset_stream_raises_error(self):
contents = b'foobarbaz'
with open(self.filename, 'wb') as f:
f.write(contents)
with open(self.filename, 'rb') as body:
self.prepared_request.body = Unseekable(body)
# pretend the request body was partially sent
body.read()
self.assertNotEqual(body.tell(), 0)
# reset stream should fail
with self.assertRaises(UnseekableStreamError):
self.prepared_request.reset_stream()
def test_duck_type_for_file_check(self):
# As part of determining whether or not we can rewind a stream
# we first need to determine if the thing is a file like object.
# We should not be using an isinstance check. Instead, we should
# be using duck type checks.
class LooksLikeFile(object):
def __init__(self):
self.seek_called = False
def read(self, amount=None):
pass
def seek(self, where):
self.seek_called = True
looks_like_file = LooksLikeFile()
self.prepared_request.body = looks_like_file
self.prepared_request.reset_stream()
# The stream should now be reset.
self.assertTrue(looks_like_file.seek_called)
def test_can_prepare_dict_body(self):
body = {'dead': 'beef'}
request = AWSRequest(url='http://example.com/', data=body)
prepared_request = request.prepare()
self.assertEqual(prepared_request.body, 'dead=beef')
def test_inject_no_context(self):
request = AWSRequest(url='https://original.foo')
self.handler.discover_endpoint(request, 'TestOperation')
self.assertEqual(request.url, 'https://original.foo')
self.manager.describe_endpoint.assert_not_called()
def test_signed_host_is_ipv6_with_explicit_port(self):
endpoint = 'http://[::1]:6789'
expected_host = '[::1]:6789'
request = AWSRequest(method='GET', url=endpoint)
headers_to_sign = self.sigv4.headers_to_sign(request)
self.assertEqual(expected_host, headers_to_sign.get('host'))
def handle_method(fragment):
if fragment["Type"] != "AWS::ApiGateway::Method":
response_string = "Macro only supports \"AWS::ApiGateway::Method\", user supplied {}"
raise InvalidTypeException(response_string.format(fragment["Type"]))
service_name = fragment["Properties"]["Integration"].pop("Service").lower()
action = fragment["Properties"]["Integration"].pop("Action")
response_maps = fragment["Properties"]["Integration"].pop("ResponseMaps")
try:
fragment.pop("Fn::Transform")
except:
pass
loader = Loader()
service_description = loader.load_service_model(service_name=service_name, type_name='service-2')
service_model = ServiceModel(service_description)
protocol = service_model.protocol
op_model = service_model.operation_model(action["Name"])
request_parameters = action.get("Parameters", {})
params = dict(ChainMap(*request_parameters))
print("params: {}".format(params))
serializer = create_serializer(protocol)
response_parser = create_parser(protocol)
print(service_model.protocol)
request = serializer.serialize_to_request(params, op_model)
request_object = AWSRequest(
method=request['method'],
url=get_endpoint(service_model.service_name),
data=request['body'],
headers=request['headers'])
X = request_object.prepare()
print("Raw request: {}".format(request))
print("Prepared request: {}".format(X))
integration = fragment["Properties"]["Integration"]
new_integration = integration_template()
# Copy the existing values to the new template
for entry in integration.keys():
new_integration[entry] = integration[entry]
# Add headers to cfn template
if X.headers is not None and callable(getattr(X.headers, "keys", None)):
for header in X.headers.keys():
if header.lower() != 'Content-Length'.lower():
new_integration["RequestParameters"].update({"integration.request.header.{}".format(header): "'{}'".format(X.headers[header])})
# Add Query Strings to cfn template
if 'query_string' in request and callable(getattr(request['query_string'], "keys", None)):
for query in request['query_string'].keys():
new_integration["RequestParameters"].update({"integration.request.querystring.{}".format(query): "'{}'".format(request['query_string'][query])})
# Set the body
if isinstance(X.body, str):
new_integration["RequestTemplates"]["application/json"] = X.body
else:
new_integration["RequestTemplates"]["application/json"] = str(X.body, "utf-8") if X.body else ''
new_integration["Uri"] = ":".join([
"arn",
"aws",
"apigateway",
REGION,
service_model.endpoint_prefix,
"path/" + request["url_path"]
])
new_integration["IntegrationHttpMethod"] = X.method
fragment["Properties"]["Integration"] = new_integration
print(fragment)
return fragment
def signed_request(method, url, data=None, params=None, headers=None, service=None):
creds = boto3.Session().get_credentials().get_frozen_credentials()
request = AWSRequest(method=method, url=url, data=data, params=params, headers=headers)
SigV4Auth(creds, service, boto3.Session().region_name).add_auth(request)
return requests.request(method=method, url=url, headers=dict(request.headers), data=data)
def __call__(self, request: AWSRequest):
# Method hard coded to 'GET' as this prevents making accidental 'POSTS'
aws_request = AWSRequest(method='GET', url=self.url, data=request.body)
S3SigV4Auth(self.credentials, 's3', self.region).add_auth(aws_request)
request.headers.update(dict(aws_request.headers.items()))
return request
def __call__(self, request):
aws_request = AWSRequest(method=request.method.upper(),
url=request.url, data=request.body)
SigV4Auth(self.credentials, "es", self.region).add_auth(aws_request)
request.headers.update(dict(aws_request.headers.items()))
return request
def setUp(self):
self.original_url = 'https://s3.amazonaws.com/foo/key.txt'
self.request = AWSRequest(method='PUT',
headers={},
url=self.original_url)
def setUp(self):
self.tempdir = tempfile.mkdtemp()
self.request = AWSRequest(url='http://example.com')
self.prepared_request = self.request.prepare()
self.filename = os.path.join(self.tempdir, 'foo')
def test_signed_host_is_lowercase(self):
endpoint = 'https://S5.Us-WeAsT-2.AmAZonAwS.com'
expected_host = 's5.us-weast-2.amazonaws.com'
request = AWSRequest(method='GET', url=endpoint)
headers_to_sign = self.sigv4.headers_to_sign(request)
self.assertEqual(expected_host, headers_to_sign.get('host'))
class TestURLLib3Session(unittest.TestCase):
def setUp(self):
self.request = AWSRequest(
method='GET',
url='http://example.com/',
headers={},
data=b'',
)
self.response = Mock()
self.response.headers = {}
self.response.stream.return_value = b''
self.pool_manager = Mock()
self.connection = Mock()
self.connection.urlopen.return_value = self.response
self.pool_manager.connection_from_url.return_value = self.connection
self.pool_patch = patch('botocore.httpsession.PoolManager')
self.proxy_patch = patch('botocore.httpsession.proxy_from_url')
self.pool_manager_cls = self.pool_patch.start()
self.proxy_manager_fun = self.proxy_patch.start()
self.pool_manager_cls.return_value = self.pool_manager
self.proxy_manager_fun.return_value = self.pool_manager
def tearDown(self):
self.pool_patch.stop()
def assert_request_sent(self, headers=None, body=None, url='/'):
if headers is None:
headers = {}
self.connection.urlopen.assert_called_once_with(
method=self.request.method,
url=url,
body=body,
headers=headers,
retries=False,
assert_same_host=False,
preload_content=False,
decode_content=False,
)
def test_forwards_max_pool_size(self):
URLLib3Session(max_pool_connections=22)
self.pool_manager_cls.assert_called_with(
maxsize=22,
timeout=ANY,
strict=True,
ssl_context=ANY,
)
def test_basic_request(self):
session = URLLib3Session()
session.send(self.request.prepare())
self.assert_request_sent()
self.response.stream.assert_called_once_with()
def test_basic_streaming_request(self):
session = URLLib3Session()
self.request.stream_output = True
session.send(self.request.prepare())
self.assert_request_sent()
self.response.stream.assert_not_called()
def test_basic_https_request(self):
session = URLLib3Session()
self.request.url = 'https://example.com/'
session.send(self.request.prepare())
self.assert_request_sent()
def test_basic_https_proxy_request(self):
proxies = {'https': 'http://proxy.com'}
session = URLLib3Session(proxies=proxies)
self.request.url = 'https://example.com/'
session.send(self.request.prepare())
self.proxy_manager_fun.assert_any_call(
proxies['https'],
proxy_headers={},
maxsize=ANY,
timeout=ANY,
strict=True,
ssl_context=ANY,
)
self.assert_request_sent()
def test_basic_proxy_request_caches_manager(self):
proxies = {'https': 'http://proxy.com'}
session = URLLib3Session(proxies=proxies)
self.request.url = 'https://example.com/'
session.send(self.request.prepare())
# assert we created the proxy manager
self.proxy_manager_fun.assert_any_call(
proxies['https'],
proxy_headers={},
maxsize=ANY,
timeout=ANY,
strict=True,
ssl_context=ANY,
)
session.send(self.request.prepare())
# assert that we did not create another proxy manager
self.assertEqual(self.proxy_manager_fun.call_count, 1)
def test_basic_http_proxy_request(self):
proxies = {'http': 'http://proxy.com'}
session = URLLib3Session(proxies=proxies)
session.send(self.request.prepare())
self.proxy_manager_fun.assert_any_call(
proxies['http'],
proxy_headers={},
maxsize=ANY,
timeout=ANY,
strict=True,
ssl_context=ANY,
)
self.assert_request_sent(url=self.request.url)
def test_ssl_context_is_explicit(self):
session = URLLib3Session()
session.send(self.request.prepare())
_, manager_kwargs = self.pool_manager_cls.call_args
self.assertIsNotNone(manager_kwargs.get('ssl_context'))
def test_proxy_request_ssl_context_is_explicit(self):
proxies = {'http': 'http://proxy.com'}
session = URLLib3Session(proxies=proxies)
session.send(self.request.prepare())
_, proxy_kwargs = self.proxy_manager_fun.call_args
self.assertIsNotNone(proxy_kwargs.get('ssl_context'))
def make_request_with_error(self, error):
self.connection.urlopen.side_effect = error
session = URLLib3Session()
session.send(self.request.prepare())
@raises(EndpointConnectionError)
def test_catches_new_connection_error(self):
error = NewConnectionError(None, None)
self.make_request_with_error(error)
@raises(ConnectionClosedError)
def test_catches_bad_status_line(self):
error = ProtocolError(None)
self.make_request_with_error(error)
def test_aws_connection_classes_are_used(self):
session = URLLib3Session()
# ensure the pool manager is using the correct classes
http_class = self.pool_manager.pool_classes_by_scheme.get('http')
self.assertIs(http_class, AWSHTTPConnectionPool)
https_class = self.pool_manager.pool_classes_by_scheme.get('https')
self.assertIs(https_class, AWSHTTPSConnectionPool)
def make_request(endpoint, data, method='GET'):
"""
This function handles a HTTP request to a given elasticsearch domain endpoint and method.
:param endpoint: The elasticsearch domain endpoint
:param data:
:param method: the type of HTTP method to use. eg: 'GET POST DELETE etc'
:return: The response of the request
"""
host = endpoint
endpoint = '{0}/_bulk'.format(host)
region = endpoint.split('.')[1]
service = endpoint.split('.')[2]
credentials = boto3.session.Session().get_credentials()
request = AWSRequest(method=method,
url='https://{0}'.format(endpoint),
data=data)
SigV4Auth(credentials, service, region).add_auth(request)
headers = dict(request.headers.items())
opener = urllib2.build_opener(urllib2.HTTPHandler)
request = urllib2.Request('https://{0}'.format(endpoint), request.data)
request.add_header('Host', host)
request.add_header('Content-Type', 'application/json')
request.add_header('X-Amz-Date', headers['X-Amz-Date'])
request.add_header('X-Amz-Security-Token', headers['X-Amz-Security-Token'])
request.add_header('Authorization', headers['Authorization'])
request.get_method = lambda: method
print(request.data)
return opener.open(request).read()
def test_can_prepare_url_params(self):
request = AWSRequest(url='http://example.com/', params={'foo': 'bar'})
prepared_request = request.prepare()
self.assertEqual(prepared_request.url, 'http://example.com/?foo=bar')
class TestURLLib3Session(unittest.TestCase):
def setUp(self):
self.request = AWSRequest(
method='GET',
url='http://example.com/',
headers={},
data=b'',
)
self.response = Mock()
self.response.headers = {}
self.response.stream.return_value = b''
self.pool_manager = Mock()
self.connection = Mock()
self.connection.urlopen.return_value = self.response
self.pool_manager.connection_from_url.return_value = self.connection
self.pool_patch = patch('botocore.httpsession.PoolManager')
self.proxy_patch = patch('botocore.httpsession.proxy_from_url')
self.pool_manager_cls = self.pool_patch.start()
self.proxy_manager_fun = self.proxy_patch.start()
self.pool_manager_cls.return_value = self.pool_manager
self.proxy_manager_fun.return_value = self.pool_manager
def tearDown(self):
self.pool_patch.stop()
self.proxy_patch.stop()
def assert_request_sent(self,
headers=None,
body=None,
url='/',
chunked=False):
if headers is None:
headers = {}
self.connection.urlopen.assert_called_once_with(
method=self.request.method,
url=url,
body=body,
headers=headers,
retries=False,
assert_same_host=False,
preload_content=False,
decode_content=False,
chunked=chunked,
)
def _assert_manager_call(self, manager, *assert_args, **assert_kwargs):
call_kwargs = {
'strict': True,
'maxsize': ANY,
'timeout': ANY,
'ssl_context': ANY,
'socket_options': [],
'cert_file': None,
'key_file': None,
}
call_kwargs.update(assert_kwargs)
manager.assert_called_with(*assert_args, **call_kwargs)
def assert_pool_manager_call(self, *args, **kwargs):
self._assert_manager_call(self.pool_manager_cls, *args, **kwargs)
def assert_proxy_manager_call(self, *args, **kwargs):
self._assert_manager_call(self.proxy_manager_fun, *args, **kwargs)
def test_forwards_max_pool_size(self):
URLLib3Session(max_pool_connections=22)
self.assert_pool_manager_call(maxsize=22)
def test_forwards_client_cert(self):
URLLib3Session(client_cert='/some/cert')
self.assert_pool_manager_call(cert_file='/some/cert', key_file=None)
def test_forwards_client_cert_and_key_tuple(self):
cert = ('/some/cert', '/some/key')
URLLib3Session(client_cert=cert)
self.assert_pool_manager_call(cert_file=cert[0], key_file=cert[1])
def test_basic_https_proxy_with_client_cert(self):
proxies = {'https': 'http://proxy.com'}
session = URLLib3Session(proxies=proxies, client_cert='/some/cert')
self.request.url = 'https://example.com/'
session.send(self.request.prepare())
self.assert_proxy_manager_call(
proxies['https'],
proxy_headers={},
cert_file='/some/cert',
key_file=None,
)
self.assert_request_sent()
def test_basic_https_proxy_with_client_cert_and_key(self):
cert = ('/some/cert', '/some/key')
proxies = {'https': 'http://proxy.com'}
session = URLLib3Session(proxies=proxies, client_cert=cert)
self.request.url = 'https://example.com/'
session.send(self.request.prepare())
self.assert_proxy_manager_call(
proxies['https'],
proxy_headers={},
cert_file=cert[0],
key_file=cert[1],
)
self.assert_request_sent()
def test_basic_request(self):
session = URLLib3Session()
session.send(self.request.prepare())
self.assert_request_sent()
self.response.stream.assert_called_once_with()
def test_basic_streaming_request(self):
session = URLLib3Session()
self.request.stream_output = True
session.send(self.request.prepare())
self.assert_request_sent()
self.response.stream.assert_not_called()
def test_basic_https_request(self):
session = URLLib3Session()
self.request.url = 'https://example.com/'
session.send(self.request.prepare())
self.assert_request_sent()
def test_basic_https_proxy_request(self):
proxies = {'https': 'http://proxy.com'}
session = URLLib3Session(proxies=proxies)
self.request.url = 'https://example.com/'
session.send(self.request.prepare())
self.assert_proxy_manager_call(proxies['https'], proxy_headers={})
self.assert_request_sent()
def test_basic_proxy_request_caches_manager(self):
proxies = {'https': 'http://proxy.com'}
session = URLLib3Session(proxies=proxies)
self.request.url = 'https://example.com/'
session.send(self.request.prepare())
# assert we created the proxy manager
self.assert_proxy_manager_call(proxies['https'], proxy_headers={})
session.send(self.request.prepare())
# assert that we did not create another proxy manager
self.assertEqual(self.proxy_manager_fun.call_count, 1)
def test_basic_http_proxy_request(self):
proxies = {'http': 'http://proxy.com'}
session = URLLib3Session(proxies=proxies)
session.send(self.request.prepare())
self.assert_proxy_manager_call(proxies['http'], proxy_headers={})
self.assert_request_sent(url=self.request.url)
def test_ssl_context_is_explicit(self):
session = URLLib3Session()
session.send(self.request.prepare())
_, manager_kwargs = self.pool_manager_cls.call_args
self.assertIsNotNone(manager_kwargs.get('ssl_context'))
def test_proxy_request_ssl_context_is_explicit(self):
proxies = {'http': 'http://proxy.com'}
session = URLLib3Session(proxies=proxies)
session.send(self.request.prepare())
_, proxy_kwargs = self.proxy_manager_fun.call_args
self.assertIsNotNone(proxy_kwargs.get('ssl_context'))
def test_session_forwards_socket_options_to_pool_manager(self):
socket_options = [(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)]
URLLib3Session(socket_options=socket_options)
self.assert_pool_manager_call(socket_options=socket_options)
def test_session_forwards_socket_options_to_proxy_manager(self):
proxies = {'http': 'http://proxy.com'}
socket_options = [(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)]
session = URLLib3Session(
proxies=proxies,
socket_options=socket_options,
)
session.send(self.request.prepare())
self.assert_proxy_manager_call(
proxies['http'],
proxy_headers={},
socket_options=socket_options,
)
def make_request_with_error(self, error):
self.connection.urlopen.side_effect = error
session = URLLib3Session()
session.send(self.request.prepare())
@raises(EndpointConnectionError)
def test_catches_new_connection_error(self):
error = NewConnectionError(None, None)
self.make_request_with_error(error)
@raises(ConnectionClosedError)
def test_catches_bad_status_line(self):
error = ProtocolError(None)
self.make_request_with_error(error)
def test_aws_connection_classes_are_used(self):
session = URLLib3Session()
# ensure the pool manager is using the correct classes
http_class = self.pool_manager.pool_classes_by_scheme.get('http')
self.assertIs(http_class, AWSHTTPConnectionPool)
https_class = self.pool_manager.pool_classes_by_scheme.get('https')
self.assertIs(https_class, AWSHTTPSConnectionPool)
def test_chunked_encoding_is_set_with_header(self):
session = URLLib3Session()
self.request.headers['Transfer-Encoding'] = 'chunked'
session.send(self.request.prepare())
self.assert_request_sent(
chunked=True,
headers={'Transfer-Encoding': 'chunked'},
)
def test_chunked_encoding_is_not_set_without_header(self):
session = URLLib3Session()
session.send(self.request.prepare())
self.assert_request_sent(chunked=False)
class TestURLLib3Session(unittest.TestCase):
def setUp(self):
self.request = AWSRequest(
method='GET',
url='http://example.com/',
headers={},
data=b'',
)
self.response = mock.Mock()
self.response.headers = {}
self.response.stream.return_value = b''
self.pool_manager = mock.Mock()
self.connection = mock.Mock()
self.connection.urlopen.return_value = self.response
self.pool_manager.connection_from_url.return_value = self.connection
self.pool_patch = mock.patch('botocore.httpsession.PoolManager')
self.proxy_patch = mock.patch('botocore.httpsession.proxy_from_url')
self.pool_manager_cls = self.pool_patch.start()
self.proxy_manager_fun = self.proxy_patch.start()
self.pool_manager_cls.return_value = self.pool_manager
self.proxy_manager_fun.return_value = self.pool_manager
def tearDown(self):
self.pool_patch.stop()
self.proxy_patch.stop()
def assert_request_sent(self, headers=None, body=None, url='/', chunked=False):
if headers is None:
headers = {}
self.connection.urlopen.assert_called_once_with(
method=self.request.method,
url=url,
body=body,
headers=headers,
retries=mock.ANY,
assert_same_host=False,
preload_content=False,
decode_content=False,
chunked=chunked,
)
def _assert_manager_call(self, manager, *assert_args, **assert_kwargs):
call_kwargs = {
'strict': True,
'maxsize': mock.ANY,
'timeout': mock.ANY,
'ssl_context': mock.ANY,
'socket_options': [],
'cert_file': None,
'key_file': None,
}
call_kwargs.update(assert_kwargs)
manager.assert_called_with(*assert_args, **call_kwargs)
def assert_pool_manager_call(self, *args, **kwargs):
self._assert_manager_call(self.pool_manager_cls, *args, **kwargs)
def assert_proxy_manager_call(self, *args, **kwargs):
self._assert_manager_call(self.proxy_manager_fun, *args, **kwargs)
def test_forwards_max_pool_size(self):
URLLib3Session(max_pool_connections=22)
self.assert_pool_manager_call(maxsize=22)
def test_forwards_client_cert(self):
URLLib3Session(client_cert='/some/cert')
self.assert_pool_manager_call(cert_file='/some/cert', key_file=None)
def test_forwards_client_cert_and_key_tuple(self):
cert = ('/some/cert', '/some/key')
URLLib3Session(client_cert=cert)
self.assert_pool_manager_call(cert_file=cert[0], key_file=cert[1])
def test_proxies_config_settings(self):
proxies = {'http': 'http://proxy.com'}
proxies_config = {
'proxy_ca_bundle': 'path/to/bundle',
'proxy_client_cert': ('path/to/cert', 'path/to/key'),
'proxy_use_forwarding_for_https': False,
}
use_forwarding = proxies_config['proxy_use_forwarding_for_https']
with mock.patch('botocore.httpsession.create_urllib3_context'):
session = URLLib3Session(
proxies=proxies,
proxies_config=proxies_config
)
self.request.url = 'http://example.com/'
session.send(self.request.prepare())
self.assert_proxy_manager_call(
proxies['http'],
proxy_headers={},
proxy_ssl_context=mock.ANY,
use_forwarding_for_https=use_forwarding
)
self.assert_request_sent(url=self.request.url)
def test_proxies_config_settings_unknown_config(self):
proxies = {'http': 'http://proxy.com'}
proxies_config = {
'proxy_ca_bundle': None,
'proxy_client_cert': None,
'proxy_use_forwarding_for_https': True,
'proxy_not_a_real_arg': 'do not pass'
}
use_forwarding = proxies_config['proxy_use_forwarding_for_https']
session = URLLib3Session(
proxies=proxies,
proxies_config=proxies_config
)
self.request.url = 'http://example.com/'
session.send(self.request.prepare())
self.assert_proxy_manager_call(
proxies['http'],
proxy_headers={},
use_forwarding_for_https=use_forwarding
)
self.assertNotIn(
'proxy_not_a_real_arg',
self.proxy_manager_fun.call_args
)
self.assert_request_sent(url=self.request.url)
def test_http_proxy_scheme_with_http_url(self):
proxies = {'http': 'http://proxy.com'}
session = URLLib3Session(proxies=proxies)
self.request.url = 'http://example.com/'
session.send(self.request.prepare())
self.assert_proxy_manager_call(
proxies['http'],
proxy_headers={},
)
self.assert_request_sent(url=self.request.url)
def test_http_proxy_scheme_with_https_url(self):
proxies = {'https': 'http://proxy.com'}
session = URLLib3Session(proxies=proxies)
self.request.url = 'https://example.com/'
session.send(self.request.prepare())
self.assert_proxy_manager_call(
proxies['https'],
proxy_headers={},
)
self.assert_request_sent()
def test_https_proxy_scheme_with_http_url(self):
proxies = {'http': 'https://proxy.com'}
session = URLLib3Session(proxies=proxies)
self.request.url = 'http://example.com/'
session.send(self.request.prepare())
self.assert_proxy_manager_call(
proxies['http'],
proxy_headers={},
)
self.assert_request_sent(url=self.request.url)
def test_https_proxy_scheme_tls_in_tls(self):
proxies = {'https': 'https://proxy.com'}
session = URLLib3Session(proxies=proxies)
self.request.url = 'https://example.com/'
session.send(self.request.prepare())
self.assert_proxy_manager_call(
proxies['https'],
proxy_headers={},
)
self.assert_request_sent()
def test_https_proxy_scheme_forwarding_https_url(self):
proxies = {'https': 'https://proxy.com'}
proxies_config = {"proxy_use_forwarding_for_https": True}
session = URLLib3Session(proxies=proxies, proxies_config=proxies_config)
self.request.url = 'https://example.com/'
session.send(self.request.prepare())
self.assert_proxy_manager_call(
proxies['https'],
proxy_headers={},
use_forwarding_for_https=True,
)
self.assert_request_sent(url=self.request.url)
def test_basic_https_proxy_with_client_cert(self):
proxies = {'https': 'http://proxy.com'}
session = URLLib3Session(proxies=proxies, client_cert='/some/cert')
self.request.url = 'https://example.com/'
session.send(self.request.prepare())
self.assert_proxy_manager_call(
proxies['https'],
proxy_headers={},
cert_file='/some/cert',
key_file=None,
)
self.assert_request_sent()
def test_basic_https_proxy_with_client_cert_and_key(self):
cert = ('/some/cert', '/some/key')
proxies = {'https': 'http://proxy.com'}
session = URLLib3Session(proxies=proxies, client_cert=cert)
self.request.url = 'https://example.com/'
session.send(self.request.prepare())
self.assert_proxy_manager_call(
proxies['https'],
proxy_headers={},
cert_file=cert[0],
key_file=cert[1],
)
self.assert_request_sent()
def test_urllib3_proxies_kwargs_included(self):
cert = ('/some/cert', '/some/key')
proxies = {'https': 'https://proxy.com'}
proxies_config = {'proxy_client_cert': "path/to/cert"}
with mock.patch('botocore.httpsession.create_urllib3_context'):
session = URLLib3Session(
proxies=proxies, client_cert=cert,
proxies_config=proxies_config
)
self.request.url = 'https://example.com/'
session.send(self.request.prepare())
self.assert_proxy_manager_call(
proxies['https'],
proxy_headers={},
cert_file=cert[0],
key_file=cert[1],
proxy_ssl_context=mock.ANY
)
self.assert_request_sent()
def test_proxy_ssl_context_uses_check_hostname(self):
cert = ('/some/cert', '/some/key')
proxies = {'https': 'https://proxy.com'}
proxies_config = {'proxy_client_cert': "path/to/cert"}
with mock.patch('botocore.httpsession.create_urllib3_context'):
session = URLLib3Session(
proxies=proxies, client_cert=cert,
proxies_config=proxies_config
)
self.request.url = 'https://example.com/'
session.send(self.request.prepare())
last_call = self.proxy_manager_fun.call_args[-1]
self.assertIs(last_call['ssl_context'].check_hostname, True)
def test_basic_request(self):
session = URLLib3Session()
session.send(self.request.prepare())
self.assert_request_sent()
self.response.stream.assert_called_once_with()
def test_basic_streaming_request(self):
session = URLLib3Session()
self.request.stream_output = True
session.send(self.request.prepare())
self.assert_request_sent()
self.response.stream.assert_not_called()
def test_basic_https_request(self):
session = URLLib3Session()
self.request.url = 'https://example.com/'
session.send(self.request.prepare())
self.assert_request_sent()
def test_basic_https_proxy_request(self):
proxies = {'https': 'http://proxy.com'}
session = URLLib3Session(proxies=proxies)
self.request.url = 'https://example.com/'
session.send(self.request.prepare())
self.assert_proxy_manager_call(proxies['https'], proxy_headers={})
self.assert_request_sent()
def test_basic_proxy_request_caches_manager(self):
proxies = {'https': 'http://proxy.com'}
session = URLLib3Session(proxies=proxies)
self.request.url = 'https://example.com/'
session.send(self.request.prepare())
# assert we created the proxy manager
self.assert_proxy_manager_call(proxies['https'], proxy_headers={})
session.send(self.request.prepare())
# assert that we did not create another proxy manager
self.assertEqual(self.proxy_manager_fun.call_count, 1)
def test_basic_http_proxy_request(self):
proxies = {'http': 'http://proxy.com'}
session = URLLib3Session(proxies=proxies)
session.send(self.request.prepare())
self.assert_proxy_manager_call(proxies['http'], proxy_headers={})
self.assert_request_sent(url=self.request.url)
def test_ssl_context_is_explicit(self):
session = URLLib3Session()
session.send(self.request.prepare())
_, manager_kwargs = self.pool_manager_cls.call_args
self.assertIsNotNone(manager_kwargs.get('ssl_context'))
def test_proxy_request_ssl_context_is_explicit(self):
proxies = {'http': 'http://proxy.com'}
session = URLLib3Session(proxies=proxies)
session.send(self.request.prepare())
_, proxy_kwargs = self.proxy_manager_fun.call_args
self.assertIsNotNone(proxy_kwargs.get('ssl_context'))
def test_session_forwards_socket_options_to_pool_manager(self):
socket_options = [(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)]
URLLib3Session(socket_options=socket_options)
self.assert_pool_manager_call(socket_options=socket_options)
def test_session_forwards_socket_options_to_proxy_manager(self):
proxies = {'http': 'http://proxy.com'}
socket_options = [(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)]
session = URLLib3Session(
proxies=proxies,
socket_options=socket_options,
)
session.send(self.request.prepare())
self.assert_proxy_manager_call(
proxies['http'],
proxy_headers={},
socket_options=socket_options,
)
def make_request_with_error(self, error):
self.connection.urlopen.side_effect = error
session = URLLib3Session()
session.send(self.request.prepare())
def test_catches_new_connection_error(self):
error = NewConnectionError(None, None)
with pytest.raises(EndpointConnectionError):
self.make_request_with_error(error)
def test_catches_bad_status_line(self):
error = ProtocolError(None)
with pytest.raises(ConnectionClosedError):
self.make_request_with_error(error)
def test_catches_proxy_error(self):
self.connection.urlopen.side_effect = ProxyError('test', None)
session = URLLib3Session(proxies={'http': 'http://*****:*****@proxy.com'})
with pytest.raises(ProxyConnectionError) as e:
session.send(self.request.prepare())
assert 'user:pass' not in str(e.value)
assert 'http://***:***@proxy.com' in str(e.value)
def test_aws_connection_classes_are_used(self):
session = URLLib3Session() # noqa
# ensure the pool manager is using the correct classes
http_class = self.pool_manager.pool_classes_by_scheme.get('http')
self.assertIs(http_class, AWSHTTPConnectionPool)
https_class = self.pool_manager.pool_classes_by_scheme.get('https')
self.assertIs(https_class, AWSHTTPSConnectionPool)
def test_chunked_encoding_is_set_with_header(self):
session = URLLib3Session()
self.request.headers['Transfer-Encoding'] = 'chunked'
session.send(self.request.prepare())
self.assert_request_sent(
chunked=True,
headers={'Transfer-Encoding': 'chunked'},
)
def test_chunked_encoding_is_not_set_without_header(self):
session = URLLib3Session()
session.send(self.request.prepare())
self.assert_request_sent(chunked=False)