def test_s3_sigv4_presign(self):
auth = ibm_botocore.auth.S3SigV4QueryAuth(self.credentials,
self.service_name,
self.region_name,
expires=60)
request = AWSRequest()
request.method = 'GET'
request.url = (
'https://s3.us-west-2.amazonaws.com/mybucket/keyname/.bar')
auth.add_auth(request)
query_string = self.get_parsed_query_string(request)
# We use a different payload:
self.assertEqual(auth.payload(request), 'UNSIGNED-PAYLOAD')
# which will result in a different X-Amz-Signature:
self.assertEqual(
query_string, {
'X-Amz-Algorithm':
'AWS4-HMAC-SHA256',
'X-Amz-Credential': ('access_key/20140101/myregion/'
'myservice/aws4_request'),
'X-Amz-Date':
'20140101T000000Z',
'X-Amz-Expires':
'60',
'X-Amz-Signature': ('ac1b8b9e47e8685c5c963d75e35e8741d55251'
'cd955239cc1efad4dc7201db66'),
'X-Amz-SignedHeaders':
'host'
})
def setUp(self):
self.access_key = 'access_key'
self.secret_key = 'secret_key'
self.credentials = ibm_botocore.credentials.Credentials(
self.access_key, self.secret_key)
self.service_name = 'myservice'
self.region_name = 'myregion'
self.bucket = 'mybucket'
self.key = 'mykey'
self.policy = {
"expiration": "2007-12-01T12:00:00.000Z",
"conditions": [
{"acl": "public-read"},
{"bucket": self.bucket},
["starts-with", "$key", self.key],
]
}
self.fields = {
'key': self.key,
'acl': 'public-read',
}
self.request = AWSRequest()
self.request.url = 'https://s3.amazonaws.com/%s' % self.bucket
self.request.method = 'POST'
self.request.context['s3-presign-post-fields'] = self.fields
self.request.context['s3-presign-post-policy'] = self.policy
def create_request_from_raw_request(raw_request):
request = AWSRequest()
raw = RawHTTPRequest(raw_request)
if raw.error_code is not None:
raise Exception(raw.error_message)
request.method = raw.command
datetime_now = DATE
request.context['timestamp'] = datetime_now.strftime('%Y%m%dT%H%M%SZ')
for key, val in raw.headers.items():
request.headers[key] = val
request.data = raw.rfile.read()
host = raw.headers.get('host', '')
# For whatever reason, the BaseHTTPRequestHandler encodes
# the first line of the response as 'iso-8859-1',
# so we need decode this into utf-8.
if isinstance(raw.path, six.text_type):
raw.path = raw.path.encode('iso-8859-1').decode('utf-8')
url = 'https://%s%s' % (host, raw.path)
if '?' in url:
split_url = urlsplit(url)
params = dict(parse_qsl(split_url.query))
request.url = split_url.path
request.params = params
else:
request.url = url
return request
def test_empty_fields_and_policy(self):
self.request = AWSRequest()
self.request.url = 'https://s3.amazonaws.com/%s' % self.bucket
self.request.method = 'POST'
self.auth.add_auth(self.request)
result_fields = self.request.context['s3-presign-post-fields']
self.assertEqual(result_fields['x-amz-algorithm'], 'AWS4-HMAC-SHA256')
self.assertEqual(
result_fields['x-amz-credential'],
'access_key/20140101/myregion/myservice/aws4_request')
self.assertEqual(result_fields['x-amz-date'], '20140101T000000Z')
result_policy = json.loads(
base64.b64decode(result_fields['policy']).decode('utf-8'))
self.assertEqual(
result_policy['conditions'], [{
"x-amz-algorithm": "AWS4-HMAC-SHA256"
}, {
"x-amz-credential":
"access_key/20140101/myregion/myservice/aws4_request"
}, {
"x-amz-date": "20140101T000000Z"
}])
self.assertIn('x-amz-signature', result_fields)
def test_auth_header_preserved_from_s3_redirects(self):
request = AWSRequest()
request.url = 'https://bucket.s3.amazonaws.com/'
request.method = 'GET'
request.headers['Authorization'] = 'original auth header'
prepared_request = request.prepare()
fake_response = Mock()
fake_response.headers = {
'location': 'https://bucket.s3-us-west-2.amazonaws.com'}
fake_response.url = request.url
fake_response.status_code = 307
fake_response.is_permanent_redirect = False
# This line is needed to disable the cookie handling
# code in requests.
fake_response.raw._original_response = None
success_response = Mock()
success_response.raw._original_response = None
success_response.is_redirect = False
success_response.status_code = 200
session = BotocoreHTTPSession()
session.send = Mock(return_value=success_response)
list(session.resolve_redirects(
fake_response, prepared_request, stream=False))
redirected_request = session.send.call_args[0][0]
# The Authorization header for the newly sent request should
# still have our original Authorization header.
self.assertEqual(
redirected_request.headers['Authorization'],
'original auth header')
def test_payload_not_signed_if_disabled_in_context(self):
request = AWSRequest()
request.data = u'\u2713'.encode('utf-8')
request.url = 'https://amazonaws.com'
request.context['payload_signing_enabled'] = False
auth = self.create_signer()
payload = auth.payload(request)
self.assertEqual(payload, 'UNSIGNED-PAYLOAD')
def test_strips_default_port_and_http_auth(self):
request = AWSRequest()
request.url = 'http://*****:*****@s3.us-west-2.amazonaws.com:80/'
request.method = 'GET'
auth = self.create_signer('s3', 'us-west-2')
actual = auth.headers_to_sign(request)['host']
expected = 's3.us-west-2.amazonaws.com'
self.assertEqual(actual, expected)
def test_presign_with_spaces_in_param(self):
request = AWSRequest()
request.method = 'GET'
request.url = 'https://ec2.us-east-1.amazonaws.com/'
request.data = {'Action': 'MyOperation', 'Description': 'With Spaces'}
self.auth.add_auth(request)
# Verify we encode spaces as '%20, and we don't use '+'.
self.assertIn('Description=With%20Spaces', request.url)
def test_presign_with_empty_param_value(self):
request = AWSRequest()
request.method = 'POST'
# actual URL format for creating a multipart upload
request.url = 'https://s3.amazonaws.com/mybucket/mykey?uploads'
self.auth.add_auth(request)
# verify that uploads param is still in URL
self.assertIn('uploads', request.url)
def test_operation_params_before_auth_params(self):
# The spec is picky about this.
request = AWSRequest()
request.method = 'GET'
request.url = 'https://ec2.us-east-1.amazonaws.com/?Action=MyOperation'
self.auth.add_auth(request)
# Verify auth params come after the existing params.
self.assertIn('?Action=MyOperation&X-Amz', request.url)
def test_payload_is_bytes_type(self):
request = AWSRequest()
request.data = u'\u2713'.encode('utf-8')
auth = self.create_signer()
payload = auth.payload(request)
self.assertEqual(
payload,
'1dabba21cdad44541f6b15796f8d22978fc7ea10c46aeceeeeb66c23b3ac7604')
def test_can_prepare_url_params_with_existing_query(self):
request = AWSRequest(
url='http://example.com/?bar=foo', params={'foo': 'bar'}
)
prepared_request = request.prepare()
self.assertEqual(
prepared_request.url, 'http://example.com/?bar=foo&foo=bar'
)
def test_signature_with_date_headers(self):
request = AWSRequest()
request.headers = {'Date': 'Thu, 17 Nov 2005 18:49:58 GMT'}
request.url = 'https://route53.amazonaws.com'
self.auth.add_auth(request)
self.assertEqual(
request.headers['X-Amzn-Authorization'],
('AWS3-HTTPS AWSAccessKeyId=access_key,Algorithm=HmacSHA256,'
'Signature=M245fo86nVKI8rLpH4HgWs841sBTUKuwciiTpjMDgPs='))
def test_operation_params_before_auth_params_in_body(self):
request = AWSRequest()
request.method = 'GET'
request.url = 'https://ec2.us-east-1.amazonaws.com/'
request.data = {'Action': 'MyOperation'}
self.auth.add_auth(request)
# Same situation, the params from request.data come before the auth
# params in the query string.
self.assertIn('?Action=MyOperation&X-Amz', request.url)
def test_switch_host_with_param(self):
request = AWSRequest()
url = 'https://machinelearning.us-east-1.amazonaws.com'
new_endpoint = 'https://my-custom-endpoint.amazonaws.com'
data = '{"PredictEndpoint":"%s"}' % new_endpoint
request.data = data.encode('utf-8')
request.url = url
handlers.switch_host_with_param(request, 'PredictEndpoint')
self.assertEqual(request.url, new_endpoint)
def test_payload_is_binary_file(self):
request = AWSRequest()
request.data = six.BytesIO(u'\u2713'.encode('utf-8'))
request.url = 'https://amazonaws.com'
auth = self.create_signer()
payload = auth.payload(request)
self.assertEqual(
payload,
'1dabba21cdad44541f6b15796f8d22978fc7ea10c46aeceeeeb66c23b3ac7604')
def setUp(self):
super(TestSigV4Resign, self).setUp()
self.credentials = ibm_botocore.credentials.Credentials(
access_key='foo', secret_key='bar', token='baz')
self.auth = ibm_botocore.auth.SigV4Auth(self.credentials,
'ec2', 'us-west-2')
self.request = AWSRequest()
self.request.method = 'PUT'
self.request.url = 'https://ec2.amazonaws.com/'
def test_signature_is_not_normalized(self):
request = AWSRequest()
request.url = 'https://s3.amazonaws.com/bucket/foo/./bar/../bar'
request.method = 'GET'
credentials = ibm_botocore.credentials.Credentials('access_key',
'secret_key')
auth = ibm_botocore.auth.S3SigV4Auth(credentials, 's3', 'us-east-1')
auth.add_auth(request)
self.assertTrue(
request.headers['Authorization'].startswith('AWS4-HMAC-SHA256'))
def test_presign_content_type_form_encoded_not_signed(self):
request = AWSRequest()
request.method = 'GET'
request.url = 'https://myservice.us-east-1.amazonaws.com/'
request.headers['Content-Type'] = (
'application/x-www-form-urlencoded; charset=utf-8')
self.auth.add_auth(request)
query_string = self.get_parsed_query_string(request)
signed_headers = query_string.get('X-Amz-SignedHeaders')
self.assertNotIn('content-type', signed_headers)
def test_content_sha256_set_if_payload_signing_disabled(self):
request = AWSRequest()
request.data = six.BytesIO(u'\u2713'.encode('utf-8'))
request.url = 'https://amazonaws.com'
request.context['payload_signing_enabled'] = False
request.method = 'PUT'
auth = self.create_signer()
auth.add_auth(request)
sha_header = request.headers['X-Amz-Content-SHA256']
self.assertEqual(sha_header, 'UNSIGNED-PAYLOAD')
def _test_blacklist_header(self, header, value):
request = AWSRequest()
request.url = 'https://s3.amazonaws.com/bucket/foo'
request.method = 'PUT'
request.headers[header] = value
credentials = ibm_botocore.credentials.Credentials('access_key',
'secret_key')
auth = ibm_botocore.auth.S3SigV4Auth(credentials, 's3', 'us-east-1')
auth.add_auth(request)
self.assertNotIn(header, request.headers['Authorization'])
def test_presign_with_security_token(self):
self.credentials.token = 'security-token'
auth = ibm_botocore.auth.S3SigV4QueryAuth(
self.credentials, self.service_name, self.region_name, expires=60)
request = AWSRequest()
request.method = 'GET'
request.url = 'https://ec2.us-east-1.amazonaws.com/'
auth.add_auth(request)
query_string = self.get_parsed_query_string(request)
self.assertEqual(
query_string['X-Amz-Security-Token'], 'security-token')
def test_query_string_params_in_urls(self):
request = AWSRequest()
request.url = ('https://s3.amazonaws.com/bucket?'
'marker=%C3%A4%C3%B6%C3%BC-01.txt&prefix')
request.data = {'Action': 'MyOperation'}
request.method = 'GET'
# Check that the canonical query string is correct formatting
# by ensuring that query string paramters that are added to the
# canonical query string are correctly formatted.
cqs = self.auth.canonical_query_string(request)
self.assertEqual('marker=%C3%A4%C3%B6%C3%BC-01.txt&prefix=', cqs)
def test_empty_fields_and_policy(self):
self.request = AWSRequest()
self.request.url = 'https://s3.amazonaws.com/%s' % self.bucket
self.request.method = 'POST'
self.auth.add_auth(self.request)
result_fields = self.request.context['s3-presign-post-fields']
self.assertEqual(
result_fields['AWSAccessKeyId'], self.credentials.access_key)
result_policy = json.loads(base64.b64decode(
result_fields['policy']).decode('utf-8'))
self.assertEqual(result_policy['conditions'], [])
self.assertIn('signature', result_fields)
def test_fields(self):
request = AWSRequest()
request.url = '/'
request.method = 'POST'
request.data = {'Foo': u'\u2713'}
self.signer.add_auth(request)
self.assertEqual(request.data['AWSAccessKeyId'], 'foo')
self.assertEqual(request.data['Foo'], u'\u2713')
self.assertEqual(request.data['Timestamp'], '2014-06-20T08:40:23Z')
self.assertEqual(request.data['Signature'],
u'Tiecw+t51tok4dTT8B4bg47zxHEM/KcD55f2/x6K22o=')
self.assertEqual(request.data['SignatureMethod'], 'HmacSHA256')
self.assertEqual(request.data['SignatureVersion'], '2')
def test_resign(self):
# Make sure that resigning after e.g. retries works
request = AWSRequest()
request.url = '/'
request.method = 'POST'
params = {
'Foo': u'\u2713',
'Signature': u'VCtWuwaOL0yMffAT8W4y0AFW3W4KUykBqah9S40rB+Q='
}
result = self.signer.calc_signature(request, params)
self.assertEqual(
result, ('Foo=%E2%9C%93',
u'VCtWuwaOL0yMffAT8W4y0AFW3W4KUykBqah9S40rB+Q='))
def test_get(self):
request = AWSRequest()
request.url = '/'
request.method = 'GET'
request.params = {'Foo': u'\u2713'}
self.signer.add_auth(request)
self.assertEqual(request.params['AWSAccessKeyId'], 'foo')
self.assertEqual(request.params['Foo'], u'\u2713')
self.assertEqual(request.params['Timestamp'], '2014-06-20T08:40:23Z')
self.assertEqual(request.params['Signature'],
u'Un97klqZCONP65bA1+Iv4H3AcB2I40I4DBvw5ZERFPw=')
self.assertEqual(request.params['SignatureMethod'], 'HmacSHA256')
self.assertEqual(request.params['SignatureVersion'], '2')
def test_sign_with_token(self):
credentials = ibm_botocore.credentials.Credentials(
access_key='foo', secret_key='bar', token='baz')
auth = ibm_botocore.auth.HmacV1Auth(credentials)
request = AWSRequest()
request.headers['Date'] = 'Thu, 17 Nov 2005 18:49:58 GMT'
request.headers['Content-Type'] = 'text/html'
request.method = 'PUT'
request.url = 'https://s3.amazonaws.com/bucket/key'
auth.add_auth(request)
self.assertIn('Authorization', request.headers)
# We're not actually checking the signature here, we're
# just making sure the auth header has the right format.
self.assertTrue(request.headers['Authorization'].startswith('AWS '))
def collect_body(self, params, model, **kwargs):
# A handler to simulate the reading of the body including the
# request-created event that signals to simulate the progress
# callbacks
if 'Body' in params:
# TODO: This is not ideal. Need to figure out a better idea of
# simulating reading of the request across the wire to trigger
# progress callbacks
request = AWSRequest(method='PUT',
url='https://s3.amazonaws.com',
data=params['Body'])
self.client.meta.events.emit('request-created.s3.%s' % model.name,
request=request,
operation_name=model.name)
self.sent_bodies.append(params['Body'].read())
def test_resign_with_token(self):
credentials = ibm_botocore.credentials.Credentials(
access_key='foo', secret_key='bar', token='baz')
auth = ibm_botocore.auth.SigV3Auth(credentials)
request = AWSRequest()
request.headers['Date'] = 'Thu, 17 Nov 2005 18:49:58 GMT'
request.method = 'PUT'
request.url = 'https://route53.amazonaws.com/'
auth.add_auth(request)
original_auth = request.headers['X-Amzn-Authorization']
# Resigning the request shouldn't change the authorization
# header.
auth.add_auth(request)
self.assertEqual(request.headers.get_all('X-Amzn-Authorization'),
[original_auth])
