def _modify_request_before_signing(self, request):
# We automatically set this header, so if it's the auto-set value we
# want to get rid of it since it doesn't make sense for presigned urls.
content_type = request.headers.get('content-type')
blacklisted_content_type = (
'application/x-www-form-urlencoded; charset=utf-8'
)
if content_type == blacklisted_content_type:
del request.headers['content-type']
# Note that we're not including X-Amz-Signature.
# From the docs: "The Canonical Query String must include all the query
# parameters from the preceding table except for X-Amz-Signature.
signed_headers = self.signed_headers(self.headers_to_sign(request))
auth_params = {
'X-Amz-Algorithm': 'AWS4-HMAC-SHA256',
'X-Amz-Credential': self.scope(request),
'X-Amz-Date': request.context['timestamp'],
'X-Amz-Expires': self._expires,
'X-Amz-SignedHeaders': signed_headers,
}
if self.credentials.token is not None:
auth_params['X-Amz-Security-Token'] = self.credentials.token
# Now parse the original query string to a dict, inject our new query
# params, and serialize back to a query string.
url_parts = urlsplit(request.url)
# parse_qs makes each value a list, but in our case we know we won't
# have repeated keys so we know we have single element lists which we
# can convert back to scalar values.
query_dict = dict(
[(k, v[0]) for k, v in
parse_qs(url_parts.query, keep_blank_values=True).items()])
# The spec is particular about this. It *has* to be:
# https://<endpoint>?<operation params>&<auth params>
# You can't mix the two types of params together, i.e just keep doing
# new_query_params.update(op_params)
# new_query_params.update(auth_params)
# percent_encode_sequence(new_query_params)
operation_params = ''
if request.data:
# We also need to move the body params into the query string. To
# do this, we first have to convert it to a dict.
query_dict.update(self._get_body_as_dict(request))
request.data = ''
if query_dict:
operation_params = percent_encode_sequence(query_dict) + '&'
new_query_string = (operation_params +
percent_encode_sequence(auth_params))
# url_parts is a tuple (and therefore immutable) so we need to create
# a new url_parts with the new query string.
# <part> - <index>
# scheme - 0
# netloc - 1
# path - 2
# query - 3 <-- we're replacing this.
# fragment - 4
p = url_parts
new_url_parts = (p[0], p[1], p[2], new_query_string, p[4])
request.url = urlunsplit(new_url_parts)
def _modify_request_before_signing(self, request):
# We automatically set this header, so if it's the auto-set value we
# want to get rid of it since it doesn't make sense for presigned urls.
content_type = request.headers.get('content-type')
blacklisted_content_type = (
'application/x-www-form-urlencoded; charset=utf-8'
)
if content_type == blacklisted_content_type:
del request.headers['content-type']
# Note that we're not including X-Amz-Signature.
# From the docs: "The Canonical Query String must include all the query
# parameters from the preceding table except for X-Amz-Signature.
signed_headers = self.signed_headers(self.headers_to_sign(request))
auth_params = {
'X-Amz-Algorithm': 'AWS4-HMAC-SHA256',
'X-Amz-Credential': self.scope(request),
'X-Amz-Date': request.context['timestamp'],
'X-Amz-Expires': self._expires,
'X-Amz-SignedHeaders': signed_headers,
}
if self.credentials.token is not None:
auth_params['X-Amz-Security-Token'] = self.credentials.token
# Now parse the original query string to a dict, inject our new query
# params, and serialize back to a query string.
url_parts = urlsplit(request.url)
# parse_qs makes each value a list, but in our case we know we won't
# have repeated keys so we know we have single element lists which we
# can convert back to scalar values.
query_dict = dict(
[(k, v[0]) for k, v in
parse_qs(url_parts.query, keep_blank_values=True).items()])
# The spec is particular about this. It *has* to be:
# https://<endpoint>?<operation params>&<auth params>
# You can't mix the two types of params together, i.e just keep doing
# new_query_params.update(op_params)
# new_query_params.update(auth_params)
# percent_encode_sequence(new_query_params)
operation_params = ''
if request.data:
# We also need to move the body params into the query string. To
# do this, we first have to convert it to a dict.
query_dict.update(self._get_body_as_dict(request))
request.data = ''
if query_dict:
operation_params = percent_encode_sequence(query_dict) + '&'
new_query_string = (operation_params +
percent_encode_sequence(auth_params))
# url_parts is a tuple (and therefore immutable) so we need to create
# a new url_parts with the new query string.
# <part> - <index>
# scheme - 0
# netloc - 1
# path - 2
# query - 3 <-- we're replacing this.
# fragment - 4
p = url_parts
new_url_parts = (p[0], p[1], p[2], new_query_string, p[4])
request.url = urlunsplit(new_url_parts)
def _modify_request_before_signing(self, request):
# This is our chance to add additional query params we need
# before we go about calculating the signature.
request.headers = {}
request.method = 'GET'
# Note that we're not including X-Amz-Signature.
# From the docs: "The Canonical Query String must include all the query
# parameters from the preceding table except for X-Amz-Signature.
auth_params = {
'X-Amz-Algorithm': 'AWS4-HMAC-SHA256',
'X-Amz-Credential': self.scope(request),
'X-Amz-Date': request.context['timestamp'],
'X-Amz-Expires': self._expires,
'X-Amz-SignedHeaders': 'host',
}
if self.credentials.token is not None:
auth_params['X-Amz-Security-Token'] = self.credentials.token
# Now parse the original query string to a dict, inject our new query
# params, and serialize back to a query string.
url_parts = urlsplit(request.url)
# parse_qs makes each value a list, but in our case we know we won't
# have repeated keys so we know we have single element lists which we
# can convert back to scalar values.
query_dict = dict([(k, v[0])
for k, v in parse_qs(url_parts.query).items()])
# The spec is particular about this. It *has* to be:
# https://<endpoint>?<operation params>&<auth params>
# You can't mix the two types of params together, i.e just keep doing
# new_query_params.update(op_params)
# new_query_params.update(auth_params)
# percent_encode_sequence(new_query_params)
operation_params = ''
if request.data:
# We also need to move the body params into the query string.
# request.data will be populated, for example, with query services
# which normally form encode the params into the body.
# This means that request.data is a dict() of the operation params.
query_dict.update(request.data)
request.data = ''
if query_dict:
operation_params = percent_encode_sequence(query_dict) + '&'
new_query_string = (operation_params +
percent_encode_sequence(auth_params))
# url_parts is a tuple (and therefore immutable) so we need to create
# a new url_parts with the new query string.
# <part> - <index>
# scheme - 0
# netloc - 1
# path - 2
# query - 3 <-- we're replacing this.
# fragment - 4
p = url_parts
new_url_parts = (p[0], p[1], p[2], new_query_string, p[4])
request.url = urlunsplit(new_url_parts)
def _modify_request_before_signing(self, request):
# This is our chance to add additional query params we need
# before we go about calculating the signature.
request.headers = {}
request.method = 'GET'
# Note that we're not including X-Amz-Signature.
# From the docs: "The Canonical Query String must include all the query
# parameters from the preceding table except for X-Amz-Signature.
auth_params = {
'X-Amz-Algorithm': 'AWS4-HMAC-SHA256',
'X-Amz-Credential': self.scope(request),
'X-Amz-Date': self.timestamp,
'X-Amz-Expires': self._expires,
'X-Amz-SignedHeaders': 'host',
}
if self.credentials.token is not None:
auth_params['X-Amz-Security-Token'] = self.credentials.token
# Now parse the original query string to a dict, inject our new query
# params, and serialize back to a query string.
url_parts = urlsplit(request.url)
# parse_qs makes each value a list, but in our case we know we won't
# have repeated keys so we know we have single element lists which we
# can convert back to scalar values.
query_dict = dict(
[(k, v[0]) for k, v in parse_qs(url_parts.query).items()])
# The spec is particular about this. It *has* to be:
# https://<endpoint>?<operation params>&<auth params>
# You can't mix the two types of params together, i.e just keep doing
# new_query_params.update(op_params)
# new_query_params.update(auth_params)
# percent_encode_sequence(new_query_params)
operation_params = ''
if request.data:
# We also need to move the body params into the query string.
# request.data will be populated, for example, with query services
# which normally form encode the params into the body.
# This means that request.data is a dict() of the operation params.
query_dict.update(request.data)
request.data = ''
if query_dict:
operation_params = percent_encode_sequence(query_dict) + '&'
new_query_string = (operation_params +
percent_encode_sequence(auth_params))
# url_parts is a tuple (and therefore immutable) so we need to create
# a new url_parts with the new query string.
# <part> - <index>
# scheme - 0
# netloc - 1
# path - 2
# query - 3 <-- we're replacing this.
# fragment - 4
p = url_parts
new_url_parts = (p[0], p[1], p[2], new_query_string, p[4])
request.url = urlunsplit(new_url_parts)
def _serialize_request_description(request_dict):
if isinstance(request_dict.get('body'), dict):
# urlencode the request body.
encoded = percent_encode_sequence(request_dict['body']).encode('utf-8')
request_dict['body'] = encoded
if isinstance(request_dict.get('query_string'), dict):
encoded = percent_encode_sequence(request_dict.pop('query_string'))
if encoded:
# 'requests' automatically handle this, but we in the
# test runner we need to handle the case where the url_path
# already has query params.
if '?' not in request_dict['url_path']:
request_dict['url_path'] += '?%s' % encoded
else:
request_dict['url_path'] += '&%s' % encoded
def prepare_request_dict(request_dict, endpoint_url, user_agent=None):
"""
This method prepares a request dict to be created into an
AWSRequestObject. This prepares the request dict by adding the
url and the user agent to the request dict.
:type request_dict: dict
:param request_dict: The request dict (created from the
``serialize`` module).
:type user_agent: string
:param user_agent: The user agent to use for this request.
:type endpoint_url: string
:param endpoint_url: The full endpoint url, which contains at least
the scheme, the hostname, and optionally any path components.
"""
r = request_dict
if user_agent is not None:
headers = r['headers']
headers['User-Agent'] = user_agent
url = _urljoin(endpoint_url, r['url_path'])
if r['query_string']:
encoded_query_string = percent_encode_sequence(r['query_string'])
if '?' not in url:
url += '?%s' % encoded_query_string
else:
url += '&%s' % encoded_query_string
r['url'] = url
def _inject_signature(self, request, signature):
query_dict = {}
query_dict['AWSAccessKeyId'] = self.credentials.access_key
query_dict['Signature'] = signature
for header_key in request.headers:
lk = header_key.lower()
# For query string requests, Expires is used instead of the
# Date header.
if header_key == 'Date':
query_dict['Expires'] = request.headers['Date']
# We only want to include relevant headers in the query string.
# These can be anything that starts with x-amz, is Content-MD5,
# or is Content-Type.
elif lk.startswith('x-amz-') or lk in ['content-md5',
'content-type']:
query_dict[lk] = request.headers[lk]
# Combine all of the identified headers into an encoded
# query string
new_query_string = percent_encode_sequence(query_dict)
# Create a new url with the presigned url.
p = urlsplit(request.url)
if p[3]:
# If there was a pre-existing query string, we should
# add that back before injecting the new query string.
new_query_string ='%s&%s' % (p[3], new_query_string)
new_url_parts = (p[0], p[1], p[2], new_query_string, p[4])
request.url = urlunsplit(new_url_parts)
def _inject_signature(self, request, signature):
query_dict = {}
query_dict['AWSAccessKeyId'] = self.credentials.access_key
query_dict['Signature'] = signature
for header_key in request.headers:
lk = header_key.lower()
# For query string requests, Expires is used instead of the
# Date header.
if header_key == 'Date':
query_dict['Expires'] = request.headers['Date']
# We only want to include relevant headers in the query string.
# These can be anything that starts with x-amz, is Content-MD5,
# or is Content-Type.
elif lk.startswith('x-amz-') or lk in [
'content-md5', 'content-type'
]:
query_dict[lk] = request.headers[lk]
# Combine all of the identified headers into an encoded
# query string
new_query_string = percent_encode_sequence(query_dict)
# Create a new url with the presigned url.
p = urlsplit(request.url)
if p[3]:
# If there was a pre-existing query string, we should
# add that back before injecting the new query string.
new_query_string = '%s&%s' % (p[3], new_query_string)
new_url_parts = (p[0], p[1], p[2], new_query_string, p[4])
request.url = urlunsplit(new_url_parts)
def prepare_request_dict(request_dict, endpoint_url, user_agent=None):
"""
This method prepares a request dict to be created into an
AWSRequestObject. This prepares the request dict by adding the
url and the user agent to the request dict.
:type request_dict: dict
:param request_dict: The request dict (created from the
``serialize`` module).
:type user_agent: string
:param user_agent: The user agent to use for this request.
:type endpoint_url: string
:param endpoint_url: The full endpoint url, which contains at least
the scheme, the hostname, and optionally any path components.
"""
r = request_dict
if user_agent is not None:
headers = r['headers']
headers['User-Agent'] = user_agent
url = _urljoin(endpoint_url, r['url_path'])
if r['query_string']:
encoded_query_string = percent_encode_sequence(r['query_string'])
if '?' not in url:
url += '?%s' % encoded_query_string
else:
url += '&%s' % encoded_query_string
r['url'] = url
def create_request_object(self, request_dict, user_agent, endpoint_url):
"""
:type request_dict: dict
:param request_dict: The request dict (created from the ``serialize``
module).
:type user_agent: string
:param user_agent: The user agent to use for this request.
:type endpoint_url: string
:param endpoint_url: The full endpoint url, which contains at least
the scheme, the hostname, and optionally any path components.
:rtype: ``botocore.awsrequest.AWSRequest``
:return: An AWSRequest object based on the request_dict.
"""
r = request_dict
headers = r['headers']
headers['User-Agent'] = user_agent
url = self._urljoin(endpoint_url, r['url_path'])
if r['query_string']:
encoded_query_string = percent_encode_sequence(r['query_string'])
if '?' not in url:
url += '?%s' % encoded_query_string
else:
url += '&%s' % encoded_query_string
request = AWSRequest(method=r['method'], url=url,
data=r['body'],
headers=headers)
return request
def create_request_object(self, request_dict, user_agent, endpoint_url):
"""
:type request_dict: dict
:param request_dict: The request dict (created from the ``serialize``
module).
:type user_agent: string
:param user_agent: The user agent to use for this request.
:type endpoint_url: string
:param endpoint_url: The full endpoint url, which contains at least
the scheme, the hostname, and optionally any path components.
:rtype: ``botocore.awsrequest.AWSRequest``
:return: An AWSRequest object based on the request_dict.
"""
r = request_dict
headers = r['headers']
headers['User-Agent'] = user_agent
url = self._urljoin(endpoint_url, r['url_path'])
if r['query_string']:
encoded_query_string = percent_encode_sequence(r['query_string'])
if '?' not in url:
url += '?%s' % encoded_query_string
else:
url += '&%s' % encoded_query_string
request = AWSRequest(method=r['method'],
url=url,
data=r['body'],
headers=headers)
return request
def _create_request_object(self, request_dict):
r = request_dict
user_agent = self._user_agent
headers = r['headers']
headers['User-Agent'] = user_agent
url = urljoin(self.host, r['url_path'])
if r['query_string']:
encoded_query_string = percent_encode_sequence(r['query_string'])
if '?' not in url:
url += '?%s' % encoded_query_string
else:
url += '&%s' % encoded_query_string
request = AWSRequest(method=r['method'], url=url,
data=r['body'],
headers=headers)
return request
def _create_request_object(self, request_dict):
r = request_dict
user_agent = self._user_agent
headers = r['headers']
headers['User-Agent'] = user_agent
url = urljoin(self.host, r['url_path'])
if r['query_string']:
encoded_query_string = percent_encode_sequence(r['query_string'])
if '?' not in url:
url += '?%s' % encoded_query_string
else:
url += '&%s' % encoded_query_string
request = AWSRequest(method=r['method'],
url=url,
data=r['body'],
headers=headers)
return request
def _modify_request_before_signing(self, request):
super()._modify_request_before_signing(request)
# We automatically set this header, so if it's the auto-set value we
# want to get rid of it since it doesn't make sense for presigned urls.
content_type = request.headers.get('content-type')
if content_type == 'application/x-www-form-urlencoded; charset=utf-8':
del request.headers['content-type']
# Now parse the original query string to a dict, inject our new query
# params, and serialize back to a query string.
url_parts = urlsplit(request.url)
# parse_qs makes each value a list, but in our case we know we won't
# have repeated keys so we know we have single element lists which we
# can convert back to scalar values.
query_dict = dict(
[(k, v[0]) for k, v in
parse_qs(url_parts.query, keep_blank_values=True).items()])
if request.params:
query_dict.update(request.params)
request.params = {}
# The spec is particular about this. It *has* to be:
# https://<endpoint>?<operation params>&<auth params>
# You can't mix the two types of params together, i.e just keep doing
# new_query_params.update(op_params)
# new_query_params.update(auth_params)
# percent_encode_sequence(new_query_params)
if request.data:
# We also need to move the body params into the query string. To
# do this, we first have to convert it to a dict.
query_dict.update(_get_body_as_dict(request))
request.data = ''
new_query_string = percent_encode_sequence(query_dict)
# url_parts is a tuple (and therefore immutable) so we need to create
# a new url_parts with the new query string.
# <part> - <index>
# scheme - 0
# netloc - 1
# path - 2
# query - 3 <-- we're replacing this.
# fragment - 4
p = url_parts
new_url_parts = (p[0], p[1], p[2], new_query_string, p[4])
request.url = urlunsplit(new_url_parts)
def test_percent_encode_list_values_of_string(self):
self.assertEqual(
percent_encode_sequence(
OrderedDict([('k1', ['a', 'list']),
('k2', ['another', 'list'])])),
'k1=a&k1=list&k2=another&k2=list')
def _serialize_to_header_value(self, tags):
return percent_encode_sequence([(tag['Key'], tag['Value'])
for tag in tags])
def test_percent_encode_dict_string_string(self):
self.assertEqual(
percent_encode_sequence(OrderedDict([('k1', 'v1'), ('k2', 'v2')])),
'k1=v1&k2=v2')
def test_percent_encode_single_list_of_values(self):
self.assertEqual(percent_encode_sequence({'k1': ['a', 'b', 'c']}),
'k1=a&k1=b&k1=c')
def test_percent_encode_string_string_tuples(self):
self.assertEqual(percent_encode_sequence([('k1', 'v1'), ('k2', 'v2')]),
'k1=v1&k2=v2')
def test_percent_encode_dict_single_pair(self):
self.assertEqual(percent_encode_sequence({'k1': 'v1'}), 'k1=v1')
def test_percent_encode_empty(self):
self.assertEqual(percent_encode_sequence({}), '')
def test_percent_encode_special_chars(self):
self.assertEqual(
percent_encode_sequence({'k1': 'with spaces++/'}),
'k1=with%20spaces%2B%2B%2F')
def test_percent_encode_special_chars(self):
self.assertEqual(percent_encode_sequence({'k1': 'with spaces++/'}),
'k1=with%20spaces%2B%2B%2F')
def test_percent_encode_empty(self):
self.assertEqual(percent_encode_sequence({}), '')
def test_percent_encode_string_string_tuples(self):
self.assertEqual(percent_encode_sequence([('k1', 'v1'), ('k2', 'v2')]),
'k1=v1&k2=v2')
def test_percent_encode_dict_single_pair(self):
self.assertEqual(percent_encode_sequence({'k1': 'v1'}), 'k1=v1')
def test_percent_encode_dict_string_string(self):
self.assertEqual(
percent_encode_sequence(OrderedDict([('k1', 'v1'), ('k2', 'v2')])),
'k1=v1&k2=v2')
def test_percent_encode_single_list_of_values(self):
self.assertEqual(percent_encode_sequence({'k1': ['a', 'b', 'c']}),
'k1=a&k1=b&k1=c')
def test_percent_encode_list_values_of_string(self):
self.assertEqual(
percent_encode_sequence(
OrderedDict([('k1', ['a', 'list']), ('k2', ['another',
'list'])])),
'k1=a&k1=list&k2=another&k2=list')
