def test_expand_multipart_filename(self):
headers = {'Host': '10.0.1.19:4572', 'User-Agent': 'curl/7.51.0',
'Accept': '*/*', 'Content-Length': '992', 'Expect': '100-continue',
'Content-Type': 'multipart/form-data; boundary=------------------------3c48c744237517ac'}
data1 = (b'--------------------------3c48c744237517ac\r\nContent-Disposition: form-data; name="key"\r\n\r\n'
b'uploads/20170826T181315.679087009Z/upload/${filename}\r\n--------------------------3c48c744237517ac'
b'\r\nContent-Disposition: form-data; name="AWSAccessKeyId"\r\n\r\nWHAT\r\n--------------------------'
b'3c48c744237517ac\r\nContent-Disposition: form-data; name="policy"\r\n\r\nNO\r\n--------------------'
b'------3c48c744237517ac\r\nContent-Disposition: form-data; name="signature"\r\n\r\nYUP\r\n----------'
b'----------------3c48c744237517ac\r\nContent-Disposition: form-data; name="acl"\r\n\r\nprivate\r\n--'
b'------------------------3c48c744237517ac\r\nContent-Disposition: form-data; name="success_action_re'
b'direct"\r\n\r\nhttp://127.0.0.1:5000/\r\n--------------------------3c48c744237517ac\r\nContent-Disp'
b'osition: form-data; name="file"; filename="pixel.png"\r\nContent-Type: application/octet-stream\r\n'
b'\r\n\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x06\x00\x00\x00\x1f\x15'
b'\xc4\x89\x00\x00\x00\x19tEXtSoftware\x00Adobe ImageReadyq\xc9e<\x00\x00\x00\x0eIDATx\xdabb\x00\x02'
b'\x80\x00\x03\x00\x00\x0f\x00\x03`|\xce\xe9\x00\x00\x00\x00IEND\xaeB`\x82\r\n-----------------------'
b'---3c48c744237517ac--\r\n')
data2 = (b'--------------------------3c48c744237517ac\r\nContent-Disposition: form-data; name="key"\r\n\r\n'
b'uploads/20170826T181315.679087009Z/upload/pixel.png\r\n--------------------------3c48c744237517ac'
b'\r\nContent-Disposition: form-data; name="AWSAccessKeyId"\r\n\r\nWHAT\r\n--------------------------'
b'3c48c744237517ac\r\nContent-Disposition: form-data; name="policy"\r\n\r\nNO\r\n--------------------'
b'------3c48c744237517ac\r\nContent-Disposition: form-data; name="signature"\r\n\r\nYUP\r\n----------'
b'----------------3c48c744237517ac\r\nContent-Disposition: form-data; name="acl"\r\n\r\nprivate\r\n--'
b'------------------------3c48c744237517ac\r\nContent-Disposition: form-data; name="success_action_re'
b'direct"\r\n\r\nhttp://127.0.0.1:5000/\r\n--------------------------3c48c744237517ac\r\nContent-Disp'
b'osition: form-data; name="file"; filename="pixel.png"\r\nContent-Type: application/octet-stream\r\n'
b'\r\n\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x06\x00\x00\x00\x1f\x15'
b'\xc4\x89\x00\x00\x00\x19tEXtSoftware\x00Adobe ImageReadyq\xc9e<\x00\x00\x00\x0eIDATx\xdabb\x00\x02'
b'\x80\x00\x03\x00\x00\x0f\x00\x03`|\xce\xe9\x00\x00\x00\x00IEND\xaeB`\x82\r\n-----------------------'
b'---3c48c744237517ac--\r\n')
data3 = (u'--------------------------3c48c744237517ac\r\nContent-Disposition: form-data; name="key"\r\n\r\n'
u'uploads/20170826T181315.679087009Z/upload/${filename}\r\n--------------------------3c48c744237517ac'
u'\r\nContent-Disposition: form-data; name="AWSAccessKeyId"\r\n\r\nWHAT\r\n--------------------------'
u'3c48c744237517ac\r\nContent-Disposition: form-data; name="policy"\r\n\r\nNO\r\n--------------------'
u'------3c48c744237517ac\r\nContent-Disposition: form-data; name="signature"\r\n\r\nYUP\r\n----------'
u'----------------3c48c744237517ac\r\nContent-Disposition: form-data; name="acl"\r\n\r\nprivate\r\n--'
u'------------------------3c48c744237517ac\r\nContent-Disposition: form-data; name="success_action_re'
u'direct"\r\n\r\nhttp://127.0.0.1:5000/\r\n--------------------------3c48c744237517ac\r\nContent-Disp'
u'osition: form-data; name="file"; filename="pixel.txt"\r\nContent-Type: text/plain\r\n\r\nHello World'
u'\r\n--------------------------3c48c744237517ac--\r\n')
expanded1 = multipart_content.expand_multipart_filename(data1, headers)
self.assertIsNot(expanded1, data1, 'Should have changed content of data with filename to interpolate')
self.assertIn(b'uploads/20170826T181315.679087009Z/upload/pixel.png', expanded1,
'Should see the interpolated filename')
expanded2 = multipart_content.expand_multipart_filename(data2, headers)
self.assertIs(expanded2, data2, 'Should not have changed content of data with no filename to interpolate')
expanded3 = multipart_content.expand_multipart_filename(data3, headers)
self.assertIsNot(expanded3, data3, 'Should have changed content of string data with filename to interpolate')
self.assertIn(b'uploads/20170826T181315.679087009Z/upload/pixel.txt', expanded3,
'Should see the interpolated filename')
def test_expand_multipart_filename(self):
headers = {'Host': '10.0.1.19:4572', 'User-Agent': 'curl/7.51.0',
'Accept': '*/*', 'Content-Length': '992', 'Expect': '100-continue',
'Content-Type': 'multipart/form-data; boundary=------------------------3c48c744237517ac'}
data1 = (b'--------------------------3c48c744237517ac\r\nContent-Disposition: form-data; name="key"\r\n\r\n'
b'uploads/20170826T181315.679087009Z/upload/${filename}\r\n--------------------------3c48c744237517ac'
b'\r\nContent-Disposition: form-data; name="AWSAccessKeyId"\r\n\r\nWHAT\r\n--------------------------'
b'3c48c744237517ac\r\nContent-Disposition: form-data; name="policy"\r\n\r\nNO\r\n--------------------'
b'------3c48c744237517ac\r\nContent-Disposition: form-data; name="signature"\r\n\r\nYUP\r\n----------'
b'----------------3c48c744237517ac\r\nContent-Disposition: form-data; name="acl"\r\n\r\nprivate\r\n--'
b'------------------------3c48c744237517ac\r\nContent-Disposition: form-data; name="success_action_re'
b'direct"\r\n\r\nhttp://127.0.0.1:5000/\r\n--------------------------3c48c744237517ac\r\nContent-Disp'
b'osition: form-data; name="file"; filename="pixel.png"\r\nContent-Type: application/octet-stream\r\n'
b'\r\n\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x06\x00\x00\x00\x1f\x15'
b'\xc4\x89\x00\x00\x00\x19tEXtSoftware\x00Adobe ImageReadyq\xc9e<\x00\x00\x00\x0eIDATx\xdabb\x00\x02'
b'\x80\x00\x03\x00\x00\x0f\x00\x03`|\xce\xe9\x00\x00\x00\x00IEND\xaeB`\x82\r\n-----------------------'
b'---3c48c744237517ac--\r\n')
data2 = (b'--------------------------3c48c744237517ac\r\nContent-Disposition: form-data; name="key"\r\n\r\n'
b'uploads/20170826T181315.679087009Z/upload/pixel.png\r\n--------------------------3c48c744237517ac'
b'\r\nContent-Disposition: form-data; name="AWSAccessKeyId"\r\n\r\nWHAT\r\n--------------------------'
b'3c48c744237517ac\r\nContent-Disposition: form-data; name="policy"\r\n\r\nNO\r\n--------------------'
b'------3c48c744237517ac\r\nContent-Disposition: form-data; name="signature"\r\n\r\nYUP\r\n----------'
b'----------------3c48c744237517ac\r\nContent-Disposition: form-data; name="acl"\r\n\r\nprivate\r\n--'
b'------------------------3c48c744237517ac\r\nContent-Disposition: form-data; name="success_action_re'
b'direct"\r\n\r\nhttp://127.0.0.1:5000/\r\n--------------------------3c48c744237517ac\r\nContent-Disp'
b'osition: form-data; name="file"; filename="pixel.png"\r\nContent-Type: application/octet-stream\r\n'
b'\r\n\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x06\x00\x00\x00\x1f\x15'
b'\xc4\x89\x00\x00\x00\x19tEXtSoftware\x00Adobe ImageReadyq\xc9e<\x00\x00\x00\x0eIDATx\xdabb\x00\x02'
b'\x80\x00\x03\x00\x00\x0f\x00\x03`|\xce\xe9\x00\x00\x00\x00IEND\xaeB`\x82\r\n-----------------------'
b'---3c48c744237517ac--\r\n')
data3 = (u'--------------------------3c48c744237517ac\r\nContent-Disposition: form-data; name="key"\r\n\r\n'
u'uploads/20170826T181315.679087009Z/upload/${filename}\r\n--------------------------3c48c744237517ac'
u'\r\nContent-Disposition: form-data; name="AWSAccessKeyId"\r\n\r\nWHAT\r\n--------------------------'
u'3c48c744237517ac\r\nContent-Disposition: form-data; name="policy"\r\n\r\nNO\r\n--------------------'
u'------3c48c744237517ac\r\nContent-Disposition: form-data; name="signature"\r\n\r\nYUP\r\n----------'
u'----------------3c48c744237517ac\r\nContent-Disposition: form-data; name="acl"\r\n\r\nprivate\r\n--'
u'------------------------3c48c744237517ac\r\nContent-Disposition: form-data; name="success_action_re'
u'direct"\r\n\r\nhttp://127.0.0.1:5000/\r\n--------------------------3c48c744237517ac\r\nContent-Disp'
u'osition: form-data; name="file"; filename="pixel.txt"\r\nContent-Type: text/plain\r\n\r\nHello World'
u'\r\n--------------------------3c48c744237517ac--\r\n')
expanded1 = multipart_content.expand_multipart_filename(data1, headers)
self.assertIsNot(expanded1, data1, 'Should have changed content of data with filename to interpolate')
self.assertIn(b'uploads/20170826T181315.679087009Z/upload/pixel.png', expanded1,
'Should see the interpolated filename')
expanded2 = multipart_content.expand_multipart_filename(data2, headers)
self.assertIs(expanded2, data2, 'Should not have changed content of data with no filename to interpolate')
expanded3 = multipart_content.expand_multipart_filename(data3, headers)
self.assertIsNot(expanded3, data3, 'Should have changed content of string data with filename to interpolate')
self.assertIn(b'uploads/20170826T181315.679087009Z/upload/pixel.txt', expanded3,
'Should see the interpolated filename')
def forward_request(self, method, path, data, headers):
# parse path and query params
parsed_path = urlparse.urlparse(path)
# Make sure we use 'localhost' as forward host, to ensure moto uses path style addressing.
# Note that all S3 clients using LocalStack need to enable path style addressing.
if 's3.amazonaws.com' not in headers.get('host', ''):
headers['host'] = 'localhost'
# check content md5 hash integrity if not a copy request
if 'Content-MD5' in headers and not self.is_s3_copy_request(headers, path):
response = check_content_md5(data, headers)
if response is not None:
return response
modified_data = None
# check bucket name
bucket_name = get_bucket_name(path, headers)
if method == 'PUT' and not re.match(BUCKET_NAME_REGEX, bucket_name):
if len(parsed_path.path) <= 1:
return error_response('Unable to extract valid bucket name. Please ensure that your AWS SDK is ' +
'configured to use path style addressing, or send a valid <Bucket>.s3.amazonaws.com "Host" header',
'InvalidBucketName', status_code=400)
return error_response('The specified bucket is not valid.', 'InvalidBucketName', status_code=400)
# TODO: For some reason, moto doesn't allow us to put a location constraint on us-east-1
to_find = to_bytes('<LocationConstraint>us-east-1</LocationConstraint>')
if data and data.startswith(to_bytes('<')) and to_find in data:
modified_data = data.replace(to_find, to_bytes(''))
# If this request contains streaming v4 authentication signatures, strip them from the message
# Related isse: https://github.com/localstack/localstack/issues/98
# TODO we should evaluate whether to replace moto s3 with scality/S3:
# https://github.com/scality/S3/issues/237
if headers.get(CONTENT_SHA256_HEADER) == STREAMING_HMAC_PAYLOAD:
modified_data = strip_chunk_signatures(modified_data or data)
headers['content-length'] = headers.get('x-amz-decoded-content-length')
# POST requests to S3 may include a "${filename}" placeholder in the
# key, which should be replaced with an actual file name before storing.
if method == 'POST':
original_data = modified_data or data
expanded_data = multipart_content.expand_multipart_filename(original_data, headers)
if expanded_data is not original_data:
modified_data = expanded_data
# If no content-type is provided, 'binary/octet-stream' should be used
# src: https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html
if method == 'PUT' and not headers.get('content-type'):
headers['content-type'] = 'binary/octet-stream'
# persist this API call to disk
persistence.record('s3', method, path, data, headers)
# parse query params
query = parsed_path.query
path = parsed_path.path
bucket = path.split('/')[1]
query_map = urlparse.parse_qs(query, keep_blank_values=True)
# remap metadata query params (not supported in moto) to request headers
append_metadata_headers(method, query_map, headers)
# apply fixes
headers_changed = fix_metadata_key_underscores(request_headers=headers)
if query == 'notification' or 'notification' in query_map:
# handle and return response for ?notification request
response = handle_notification_request(bucket, method, data)
return response
if query == 'cors' or 'cors' in query_map:
if method == 'GET':
return get_cors(bucket)
if method == 'PUT':
return set_cors(bucket, data)
if method == 'DELETE':
return delete_cors(bucket)
if query == 'lifecycle' or 'lifecycle' in query_map:
if method == 'GET':
return get_lifecycle(bucket)
if method == 'PUT':
return set_lifecycle(bucket, data)
if query == 'replication' or 'replication' in query_map:
if method == 'GET':
return get_replication(bucket)
if method == 'PUT':
return set_replication(bucket, data)
if query == 'encryption' or 'encryption' in query_map:
if method == 'GET':
return get_encryption(bucket)
if method == 'PUT':
return set_encryption(bucket, data)
if query == 'object-lock' or 'object-lock' in query_map:
if method == 'GET':
return get_object_lock(bucket)
if method == 'PUT':
return set_object_lock(bucket, data)
if modified_data is not None or headers_changed:
return Request(data=modified_data or data, headers=headers, method=method)
return True
def forward_request(self, method, path, data, headers):
# Make sure we use 'localhost' as forward host, to ensure moto uses path style addressing.
# Note that all S3 clients using LocalStack need to enable path style addressing.
if 's3.amazonaws.com' not in headers.get('host', ''):
headers['host'] = 'localhost'
# check content md5 hash integrity
if 'Content-MD5' in headers:
response = check_content_md5(data, headers)
if response is not None:
return response
modified_data = None
# TODO: For some reason, moto doesn't allow us to put a location constraint on us-east-1
to_find = to_bytes('<LocationConstraint>us-east-1</LocationConstraint>')
if data and data.startswith(to_bytes('<')) and to_find in data:
modified_data = data.replace(to_find, '')
# If this request contains streaming v4 authentication signatures, strip them from the message
# Related isse: https://github.com/localstack/localstack/issues/98
# TODO we should evaluate whether to replace moto s3 with scality/S3:
# https://github.com/scality/S3/issues/237
if headers.get('x-amz-content-sha256') == 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD':
modified_data = strip_chunk_signatures(modified_data or data)
headers['content-length'] = headers.get('x-amz-decoded-content-length')
# POST requests to S3 may include a "${filename}" placeholder in the
# key, which should be replaced with an actual file name before storing.
if method == 'POST':
original_data = modified_data or data
expanded_data = multipart_content.expand_multipart_filename(original_data, headers)
if expanded_data is not original_data:
modified_data = expanded_data
# If no content-type is provided, 'binary/octet-stream' should be used
# src: https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html
if method == 'PUT' and not headers.get('content-type'):
headers['content-type'] = 'binary/octet-stream'
# persist this API call to disk
persistence.record('s3', method, path, data, headers)
# parse query params
parsed = urlparse.urlparse(path)
query = parsed.query
path = parsed.path
bucket = path.split('/')[1]
query_map = urlparse.parse_qs(query, keep_blank_values=True)
if query == 'notification' or 'notification' in query_map:
# handle and return response for ?notification request
response = handle_notification_request(bucket, method, data)
return response
if query == 'cors' or 'cors' in query_map:
if method == 'GET':
return get_cors(bucket)
if method == 'PUT':
return set_cors(bucket, data)
if method == 'DELETE':
return delete_cors(bucket)
if query == 'lifecycle' or 'lifecycle' in query_map:
if method == 'GET':
return get_lifecycle(bucket)
if method == 'PUT':
return set_lifecycle(bucket, data)
if modified_data is not None:
return Request(data=modified_data, headers=headers, method=method)
return True
def forward_request(self, method, path, data, headers):
LOGGER.debug('forward_request - method: "%s"' % method)
modified_data = None
# If this request contains streaming v4 authentication signatures, strip them from the message
# Related isse: https://github.com/localstack/localstack/issues/98
# TODO we should evaluate whether to replace moto s3 with scality/S3:
# https://github.com/scality/S3/issues/237
if headers.get('x-amz-content-sha256'
) == 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD':
modified_data = strip_chunk_signatures(data)
# POST requests to S3 may include a "${filename}" placeholder in the
# key, which should be replaced with an actual file name before storing.
if method == 'POST':
original_data = modified_data or data
expanded_data = multipart_content.expand_multipart_filename(
original_data, headers)
if expanded_data is not original_data:
modified_data = expanded_data
# If no content-type is provided, 'binary/octet-stream' should be used
# src: https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html
if method == 'PUT' and not headers.get('content-type'):
headers['content-type'] = 'binary/octet-stream'
# persist this API call to disk
persistence.record('s3', method, path, data, headers)
parsed = urlparse.urlparse(path)
query = parsed.query
path = parsed.path
object_path = parsed.path
LOGGER.debug('forward_request - query: "%s"' % query)
LOGGER.debug('forward_request - path: "%s"' % path)
LOGGER.debug('forward_request - host: "%s"' % headers['host'])
LOGGER.debug('forward_request - object_path: "%s"' % object_path)
# Check bucket name in host
bucket = None
hostname_parts = headers['host'].split('.')
if len(hostname_parts) > 1:
bucket = hostname_parts[0]
# No bucket name in host, check in path.
if (not bucket or len(bucket) == 0):
bucket = get_bucket_name(path, headers)
LOGGER.debug('forward_request - Bucket name: "%s"' % bucket)
query_map = urlparse.parse_qs(query)
if query == 'notification' or 'notification' in query_map:
LOGGER.debug('forward_request - query: "%s"' % query)
response = Response()
response.status_code = 200
if method == 'GET':
# TODO check if bucket exists
result = '<NotificationConfiguration xmlns="%s">' % XMLNS_S3
if bucket in S3_NOTIFICATIONS:
notif = S3_NOTIFICATIONS[bucket]
for dest in NOTIFICATION_DESTINATION_TYPES:
if dest in notif:
dest_dict = {
'%sConfiguration' % dest: {
'Id': uuid.uuid4(),
dest: notif[dest],
'Event': notif['Event'],
'Filter': notif['Filter']
}
}
result += xmltodict.unparse(dest_dict,
full_document=False)
result += '</NotificationConfiguration>'
response._content = result
if method == 'PUT':
LOGGER.debug('forward_request - method: "%s"' % method)
parsed = xmltodict.parse(data)
notif_config = parsed.get('NotificationConfiguration')
S3_NOTIFICATIONS.pop(bucket, None)
for dest in NOTIFICATION_DESTINATION_TYPES:
LOGGER.debug(
'forward_request - NOTIFICATION_DESTINATION_TYPES - dest: "%s"'
% dest)
config = notif_config.get('%sConfiguration' % (dest))
if config:
events = config.get('Event')
if isinstance(events, six.string_types):
events = [events]
event_filter = config.get('Filter', {})
# make sure FilterRule is an array
s3_filter = _get_s3_filter(event_filter)
if s3_filter and not isinstance(
s3_filter.get('FilterRule', []), list):
s3_filter['FilterRule'] = [s3_filter['FilterRule']]
# create final details dict
notification_details = {
'Id': config.get('Id'),
'Event': events,
dest: config.get(dest),
'Filter': event_filter
}
# TODO: what if we have multiple destinations - would we overwrite the config?
LOGGER.debug(
'forward_request - S3_NOTIFICATIONS - bucket: "%s"'
% bucket)
S3_NOTIFICATIONS[bucket] = clone(notification_details)
# return response for ?notification request
return response
if query == 'cors' or 'cors' in query_map:
if method == 'GET':
return get_cors(bucket)
if method == 'PUT':
return set_cors(bucket, data)
if method == 'DELETE':
return delete_cors(bucket)
if query == 'lifecycle' or 'lifecycle' in query_map:
if method == 'GET':
return get_lifecycle(bucket)
if method == 'PUT':
return set_lifecycle(bucket, data)
LOGGER.debug('forward_request - query_map: "%s"' % query_map)
if method == 'PUT' and 'x-amz-meta-filename' in query_map and bucket is not None and object_path is not None:
unique_id = get_unique_id(bucket, object_path)
set_user_defined_metadata(unique_id, query_map)
if modified_data:
return Request(data=modified_data, headers=headers, method=method)
return True
def test_expand_multipart_filename(self):
headers = {
"Host": "10.0.1.19:4572",
"User-Agent": "curl/7.51.0",
"Accept": "*/*",
"Content-Length": "992",
"Expect": "100-continue",
"Content-Type": "multipart/form-data; boundary=------------------------3c48c744237517ac",
}
data1 = (
b'--------------------------3c48c744237517ac\r\nContent-Disposition: form-data; name="key"\r\n\r\n'
b"uploads/20170826T181315.679087009Z/upload/${filename}\r\n--------------------------3c48c744237517ac"
b'\r\nContent-Disposition: form-data; name="AWSAccessKeyId"\r\n\r\nWHAT\r\n--------------------------'
b'3c48c744237517ac\r\nContent-Disposition: form-data; name="policy"\r\n\r\nNO\r\n--------------------'
b'------3c48c744237517ac\r\nContent-Disposition: form-data; name="signature"\r\n\r\nYUP\r\n----------'
b'----------------3c48c744237517ac\r\nContent-Disposition: form-data; name="acl"\r\n\r\nprivate\r\n--'
b'------------------------3c48c744237517ac\r\nContent-Disposition: form-data; name="success_action_re'
b'direct"\r\n\r\nhttp://127.0.0.1:5000/\r\n--------------------------3c48c744237517ac\r\nContent-Disp'
b'osition: form-data; name="file"; filename="pixel.png"\r\nContent-Type: application/octet-stream\r\n'
b"\r\n\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x06\x00\x00\x00\x1f\x15"
b"\xc4\x89\x00\x00\x00\x19tEXtSoftware\x00Adobe ImageReadyq\xc9e<\x00\x00\x00\x0eIDATx\xdabb\x00\x02"
b"\x80\x00\x03\x00\x00\x0f\x00\x03`|\xce\xe9\x00\x00\x00\x00IEND\xaeB`\x82\r\n-----------------------"
b"---3c48c744237517ac--\r\n"
)
data2 = (
b'--------------------------3c48c744237517ac\r\nContent-Disposition: form-data; name="key"\r\n\r\n'
b"uploads/20170826T181315.679087009Z/upload/pixel.png\r\n--------------------------3c48c744237517ac"
b'\r\nContent-Disposition: form-data; name="AWSAccessKeyId"\r\n\r\nWHAT\r\n--------------------------'
b'3c48c744237517ac\r\nContent-Disposition: form-data; name="policy"\r\n\r\nNO\r\n--------------------'
b'------3c48c744237517ac\r\nContent-Disposition: form-data; name="signature"\r\n\r\nYUP\r\n----------'
b'----------------3c48c744237517ac\r\nContent-Disposition: form-data; name="acl"\r\n\r\nprivate\r\n--'
b'------------------------3c48c744237517ac\r\nContent-Disposition: form-data; name="success_action_re'
b'direct"\r\n\r\nhttp://127.0.0.1:5000/\r\n--------------------------3c48c744237517ac\r\nContent-Disp'
b'osition: form-data; name="file"; filename="pixel.png"\r\nContent-Type: application/octet-stream\r\n'
b"\r\n\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x06\x00\x00\x00\x1f\x15"
b"\xc4\x89\x00\x00\x00\x19tEXtSoftware\x00Adobe ImageReadyq\xc9e<\x00\x00\x00\x0eIDATx\xdabb\x00\x02"
b"\x80\x00\x03\x00\x00\x0f\x00\x03`|\xce\xe9\x00\x00\x00\x00IEND\xaeB`\x82\r\n-----------------------"
b"---3c48c744237517ac--\r\n"
)
data3 = (
'--------------------------3c48c744237517ac\r\nContent-Disposition: form-data; name="key"\r\n\r\n'
"uploads/20170826T181315.679087009Z/upload/${filename}\r\n--------------------------3c48c744237517ac"
'\r\nContent-Disposition: form-data; name="AWSAccessKeyId"\r\n\r\nWHAT\r\n--------------------------'
'3c48c744237517ac\r\nContent-Disposition: form-data; name="policy"\r\n\r\nNO\r\n--------------------'
'------3c48c744237517ac\r\nContent-Disposition: form-data; name="signature"\r\n\r\nYUP\r\n----------'
'----------------3c48c744237517ac\r\nContent-Disposition: form-data; name="acl"\r\n\r\nprivate\r\n--'
'------------------------3c48c744237517ac\r\nContent-Disposition: form-data; name="success_action_re'
'direct"\r\n\r\nhttp://127.0.0.1:5000/\r\n--------------------------3c48c744237517ac\r\nContent-Disp'
'osition: form-data; name="file"; filename="pixel.txt"\r\nContent-Type: text/plain\r\n\r\nHello World'
"\r\n--------------------------3c48c744237517ac--\r\n"
)
expanded1 = multipart_content.expand_multipart_filename(data1, headers)
self.assertIsNot(
expanded1,
data1,
"Should have changed content of data with filename to interpolate",
)
self.assertIn(
b"uploads/20170826T181315.679087009Z/upload/pixel.png",
expanded1,
"Should see the interpolated filename",
)
expanded2 = multipart_content.expand_multipart_filename(data2, headers)
self.assertIs(
expanded2,
data2,
"Should not have changed content of data with no filename to interpolate",
)
expanded3 = multipart_content.expand_multipart_filename(data3, headers)
self.assertIsNot(
expanded3,
data3,
"Should have changed content of string data with filename to interpolate",
)
self.assertIn(
b"uploads/20170826T181315.679087009Z/upload/pixel.txt",
expanded3,
"Should see the interpolated filename",
)
def forward_request(self, method, path, data, headers):
modified_data = None
# If this request contains streaming v4 authentication signatures, strip them from the message
# Related isse: https://github.com/localstack/localstack/issues/98
# TODO we should evaluate whether to replace moto s3 with scality/S3:
# https://github.com/scality/S3/issues/237
if headers.get('x-amz-content-sha256'
) == 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD':
modified_data = strip_chunk_signatures(data)
# POST requests to S3 may include a "${filename}" placeholder in the
# key, which should be replaced with an actual file name before storing.
if method == 'POST':
original_data = modified_data or data
expanded_data = multipart_content.expand_multipart_filename(
original_data, headers)
if expanded_data is not original_data:
modified_data = expanded_data
# persist this API call to disk
persistence.record('s3', method, path, data, headers)
parsed = urlparse.urlparse(path)
query = parsed.query
path = parsed.path
bucket = path.split('/')[1]
query_map = urlparse.parse_qs(query)
if query == 'notification' or 'notification' in query_map:
response = Response()
response.status_code = 200
if method == 'GET':
# TODO check if bucket exists
result = '<NotificationConfiguration xmlns="%s">' % XMLNS_S3
if bucket in S3_NOTIFICATIONS:
notif = S3_NOTIFICATIONS[bucket]
for dest in ['Queue', 'Topic', 'CloudFunction']:
if dest in notif:
dest_dict = {
'%sConfiguration' % dest: {
'Id': uuid.uuid4(),
dest: notif[dest],
'Event': notif['Event'],
'Filter': notif['Filter']
}
}
result += xmltodict.unparse(dest_dict,
full_document=False)
result += '</NotificationConfiguration>'
response._content = result
if method == 'PUT':
parsed = xmltodict.parse(data)
notif_config = parsed.get('NotificationConfiguration')
S3_NOTIFICATIONS.pop(bucket, None)
for dest in ['Queue', 'Topic', 'CloudFunction']:
config = notif_config.get('%sConfiguration' % (dest))
if config:
events = config.get('Event')
if isinstance(events, six.string_types):
events = [events]
event_filter = config.get('Filter', {})
# make sure FilterRule is an array
s3_filter = _get_s3_filter(event_filter)
if s3_filter and not isinstance(
s3_filter.get('FilterRule', []), list):
s3_filter['FilterRule'] = [s3_filter['FilterRule']]
# create final details dict
notification_details = {
'Id': config.get('Id'),
'Event': events,
dest: config.get(dest),
'Filter': event_filter
}
# TODO: what if we have multiple destinations - would we overwrite the config?
S3_NOTIFICATIONS[bucket] = clone(notification_details)
# return response for ?notification request
return response
if query == 'cors' or 'cors' in query_map:
if method == 'GET':
return get_cors(bucket)
if method == 'PUT':
return set_cors(bucket, data)
if method == 'DELETE':
return delete_cors(bucket)
if query == 'lifecycle' or 'lifecycle' in query_map:
if method == 'GET':
return get_lifecycle(bucket)
if method == 'PUT':
return set_lifecycle(bucket, data)
if modified_data:
return Request(data=modified_data, headers=headers, method=method)
return True
def forward_request(self, method, path, data, headers):
modified_data = None
# If this request contains streaming v4 authentication signatures, strip them from the message
# Related isse: https://github.com/localstack/localstack/issues/98
# TODO we should evaluate whether to replace moto s3 with scality/S3:
# https://github.com/scality/S3/issues/237
if headers.get('x-amz-content-sha256') == 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD':
modified_data = strip_chunk_signatures(data)
# POST requests to S3 may include a "${filename}" placeholder in the
# key, which should be replaced with an actual file name before storing.
if method == 'POST':
original_data = modified_data or data
expanded_data = multipart_content.expand_multipart_filename(original_data, headers)
if expanded_data is not original_data:
modified_data = expanded_data
# If no content-type is provided, 'binary/octet-stream' should be used
# src: https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html
if method == 'PUT' and not headers.get('content-type'):
headers['content-type'] = 'binary/octet-stream'
# persist this API call to disk
persistence.record('s3', method, path, data, headers)
parsed = urlparse.urlparse(path)
query = parsed.query
path = parsed.path
bucket = path.split('/')[1]
query_map = urlparse.parse_qs(query)
if query == 'notification' or 'notification' in query_map:
response = Response()
response.status_code = 200
if method == 'GET':
# TODO check if bucket exists
result = '<NotificationConfiguration xmlns="%s">' % XMLNS_S3
if bucket in S3_NOTIFICATIONS:
notif = S3_NOTIFICATIONS[bucket]
for dest in NOTIFICATION_DESTINATION_TYPES:
if dest in notif:
dest_dict = {
'%sConfiguration' % dest: {
'Id': uuid.uuid4(),
dest: notif[dest],
'Event': notif['Event'],
'Filter': notif['Filter']
}
}
result += xmltodict.unparse(dest_dict, full_document=False)
result += '</NotificationConfiguration>'
response._content = result
if method == 'PUT':
parsed = xmltodict.parse(data)
notif_config = parsed.get('NotificationConfiguration')
S3_NOTIFICATIONS.pop(bucket, None)
for dest in NOTIFICATION_DESTINATION_TYPES:
config = notif_config.get('%sConfiguration' % (dest))
if config:
events = config.get('Event')
if isinstance(events, six.string_types):
events = [events]
event_filter = config.get('Filter', {})
# make sure FilterRule is an array
s3_filter = _get_s3_filter(event_filter)
if s3_filter and not isinstance(s3_filter.get('FilterRule', []), list):
s3_filter['FilterRule'] = [s3_filter['FilterRule']]
# create final details dict
notification_details = {
'Id': config.get('Id'),
'Event': events,
dest: config.get(dest),
'Filter': event_filter
}
# TODO: what if we have multiple destinations - would we overwrite the config?
S3_NOTIFICATIONS[bucket] = clone(notification_details)
# return response for ?notification request
return response
if query == 'cors' or 'cors' in query_map:
if method == 'GET':
return get_cors(bucket)
if method == 'PUT':
return set_cors(bucket, data)
if method == 'DELETE':
return delete_cors(bucket)
if query == 'lifecycle' or 'lifecycle' in query_map:
if method == 'GET':
return get_lifecycle(bucket)
if method == 'PUT':
return set_lifecycle(bucket, data)
if modified_data:
return Request(data=modified_data, headers=headers, method=method)
return True
