def test_expand_multipart_filename(self): headers = {'Host': '10.0.1.19:4572', 'User-Agent': 'curl/7.51.0', 'Accept': '*/*', 'Content-Length': '992', 'Expect': '100-continue', 'Content-Type': 'multipart/form-data; boundary=------------------------3c48c744237517ac'} data1 = (b'--------------------------3c48c744237517ac\r\nContent-Disposition: form-data; name="key"\r\n\r\n' b'uploads/20170826T181315.679087009Z/upload/${filename}\r\n--------------------------3c48c744237517ac' b'\r\nContent-Disposition: form-data; name="AWSAccessKeyId"\r\n\r\nWHAT\r\n--------------------------' b'3c48c744237517ac\r\nContent-Disposition: form-data; name="policy"\r\n\r\nNO\r\n--------------------' b'------3c48c744237517ac\r\nContent-Disposition: form-data; name="signature"\r\n\r\nYUP\r\n----------' b'----------------3c48c744237517ac\r\nContent-Disposition: form-data; name="acl"\r\n\r\nprivate\r\n--' b'------------------------3c48c744237517ac\r\nContent-Disposition: form-data; name="success_action_re' b'direct"\r\n\r\nhttp://127.0.0.1:5000/\r\n--------------------------3c48c744237517ac\r\nContent-Disp' b'osition: form-data; name="file"; filename="pixel.png"\r\nContent-Type: application/octet-stream\r\n' b'\r\n\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x06\x00\x00\x00\x1f\x15' b'\xc4\x89\x00\x00\x00\x19tEXtSoftware\x00Adobe ImageReadyq\xc9e<\x00\x00\x00\x0eIDATx\xdabb\x00\x02' b'\x80\x00\x03\x00\x00\x0f\x00\x03`|\xce\xe9\x00\x00\x00\x00IEND\xaeB`\x82\r\n-----------------------' b'---3c48c744237517ac--\r\n') data2 = (b'--------------------------3c48c744237517ac\r\nContent-Disposition: form-data; name="key"\r\n\r\n' b'uploads/20170826T181315.679087009Z/upload/pixel.png\r\n--------------------------3c48c744237517ac' b'\r\nContent-Disposition: form-data; name="AWSAccessKeyId"\r\n\r\nWHAT\r\n--------------------------' b'3c48c744237517ac\r\nContent-Disposition: form-data; name="policy"\r\n\r\nNO\r\n--------------------' b'------3c48c744237517ac\r\nContent-Disposition: form-data; name="signature"\r\n\r\nYUP\r\n----------' b'----------------3c48c744237517ac\r\nContent-Disposition: form-data; name="acl"\r\n\r\nprivate\r\n--' b'------------------------3c48c744237517ac\r\nContent-Disposition: form-data; name="success_action_re' b'direct"\r\n\r\nhttp://127.0.0.1:5000/\r\n--------------------------3c48c744237517ac\r\nContent-Disp' b'osition: form-data; name="file"; filename="pixel.png"\r\nContent-Type: application/octet-stream\r\n' b'\r\n\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x06\x00\x00\x00\x1f\x15' b'\xc4\x89\x00\x00\x00\x19tEXtSoftware\x00Adobe ImageReadyq\xc9e<\x00\x00\x00\x0eIDATx\xdabb\x00\x02' b'\x80\x00\x03\x00\x00\x0f\x00\x03`|\xce\xe9\x00\x00\x00\x00IEND\xaeB`\x82\r\n-----------------------' b'---3c48c744237517ac--\r\n') data3 = (u'--------------------------3c48c744237517ac\r\nContent-Disposition: form-data; name="key"\r\n\r\n' u'uploads/20170826T181315.679087009Z/upload/${filename}\r\n--------------------------3c48c744237517ac' u'\r\nContent-Disposition: form-data; name="AWSAccessKeyId"\r\n\r\nWHAT\r\n--------------------------' u'3c48c744237517ac\r\nContent-Disposition: form-data; name="policy"\r\n\r\nNO\r\n--------------------' u'------3c48c744237517ac\r\nContent-Disposition: form-data; name="signature"\r\n\r\nYUP\r\n----------' u'----------------3c48c744237517ac\r\nContent-Disposition: form-data; name="acl"\r\n\r\nprivate\r\n--' u'------------------------3c48c744237517ac\r\nContent-Disposition: form-data; name="success_action_re' u'direct"\r\n\r\nhttp://127.0.0.1:5000/\r\n--------------------------3c48c744237517ac\r\nContent-Disp' u'osition: form-data; name="file"; filename="pixel.txt"\r\nContent-Type: text/plain\r\n\r\nHello World' u'\r\n--------------------------3c48c744237517ac--\r\n') expanded1 = multipart_content.expand_multipart_filename(data1, headers) self.assertIsNot(expanded1, data1, 'Should have changed content of data with filename to interpolate') self.assertIn(b'uploads/20170826T181315.679087009Z/upload/pixel.png', expanded1, 'Should see the interpolated filename') expanded2 = multipart_content.expand_multipart_filename(data2, headers) self.assertIs(expanded2, data2, 'Should not have changed content of data with no filename to interpolate') expanded3 = multipart_content.expand_multipart_filename(data3, headers) self.assertIsNot(expanded3, data3, 'Should have changed content of string data with filename to interpolate') self.assertIn(b'uploads/20170826T181315.679087009Z/upload/pixel.txt', expanded3, 'Should see the interpolated filename')
def test_expand_multipart_filename(self): headers = {'Host': '10.0.1.19:4572', 'User-Agent': 'curl/7.51.0', 'Accept': '*/*', 'Content-Length': '992', 'Expect': '100-continue', 'Content-Type': 'multipart/form-data; boundary=------------------------3c48c744237517ac'} data1 = (b'--------------------------3c48c744237517ac\r\nContent-Disposition: form-data; name="key"\r\n\r\n' b'uploads/20170826T181315.679087009Z/upload/${filename}\r\n--------------------------3c48c744237517ac' b'\r\nContent-Disposition: form-data; name="AWSAccessKeyId"\r\n\r\nWHAT\r\n--------------------------' b'3c48c744237517ac\r\nContent-Disposition: form-data; name="policy"\r\n\r\nNO\r\n--------------------' b'------3c48c744237517ac\r\nContent-Disposition: form-data; name="signature"\r\n\r\nYUP\r\n----------' b'----------------3c48c744237517ac\r\nContent-Disposition: form-data; name="acl"\r\n\r\nprivate\r\n--' b'------------------------3c48c744237517ac\r\nContent-Disposition: form-data; name="success_action_re' b'direct"\r\n\r\nhttp://127.0.0.1:5000/\r\n--------------------------3c48c744237517ac\r\nContent-Disp' b'osition: form-data; name="file"; filename="pixel.png"\r\nContent-Type: application/octet-stream\r\n' b'\r\n\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x06\x00\x00\x00\x1f\x15' b'\xc4\x89\x00\x00\x00\x19tEXtSoftware\x00Adobe ImageReadyq\xc9e<\x00\x00\x00\x0eIDATx\xdabb\x00\x02' b'\x80\x00\x03\x00\x00\x0f\x00\x03`|\xce\xe9\x00\x00\x00\x00IEND\xaeB`\x82\r\n-----------------------' b'---3c48c744237517ac--\r\n') data2 = (b'--------------------------3c48c744237517ac\r\nContent-Disposition: form-data; name="key"\r\n\r\n' b'uploads/20170826T181315.679087009Z/upload/pixel.png\r\n--------------------------3c48c744237517ac' b'\r\nContent-Disposition: form-data; name="AWSAccessKeyId"\r\n\r\nWHAT\r\n--------------------------' b'3c48c744237517ac\r\nContent-Disposition: form-data; name="policy"\r\n\r\nNO\r\n--------------------' b'------3c48c744237517ac\r\nContent-Disposition: form-data; name="signature"\r\n\r\nYUP\r\n----------' b'----------------3c48c744237517ac\r\nContent-Disposition: form-data; name="acl"\r\n\r\nprivate\r\n--' b'------------------------3c48c744237517ac\r\nContent-Disposition: form-data; name="success_action_re' b'direct"\r\n\r\nhttp://127.0.0.1:5000/\r\n--------------------------3c48c744237517ac\r\nContent-Disp' b'osition: form-data; name="file"; filename="pixel.png"\r\nContent-Type: application/octet-stream\r\n' b'\r\n\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x06\x00\x00\x00\x1f\x15' b'\xc4\x89\x00\x00\x00\x19tEXtSoftware\x00Adobe ImageReadyq\xc9e<\x00\x00\x00\x0eIDATx\xdabb\x00\x02' b'\x80\x00\x03\x00\x00\x0f\x00\x03`|\xce\xe9\x00\x00\x00\x00IEND\xaeB`\x82\r\n-----------------------' b'---3c48c744237517ac--\r\n') data3 = (u'--------------------------3c48c744237517ac\r\nContent-Disposition: form-data; name="key"\r\n\r\n' u'uploads/20170826T181315.679087009Z/upload/${filename}\r\n--------------------------3c48c744237517ac' u'\r\nContent-Disposition: form-data; name="AWSAccessKeyId"\r\n\r\nWHAT\r\n--------------------------' u'3c48c744237517ac\r\nContent-Disposition: form-data; name="policy"\r\n\r\nNO\r\n--------------------' u'------3c48c744237517ac\r\nContent-Disposition: form-data; name="signature"\r\n\r\nYUP\r\n----------' u'----------------3c48c744237517ac\r\nContent-Disposition: form-data; name="acl"\r\n\r\nprivate\r\n--' u'------------------------3c48c744237517ac\r\nContent-Disposition: form-data; name="success_action_re' u'direct"\r\n\r\nhttp://127.0.0.1:5000/\r\n--------------------------3c48c744237517ac\r\nContent-Disp' u'osition: form-data; name="file"; filename="pixel.txt"\r\nContent-Type: text/plain\r\n\r\nHello World' u'\r\n--------------------------3c48c744237517ac--\r\n') expanded1 = multipart_content.expand_multipart_filename(data1, headers) self.assertIsNot(expanded1, data1, 'Should have changed content of data with filename to interpolate') self.assertIn(b'uploads/20170826T181315.679087009Z/upload/pixel.png', expanded1, 'Should see the interpolated filename') expanded2 = multipart_content.expand_multipart_filename(data2, headers) self.assertIs(expanded2, data2, 'Should not have changed content of data with no filename to interpolate') expanded3 = multipart_content.expand_multipart_filename(data3, headers) self.assertIsNot(expanded3, data3, 'Should have changed content of string data with filename to interpolate') self.assertIn(b'uploads/20170826T181315.679087009Z/upload/pixel.txt', expanded3, 'Should see the interpolated filename')
def forward_request(self, method, path, data, headers): # parse path and query params parsed_path = urlparse.urlparse(path) # Make sure we use 'localhost' as forward host, to ensure moto uses path style addressing. # Note that all S3 clients using LocalStack need to enable path style addressing. if 's3.amazonaws.com' not in headers.get('host', ''): headers['host'] = 'localhost' # check content md5 hash integrity if not a copy request if 'Content-MD5' in headers and not self.is_s3_copy_request(headers, path): response = check_content_md5(data, headers) if response is not None: return response modified_data = None # check bucket name bucket_name = get_bucket_name(path, headers) if method == 'PUT' and not re.match(BUCKET_NAME_REGEX, bucket_name): if len(parsed_path.path) <= 1: return error_response('Unable to extract valid bucket name. Please ensure that your AWS SDK is ' + 'configured to use path style addressing, or send a valid <Bucket>.s3.amazonaws.com "Host" header', 'InvalidBucketName', status_code=400) return error_response('The specified bucket is not valid.', 'InvalidBucketName', status_code=400) # TODO: For some reason, moto doesn't allow us to put a location constraint on us-east-1 to_find = to_bytes('<LocationConstraint>us-east-1</LocationConstraint>') if data and data.startswith(to_bytes('<')) and to_find in data: modified_data = data.replace(to_find, to_bytes('')) # If this request contains streaming v4 authentication signatures, strip them from the message # Related isse: https://github.com/localstack/localstack/issues/98 # TODO we should evaluate whether to replace moto s3 with scality/S3: # https://github.com/scality/S3/issues/237 if headers.get(CONTENT_SHA256_HEADER) == STREAMING_HMAC_PAYLOAD: modified_data = strip_chunk_signatures(modified_data or data) headers['content-length'] = headers.get('x-amz-decoded-content-length') # POST requests to S3 may include a "${filename}" placeholder in the # key, which should be replaced with an actual file name before storing. if method == 'POST': original_data = modified_data or data expanded_data = multipart_content.expand_multipart_filename(original_data, headers) if expanded_data is not original_data: modified_data = expanded_data # If no content-type is provided, 'binary/octet-stream' should be used # src: https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html if method == 'PUT' and not headers.get('content-type'): headers['content-type'] = 'binary/octet-stream' # persist this API call to disk persistence.record('s3', method, path, data, headers) # parse query params query = parsed_path.query path = parsed_path.path bucket = path.split('/')[1] query_map = urlparse.parse_qs(query, keep_blank_values=True) # remap metadata query params (not supported in moto) to request headers append_metadata_headers(method, query_map, headers) # apply fixes headers_changed = fix_metadata_key_underscores(request_headers=headers) if query == 'notification' or 'notification' in query_map: # handle and return response for ?notification request response = handle_notification_request(bucket, method, data) return response if query == 'cors' or 'cors' in query_map: if method == 'GET': return get_cors(bucket) if method == 'PUT': return set_cors(bucket, data) if method == 'DELETE': return delete_cors(bucket) if query == 'lifecycle' or 'lifecycle' in query_map: if method == 'GET': return get_lifecycle(bucket) if method == 'PUT': return set_lifecycle(bucket, data) if query == 'replication' or 'replication' in query_map: if method == 'GET': return get_replication(bucket) if method == 'PUT': return set_replication(bucket, data) if query == 'encryption' or 'encryption' in query_map: if method == 'GET': return get_encryption(bucket) if method == 'PUT': return set_encryption(bucket, data) if query == 'object-lock' or 'object-lock' in query_map: if method == 'GET': return get_object_lock(bucket) if method == 'PUT': return set_object_lock(bucket, data) if modified_data is not None or headers_changed: return Request(data=modified_data or data, headers=headers, method=method) return True
def forward_request(self, method, path, data, headers): # Make sure we use 'localhost' as forward host, to ensure moto uses path style addressing. # Note that all S3 clients using LocalStack need to enable path style addressing. if 's3.amazonaws.com' not in headers.get('host', ''): headers['host'] = 'localhost' # check content md5 hash integrity if 'Content-MD5' in headers: response = check_content_md5(data, headers) if response is not None: return response modified_data = None # TODO: For some reason, moto doesn't allow us to put a location constraint on us-east-1 to_find = to_bytes('<LocationConstraint>us-east-1</LocationConstraint>') if data and data.startswith(to_bytes('<')) and to_find in data: modified_data = data.replace(to_find, '') # If this request contains streaming v4 authentication signatures, strip them from the message # Related isse: https://github.com/localstack/localstack/issues/98 # TODO we should evaluate whether to replace moto s3 with scality/S3: # https://github.com/scality/S3/issues/237 if headers.get('x-amz-content-sha256') == 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD': modified_data = strip_chunk_signatures(modified_data or data) headers['content-length'] = headers.get('x-amz-decoded-content-length') # POST requests to S3 may include a "${filename}" placeholder in the # key, which should be replaced with an actual file name before storing. if method == 'POST': original_data = modified_data or data expanded_data = multipart_content.expand_multipart_filename(original_data, headers) if expanded_data is not original_data: modified_data = expanded_data # If no content-type is provided, 'binary/octet-stream' should be used # src: https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html if method == 'PUT' and not headers.get('content-type'): headers['content-type'] = 'binary/octet-stream' # persist this API call to disk persistence.record('s3', method, path, data, headers) # parse query params parsed = urlparse.urlparse(path) query = parsed.query path = parsed.path bucket = path.split('/')[1] query_map = urlparse.parse_qs(query, keep_blank_values=True) if query == 'notification' or 'notification' in query_map: # handle and return response for ?notification request response = handle_notification_request(bucket, method, data) return response if query == 'cors' or 'cors' in query_map: if method == 'GET': return get_cors(bucket) if method == 'PUT': return set_cors(bucket, data) if method == 'DELETE': return delete_cors(bucket) if query == 'lifecycle' or 'lifecycle' in query_map: if method == 'GET': return get_lifecycle(bucket) if method == 'PUT': return set_lifecycle(bucket, data) if modified_data is not None: return Request(data=modified_data, headers=headers, method=method) return True
def forward_request(self, method, path, data, headers): LOGGER.debug('forward_request - method: "%s"' % method) modified_data = None # If this request contains streaming v4 authentication signatures, strip them from the message # Related isse: https://github.com/localstack/localstack/issues/98 # TODO we should evaluate whether to replace moto s3 with scality/S3: # https://github.com/scality/S3/issues/237 if headers.get('x-amz-content-sha256' ) == 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD': modified_data = strip_chunk_signatures(data) # POST requests to S3 may include a "${filename}" placeholder in the # key, which should be replaced with an actual file name before storing. if method == 'POST': original_data = modified_data or data expanded_data = multipart_content.expand_multipart_filename( original_data, headers) if expanded_data is not original_data: modified_data = expanded_data # If no content-type is provided, 'binary/octet-stream' should be used # src: https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html if method == 'PUT' and not headers.get('content-type'): headers['content-type'] = 'binary/octet-stream' # persist this API call to disk persistence.record('s3', method, path, data, headers) parsed = urlparse.urlparse(path) query = parsed.query path = parsed.path object_path = parsed.path LOGGER.debug('forward_request - query: "%s"' % query) LOGGER.debug('forward_request - path: "%s"' % path) LOGGER.debug('forward_request - host: "%s"' % headers['host']) LOGGER.debug('forward_request - object_path: "%s"' % object_path) # Check bucket name in host bucket = None hostname_parts = headers['host'].split('.') if len(hostname_parts) > 1: bucket = hostname_parts[0] # No bucket name in host, check in path. if (not bucket or len(bucket) == 0): bucket = get_bucket_name(path, headers) LOGGER.debug('forward_request - Bucket name: "%s"' % bucket) query_map = urlparse.parse_qs(query) if query == 'notification' or 'notification' in query_map: LOGGER.debug('forward_request - query: "%s"' % query) response = Response() response.status_code = 200 if method == 'GET': # TODO check if bucket exists result = '<NotificationConfiguration xmlns="%s">' % XMLNS_S3 if bucket in S3_NOTIFICATIONS: notif = S3_NOTIFICATIONS[bucket] for dest in NOTIFICATION_DESTINATION_TYPES: if dest in notif: dest_dict = { '%sConfiguration' % dest: { 'Id': uuid.uuid4(), dest: notif[dest], 'Event': notif['Event'], 'Filter': notif['Filter'] } } result += xmltodict.unparse(dest_dict, full_document=False) result += '</NotificationConfiguration>' response._content = result if method == 'PUT': LOGGER.debug('forward_request - method: "%s"' % method) parsed = xmltodict.parse(data) notif_config = parsed.get('NotificationConfiguration') S3_NOTIFICATIONS.pop(bucket, None) for dest in NOTIFICATION_DESTINATION_TYPES: LOGGER.debug( 'forward_request - NOTIFICATION_DESTINATION_TYPES - dest: "%s"' % dest) config = notif_config.get('%sConfiguration' % (dest)) if config: events = config.get('Event') if isinstance(events, six.string_types): events = [events] event_filter = config.get('Filter', {}) # make sure FilterRule is an array s3_filter = _get_s3_filter(event_filter) if s3_filter and not isinstance( s3_filter.get('FilterRule', []), list): s3_filter['FilterRule'] = [s3_filter['FilterRule']] # create final details dict notification_details = { 'Id': config.get('Id'), 'Event': events, dest: config.get(dest), 'Filter': event_filter } # TODO: what if we have multiple destinations - would we overwrite the config? LOGGER.debug( 'forward_request - S3_NOTIFICATIONS - bucket: "%s"' % bucket) S3_NOTIFICATIONS[bucket] = clone(notification_details) # return response for ?notification request return response if query == 'cors' or 'cors' in query_map: if method == 'GET': return get_cors(bucket) if method == 'PUT': return set_cors(bucket, data) if method == 'DELETE': return delete_cors(bucket) if query == 'lifecycle' or 'lifecycle' in query_map: if method == 'GET': return get_lifecycle(bucket) if method == 'PUT': return set_lifecycle(bucket, data) LOGGER.debug('forward_request - query_map: "%s"' % query_map) if method == 'PUT' and 'x-amz-meta-filename' in query_map and bucket is not None and object_path is not None: unique_id = get_unique_id(bucket, object_path) set_user_defined_metadata(unique_id, query_map) if modified_data: return Request(data=modified_data, headers=headers, method=method) return True
def test_expand_multipart_filename(self): headers = { "Host": "10.0.1.19:4572", "User-Agent": "curl/7.51.0", "Accept": "*/*", "Content-Length": "992", "Expect": "100-continue", "Content-Type": "multipart/form-data; boundary=------------------------3c48c744237517ac", } data1 = ( b'--------------------------3c48c744237517ac\r\nContent-Disposition: form-data; name="key"\r\n\r\n' b"uploads/20170826T181315.679087009Z/upload/${filename}\r\n--------------------------3c48c744237517ac" b'\r\nContent-Disposition: form-data; name="AWSAccessKeyId"\r\n\r\nWHAT\r\n--------------------------' b'3c48c744237517ac\r\nContent-Disposition: form-data; name="policy"\r\n\r\nNO\r\n--------------------' b'------3c48c744237517ac\r\nContent-Disposition: form-data; name="signature"\r\n\r\nYUP\r\n----------' b'----------------3c48c744237517ac\r\nContent-Disposition: form-data; name="acl"\r\n\r\nprivate\r\n--' b'------------------------3c48c744237517ac\r\nContent-Disposition: form-data; name="success_action_re' b'direct"\r\n\r\nhttp://127.0.0.1:5000/\r\n--------------------------3c48c744237517ac\r\nContent-Disp' b'osition: form-data; name="file"; filename="pixel.png"\r\nContent-Type: application/octet-stream\r\n' b"\r\n\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x06\x00\x00\x00\x1f\x15" b"\xc4\x89\x00\x00\x00\x19tEXtSoftware\x00Adobe ImageReadyq\xc9e<\x00\x00\x00\x0eIDATx\xdabb\x00\x02" b"\x80\x00\x03\x00\x00\x0f\x00\x03`|\xce\xe9\x00\x00\x00\x00IEND\xaeB`\x82\r\n-----------------------" b"---3c48c744237517ac--\r\n" ) data2 = ( b'--------------------------3c48c744237517ac\r\nContent-Disposition: form-data; name="key"\r\n\r\n' b"uploads/20170826T181315.679087009Z/upload/pixel.png\r\n--------------------------3c48c744237517ac" b'\r\nContent-Disposition: form-data; name="AWSAccessKeyId"\r\n\r\nWHAT\r\n--------------------------' b'3c48c744237517ac\r\nContent-Disposition: form-data; name="policy"\r\n\r\nNO\r\n--------------------' b'------3c48c744237517ac\r\nContent-Disposition: form-data; name="signature"\r\n\r\nYUP\r\n----------' b'----------------3c48c744237517ac\r\nContent-Disposition: form-data; name="acl"\r\n\r\nprivate\r\n--' b'------------------------3c48c744237517ac\r\nContent-Disposition: form-data; name="success_action_re' b'direct"\r\n\r\nhttp://127.0.0.1:5000/\r\n--------------------------3c48c744237517ac\r\nContent-Disp' b'osition: form-data; name="file"; filename="pixel.png"\r\nContent-Type: application/octet-stream\r\n' b"\r\n\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x06\x00\x00\x00\x1f\x15" b"\xc4\x89\x00\x00\x00\x19tEXtSoftware\x00Adobe ImageReadyq\xc9e<\x00\x00\x00\x0eIDATx\xdabb\x00\x02" b"\x80\x00\x03\x00\x00\x0f\x00\x03`|\xce\xe9\x00\x00\x00\x00IEND\xaeB`\x82\r\n-----------------------" b"---3c48c744237517ac--\r\n" ) data3 = ( '--------------------------3c48c744237517ac\r\nContent-Disposition: form-data; name="key"\r\n\r\n' "uploads/20170826T181315.679087009Z/upload/${filename}\r\n--------------------------3c48c744237517ac" '\r\nContent-Disposition: form-data; name="AWSAccessKeyId"\r\n\r\nWHAT\r\n--------------------------' '3c48c744237517ac\r\nContent-Disposition: form-data; name="policy"\r\n\r\nNO\r\n--------------------' '------3c48c744237517ac\r\nContent-Disposition: form-data; name="signature"\r\n\r\nYUP\r\n----------' '----------------3c48c744237517ac\r\nContent-Disposition: form-data; name="acl"\r\n\r\nprivate\r\n--' '------------------------3c48c744237517ac\r\nContent-Disposition: form-data; name="success_action_re' 'direct"\r\n\r\nhttp://127.0.0.1:5000/\r\n--------------------------3c48c744237517ac\r\nContent-Disp' 'osition: form-data; name="file"; filename="pixel.txt"\r\nContent-Type: text/plain\r\n\r\nHello World' "\r\n--------------------------3c48c744237517ac--\r\n" ) expanded1 = multipart_content.expand_multipart_filename(data1, headers) self.assertIsNot( expanded1, data1, "Should have changed content of data with filename to interpolate", ) self.assertIn( b"uploads/20170826T181315.679087009Z/upload/pixel.png", expanded1, "Should see the interpolated filename", ) expanded2 = multipart_content.expand_multipart_filename(data2, headers) self.assertIs( expanded2, data2, "Should not have changed content of data with no filename to interpolate", ) expanded3 = multipart_content.expand_multipart_filename(data3, headers) self.assertIsNot( expanded3, data3, "Should have changed content of string data with filename to interpolate", ) self.assertIn( b"uploads/20170826T181315.679087009Z/upload/pixel.txt", expanded3, "Should see the interpolated filename", )
def forward_request(self, method, path, data, headers): modified_data = None # If this request contains streaming v4 authentication signatures, strip them from the message # Related isse: https://github.com/localstack/localstack/issues/98 # TODO we should evaluate whether to replace moto s3 with scality/S3: # https://github.com/scality/S3/issues/237 if headers.get('x-amz-content-sha256' ) == 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD': modified_data = strip_chunk_signatures(data) # POST requests to S3 may include a "${filename}" placeholder in the # key, which should be replaced with an actual file name before storing. if method == 'POST': original_data = modified_data or data expanded_data = multipart_content.expand_multipart_filename( original_data, headers) if expanded_data is not original_data: modified_data = expanded_data # persist this API call to disk persistence.record('s3', method, path, data, headers) parsed = urlparse.urlparse(path) query = parsed.query path = parsed.path bucket = path.split('/')[1] query_map = urlparse.parse_qs(query) if query == 'notification' or 'notification' in query_map: response = Response() response.status_code = 200 if method == 'GET': # TODO check if bucket exists result = '<NotificationConfiguration xmlns="%s">' % XMLNS_S3 if bucket in S3_NOTIFICATIONS: notif = S3_NOTIFICATIONS[bucket] for dest in ['Queue', 'Topic', 'CloudFunction']: if dest in notif: dest_dict = { '%sConfiguration' % dest: { 'Id': uuid.uuid4(), dest: notif[dest], 'Event': notif['Event'], 'Filter': notif['Filter'] } } result += xmltodict.unparse(dest_dict, full_document=False) result += '</NotificationConfiguration>' response._content = result if method == 'PUT': parsed = xmltodict.parse(data) notif_config = parsed.get('NotificationConfiguration') S3_NOTIFICATIONS.pop(bucket, None) for dest in ['Queue', 'Topic', 'CloudFunction']: config = notif_config.get('%sConfiguration' % (dest)) if config: events = config.get('Event') if isinstance(events, six.string_types): events = [events] event_filter = config.get('Filter', {}) # make sure FilterRule is an array s3_filter = _get_s3_filter(event_filter) if s3_filter and not isinstance( s3_filter.get('FilterRule', []), list): s3_filter['FilterRule'] = [s3_filter['FilterRule']] # create final details dict notification_details = { 'Id': config.get('Id'), 'Event': events, dest: config.get(dest), 'Filter': event_filter } # TODO: what if we have multiple destinations - would we overwrite the config? S3_NOTIFICATIONS[bucket] = clone(notification_details) # return response for ?notification request return response if query == 'cors' or 'cors' in query_map: if method == 'GET': return get_cors(bucket) if method == 'PUT': return set_cors(bucket, data) if method == 'DELETE': return delete_cors(bucket) if query == 'lifecycle' or 'lifecycle' in query_map: if method == 'GET': return get_lifecycle(bucket) if method == 'PUT': return set_lifecycle(bucket, data) if modified_data: return Request(data=modified_data, headers=headers, method=method) return True
def forward_request(self, method, path, data, headers): modified_data = None # If this request contains streaming v4 authentication signatures, strip them from the message # Related isse: https://github.com/localstack/localstack/issues/98 # TODO we should evaluate whether to replace moto s3 with scality/S3: # https://github.com/scality/S3/issues/237 if headers.get('x-amz-content-sha256') == 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD': modified_data = strip_chunk_signatures(data) # POST requests to S3 may include a "${filename}" placeholder in the # key, which should be replaced with an actual file name before storing. if method == 'POST': original_data = modified_data or data expanded_data = multipart_content.expand_multipart_filename(original_data, headers) if expanded_data is not original_data: modified_data = expanded_data # If no content-type is provided, 'binary/octet-stream' should be used # src: https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html if method == 'PUT' and not headers.get('content-type'): headers['content-type'] = 'binary/octet-stream' # persist this API call to disk persistence.record('s3', method, path, data, headers) parsed = urlparse.urlparse(path) query = parsed.query path = parsed.path bucket = path.split('/')[1] query_map = urlparse.parse_qs(query) if query == 'notification' or 'notification' in query_map: response = Response() response.status_code = 200 if method == 'GET': # TODO check if bucket exists result = '<NotificationConfiguration xmlns="%s">' % XMLNS_S3 if bucket in S3_NOTIFICATIONS: notif = S3_NOTIFICATIONS[bucket] for dest in NOTIFICATION_DESTINATION_TYPES: if dest in notif: dest_dict = { '%sConfiguration' % dest: { 'Id': uuid.uuid4(), dest: notif[dest], 'Event': notif['Event'], 'Filter': notif['Filter'] } } result += xmltodict.unparse(dest_dict, full_document=False) result += '</NotificationConfiguration>' response._content = result if method == 'PUT': parsed = xmltodict.parse(data) notif_config = parsed.get('NotificationConfiguration') S3_NOTIFICATIONS.pop(bucket, None) for dest in NOTIFICATION_DESTINATION_TYPES: config = notif_config.get('%sConfiguration' % (dest)) if config: events = config.get('Event') if isinstance(events, six.string_types): events = [events] event_filter = config.get('Filter', {}) # make sure FilterRule is an array s3_filter = _get_s3_filter(event_filter) if s3_filter and not isinstance(s3_filter.get('FilterRule', []), list): s3_filter['FilterRule'] = [s3_filter['FilterRule']] # create final details dict notification_details = { 'Id': config.get('Id'), 'Event': events, dest: config.get(dest), 'Filter': event_filter } # TODO: what if we have multiple destinations - would we overwrite the config? S3_NOTIFICATIONS[bucket] = clone(notification_details) # return response for ?notification request return response if query == 'cors' or 'cors' in query_map: if method == 'GET': return get_cors(bucket) if method == 'PUT': return set_cors(bucket, data) if method == 'DELETE': return delete_cors(bucket) if query == 'lifecycle' or 'lifecycle' in query_map: if method == 'GET': return get_lifecycle(bucket) if method == 'PUT': return set_lifecycle(bucket, data) if modified_data: return Request(data=modified_data, headers=headers, method=method) return True