def test_find_multipart_redirect_url(self): headers = {'Host': '10.0.1.19:4572', 'User-Agent': 'curl/7.51.0', 'Accept': '*/*', 'Content-Length': '992', 'Expect': '100-continue', 'Content-Type': 'multipart/form-data; boundary=------------------------3c48c744237517ac'} data1 = (b'--------------------------3c48c744237517ac\r\nContent-Disposition: form-data; name="key"\r\n\r\n' b'uploads/20170826T181315.679087009Z/upload/pixel.png\r\n--------------------------3c48c744237517ac' b'\r\nContent-Disposition: form-data; name="success_action_redirect"\r\n\r\nhttp://127.0.0.1:5000/' b'?id=20170826T181315.679087009Z\r\n--------------------------3c48c744237517ac--\r\n') data2 = (b'--------------------------3c48c744237517ac\r\nContent-Disposition: form-data; name="key"\r\n\r\n' b'uploads/20170826T181315.679087009Z/upload/pixel.png\r\n--------------------------3c48c744237517ac' b'--\r\n') data3 = (b'--------------------------3c48c744237517ac\r\nContent-Disposition: form-data; name="success_action_' b'redirect"\r\n\r\nhttp://127.0.0.1:5000/?id=20170826T181315.679087009Z\r\n--------------------------' b'3c48c744237517ac--\r\n') key1, url1 = multipart_content.find_multipart_redirect_url(data1, headers) self.assertEqual(key1, 'uploads/20170826T181315.679087009Z/upload/pixel.png') self.assertEqual(url1, 'http://127.0.0.1:5000/?id=20170826T181315.679087009Z') key2, url2 = multipart_content.find_multipart_redirect_url(data2, headers) self.assertEqual(key2, 'uploads/20170826T181315.679087009Z/upload/pixel.png') self.assertIsNone(url2, 'Should not get a redirect URL without success_action_redirect') key3, url3 = multipart_content.find_multipart_redirect_url(data3, headers) self.assertIsNone(key3, 'Should not get a key without provided key') self.assertIsNone(url3, 'Should not get a redirect URL without provided key')
def return_response(self, method, path, data, headers, response): path = to_str(path) method = to_str(method) bucket_name = get_bucket_name(path, headers) # No path-name based bucket name? Try host-based hostname_parts = headers['host'].split('.') if (not bucket_name or len(bucket_name) == 0) and len(hostname_parts) > 1: bucket_name = hostname_parts[0] # POST requests to S3 may include a success_action_redirect field, # which should be used to redirect a client to a new location. key = None if method == 'POST': key, redirect_url = multipart_content.find_multipart_redirect_url( data, headers) if key and redirect_url: response.status_code = 303 response.headers['Location'] = expand_redirect_url( redirect_url, key, bucket_name) LOGGER.debug('S3 POST {} to {}'.format( response.status_code, response.headers['Location'])) parsed = urlparse.urlparse(path) bucket_name_in_host = headers['host'].startswith(bucket_name) should_send_notifications = all([ method in ('PUT', 'POST', 'DELETE'), '/' in path[1:] or bucket_name_in_host, # check if this is an actual put object request, because it could also be # a put bucket request with a path like this: /bucket_name/ bucket_name_in_host or (len(path[1:].split('/')) > 1 and len(path[1:].split('/')[1]) > 0), self.is_query_allowable(method, parsed.query) ]) # get subscribers and send bucket notifications if should_send_notifications: # if we already have a good key, use it, otherwise examine the path if key: object_path = '/' + key elif bucket_name_in_host: object_path = parsed.path else: parts = parsed.path[1:].split('/', 1) object_path = parts[1] if parts[1][ 0] == '/' else '/%s' % parts[1] version_id = response.headers.get('x-amz-version-id', None) send_notifications(method, bucket_name, object_path, version_id) # publish event for creation/deletion of buckets: if method in ('PUT', 'DELETE') and ('/' not in path[1:] or len(path[1:].split('/')[1]) <= 0): event_type = (event_publisher.EVENT_S3_CREATE_BUCKET if method == 'PUT' else event_publisher.EVENT_S3_DELETE_BUCKET) event_publisher.fire_event( event_type, payload={'n': event_publisher.get_hash(bucket_name)}) # fix an upstream issue in moto S3 (see https://github.com/localstack/localstack/issues/382) if method == 'PUT' and parsed.query == 'policy': response._content = '' response.status_code = 204 return response if response: reset_content_length = False # append CORS headers to response append_cors_headers(bucket_name, request_method=method, request_headers=headers, response=response) append_last_modified_headers(response=response) # Remove body from PUT response on presigned URL # https://github.com/localstack/localstack/issues/1317 if method == 'PUT' and ('X-Amz-Security-Token=' in path or 'AWSAccessKeyId=' in path): response._content = '' reset_content_length = True response_content_str = None try: response_content_str = to_str(response._content) except Exception: pass # We need to un-pretty-print the XML, otherwise we run into this issue with Spark: # https://github.com/jserver/mock-s3/pull/9/files # https://github.com/localstack/localstack/issues/183 # Note: yet, we need to make sure we have a newline after the first line: <?xml ...>\n if response_content_str and response_content_str.startswith('<'): is_bytes = isinstance(response._content, six.binary_type) append_last_modified_headers(response=response, content=response_content_str) # un-pretty-print the XML response._content = re.sub(r'([^\?])>\n\s*<', r'\1><', response_content_str, flags=re.MULTILINE) # update Location information in response payload response._content = self._update_location( response._content, bucket_name) # convert back to bytes if is_bytes: response._content = to_bytes(response._content) # fix content-type: https://github.com/localstack/localstack/issues/618 # https://github.com/localstack/localstack/issues/549 if 'text/html' in response.headers.get('Content-Type', ''): response.headers[ 'Content-Type'] = 'application/xml; charset=utf-8' reset_content_length = True # update content-length headers (fix https://github.com/localstack/localstack/issues/541) if method == 'DELETE': reset_content_length = True if reset_content_length: response.headers['content-length'] = len(response._content)
def return_response(self, method, path, data, headers, response): path = to_str(path) method = to_str(method) bucket_name = get_bucket_name(path, headers) # No path-name based bucket name? Try host-based hostname_parts = headers['host'].split('.') if (not bucket_name or len(bucket_name) == 0) and len(hostname_parts) > 1: bucket_name = hostname_parts[0] # POST requests to S3 may include a success_action_redirect field, # which should be used to redirect a client to a new location. key = None if method == 'POST': key, redirect_url = multipart_content.find_multipart_redirect_url(data, headers) if key and redirect_url: response.status_code = 303 response.headers['Location'] = expand_redirect_url(redirect_url, key, bucket_name) LOGGER.debug('S3 POST {} to {}'.format(response.status_code, response.headers['Location'])) parsed = urlparse.urlparse(path) bucket_name_in_host = headers['host'].startswith(bucket_name) should_send_notifications = all([ method in ('PUT', 'POST', 'DELETE'), '/' in path[1:] or bucket_name_in_host, # check if this is an actual put object request, because it could also be # a put bucket request with a path like this: /bucket_name/ bucket_name_in_host or (len(path[1:].split('/')) > 1 and len(path[1:].split('/')[1]) > 0), self.is_query_allowable(method, parsed.query) ]) # get subscribers and send bucket notifications if should_send_notifications: # if we already have a good key, use it, otherwise examine the path if key: object_path = '/' + key elif bucket_name_in_host: object_path = parsed.path else: parts = parsed.path[1:].split('/', 1) object_path = parts[1] if parts[1][0] == '/' else '/%s' % parts[1] version_id = response.headers.get('x-amz-version-id', None) send_notifications(method, bucket_name, object_path, version_id) # publish event for creation/deletion of buckets: if method in ('PUT', 'DELETE') and ('/' not in path[1:] or len(path[1:].split('/')[1]) <= 0): event_type = (event_publisher.EVENT_S3_CREATE_BUCKET if method == 'PUT' else event_publisher.EVENT_S3_DELETE_BUCKET) event_publisher.fire_event(event_type, payload={'n': event_publisher.get_hash(bucket_name)}) # fix an upstream issue in moto S3 (see https://github.com/localstack/localstack/issues/382) if method == 'PUT' and parsed.query == 'policy': response._content = '' response.status_code = 204 return response # emulate ErrorDocument functionality if a website is configured if method == 'GET' and response.status_code == 404 and parsed.query != 'website': s3_client = aws_stack.connect_to_service('s3') try: # Verify the bucket exists in the first place--if not, we want normal processing of the 404 s3_client.head_bucket(Bucket=bucket_name) website_config = s3_client.get_bucket_website(Bucket=bucket_name) error_doc_key = website_config.get('ErrorDocument', {}).get('Key') if error_doc_key: error_object = s3_client.get_object(Bucket=bucket_name, Key=error_doc_key) response.status_code = 200 response._content = error_object['Body'].read() response.headers['content-length'] = len(response._content) except ClientError: # Pass on the 404 as usual pass if response: reset_content_length = False # append CORS headers and other annotations to response append_cors_headers(bucket_name, request_method=method, request_headers=headers, response=response) append_last_modified_headers(response=response) append_list_objects_marker(method, path, data, response) # Remove body from PUT response on presigned URL # https://github.com/localstack/localstack/issues/1317 if method == 'PUT' and ('X-Amz-Security-Token=' in path or 'X-Amz-Credential=' in path or 'AWSAccessKeyId=' in path): response._content = '' reset_content_length = True response_content_str = None try: response_content_str = to_str(response._content) except Exception: pass # Honor response header overrides # https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectGET.html if method == 'GET': query_map = urlparse.parse_qs(parsed.query, keep_blank_values=True) for param_name, header_name in ALLOWED_HEADER_OVERRIDES.items(): if param_name in query_map: response.headers[header_name] = query_map[param_name][0] # We need to un-pretty-print the XML, otherwise we run into this issue with Spark: # https://github.com/jserver/mock-s3/pull/9/files # https://github.com/localstack/localstack/issues/183 # Note: yet, we need to make sure we have a newline after the first line: <?xml ...>\n if response_content_str and response_content_str.startswith('<'): is_bytes = isinstance(response._content, six.binary_type) append_last_modified_headers(response=response, content=response_content_str) # un-pretty-print the XML response._content = re.sub(r'([^\?])>\n\s*<', r'\1><', response_content_str, flags=re.MULTILINE) # update Location information in response payload response._content = self._update_location(response._content, bucket_name) # convert back to bytes if is_bytes: response._content = to_bytes(response._content) # fix content-type: https://github.com/localstack/localstack/issues/618 # https://github.com/localstack/localstack/issues/549 # https://github.com/localstack/localstack/issues/854 if 'text/html' in response.headers.get('Content-Type', '') \ and not response_content_str.lower().startswith('<!doctype html'): response.headers['Content-Type'] = 'application/xml; charset=utf-8' reset_content_length = True # update content-length headers (fix https://github.com/localstack/localstack/issues/541) if method == 'DELETE': reset_content_length = True if reset_content_length: response.headers['content-length'] = len(response._content)
def return_response(self, method, path, data, headers, response): parsed = urlparse.urlparse(path) # TODO: consider the case of hostname-based (as opposed to path-based) bucket addressing bucket_name = parsed.path.split('/')[1] # POST requests to S3 may include a success_action_redirect field, # which should be used to redirect a client to a new location. if method == 'POST': key, redirect_url = multipart_content.find_multipart_redirect_url( data, headers) if key and redirect_url: response.status_code = 303 response.headers['Location'] = expand_redirect_url( redirect_url, key, bucket_name) LOGGER.debug('S3 POST {} to {}'.format( response.status_code, response.headers['Location'])) # get subscribers and send bucket notifications if method in ('PUT', 'DELETE') and '/' in path[1:]: # check if this is an actual put object request, because it could also be # a put bucket request with a path like this: /bucket_name/ if len(path[1:].split('/')[1]) > 0: parts = parsed.path[1:].split('/', 1) # ignore bucket notification configuration requests if parsed.query != 'notification' and parsed.query != 'lifecycle': object_path = parts[1] if parts[1][ 0] == '/' else '/%s' % parts[1] send_notifications(method, bucket_name, object_path) # publish event for creation/deletion of buckets: if method in ('PUT', 'DELETE') and ('/' not in path[1:] or len(path[1:].split('/')[1]) <= 0): event_type = (event_publisher.EVENT_S3_CREATE_BUCKET if method == 'PUT' else event_publisher.EVENT_S3_DELETE_BUCKET) event_publisher.fire_event( event_type, payload={'n': event_publisher.get_hash(bucket_name)}) # fix an upstream issue in moto S3 (see https://github.com/localstack/localstack/issues/382) if method == 'PUT' and parsed.query == 'policy': response._content = '' response.status_code = 204 return response # append CORS headers to response if response: append_cors_headers(bucket_name, request_method=method, request_headers=headers, response=response) response_content_str = None try: response_content_str = to_str(response._content) except Exception: pass # we need to un-pretty-print the XML, otherwise we run into this issue with Spark: # https://github.com/jserver/mock-s3/pull/9/files # https://github.com/localstack/localstack/issues/183 # Note: yet, we need to make sure we have a newline after the first line: <?xml ...>\n if response_content_str and response_content_str.startswith('<'): is_bytes = isinstance(response._content, six.binary_type) response._content = re.sub(r'([^\?])>\n\s*<', r'\1><', response_content_str, flags=re.MULTILINE) if is_bytes: response._content = to_bytes(response._content) response.headers['content-length'] = len(response._content)
def return_response(self, method, path, data, headers, response): bucket_name = get_bucket_name(path, headers) # No path-name based bucket name? Try host-based hostname_parts = headers['host'].split('.') if (not bucket_name or len(bucket_name) == 0) and len(hostname_parts) > 1: bucket_name = hostname_parts[0] # POST requests to S3 may include a success_action_redirect field, # which should be used to redirect a client to a new location. key = None if method == 'POST': key, redirect_url = multipart_content.find_multipart_redirect_url(data, headers) if key and redirect_url: response.status_code = 303 response.headers['Location'] = expand_redirect_url(redirect_url, key, bucket_name) LOGGER.debug('S3 POST {} to {}'.format(response.status_code, response.headers['Location'])) parsed = urlparse.urlparse(path) bucket_name_in_host = headers['host'].startswith(bucket_name) should_send_notifications = all([ method in ('PUT', 'POST', 'DELETE'), '/' in path[1:] or bucket_name_in_host, # check if this is an actual put object request, because it could also be # a put bucket request with a path like this: /bucket_name/ bucket_name_in_host or (len(path[1:].split('/')) > 1 and len(path[1:].split('/')[1]) > 0), # don't send notification if url has a query part (some/path/with?query) # (query can be one of 'notification', 'lifecycle', 'tagging', etc) not parsed.query ]) # get subscribers and send bucket notifications if should_send_notifications: # if we already have a good key, use it, otherwise examine the path if key: object_path = '/' + key elif bucket_name_in_host: object_path = parsed.path else: parts = parsed.path[1:].split('/', 1) object_path = parts[1] if parts[1][0] == '/' else '/%s' % parts[1] send_notifications(method, bucket_name, object_path) # publish event for creation/deletion of buckets: if method in ('PUT', 'DELETE') and ('/' not in path[1:] or len(path[1:].split('/')[1]) <= 0): event_type = (event_publisher.EVENT_S3_CREATE_BUCKET if method == 'PUT' else event_publisher.EVENT_S3_DELETE_BUCKET) event_publisher.fire_event(event_type, payload={'n': event_publisher.get_hash(bucket_name)}) # fix an upstream issue in moto S3 (see https://github.com/localstack/localstack/issues/382) if method == 'PUT' and parsed.query == 'policy': response._content = '' response.status_code = 204 return response if response: # append CORS headers to response append_cors_headers(bucket_name, request_method=method, request_headers=headers, response=response) response_content_str = None try: response_content_str = to_str(response._content) except Exception: pass # we need to un-pretty-print the XML, otherwise we run into this issue with Spark: # https://github.com/jserver/mock-s3/pull/9/files # https://github.com/localstack/localstack/issues/183 # Note: yet, we need to make sure we have a newline after the first line: <?xml ...>\n if response_content_str and response_content_str.startswith('<'): is_bytes = isinstance(response._content, six.binary_type) response._content = re.sub(r'([^\?])>\n\s*<', r'\1><', response_content_str, flags=re.MULTILINE) if is_bytes: response._content = to_bytes(response._content) # fix content-type: https://github.com/localstack/localstack/issues/618 # https://github.com/localstack/localstack/issues/549 if 'text/html' in response.headers.get('Content-Type', ''): response.headers['Content-Type'] = 'application/xml; charset=utf-8' response.headers['content-length'] = len(response._content) # update content-length headers (fix https://github.com/localstack/localstack/issues/541) if method == 'DELETE': response.headers['content-length'] = len(response._content)
def return_response(self, method, path, data, headers, response): bucket_name = get_bucket_name(path, headers) # No path-name based bucket name? Try host-based hostname_parts = headers['host'].split('.') if (not bucket_name or len(bucket_name) == 0) and len(hostname_parts) > 1: bucket_name = hostname_parts[0] # POST requests to S3 may include a success_action_redirect field, # which should be used to redirect a client to a new location. key = None if method == 'POST': key, redirect_url = multipart_content.find_multipart_redirect_url( data, headers) if key and redirect_url: response.status_code = 303 response.headers['Location'] = expand_redirect_url( redirect_url, key, bucket_name) LOGGER.debug('S3 POST {} to {}'.format( response.status_code, response.headers['Location'])) parsed = urlparse.urlparse(path) bucket_name_in_host = headers['host'].startswith(bucket_name) should_send_notifications = all([ method in ('PUT', 'POST', 'DELETE'), '/' in path[1:] or bucket_name_in_host, # check if this is an actual put object request, because it could also be # a put bucket request with a path like this: /bucket_name/ bucket_name_in_host or (len(path[1:].split('/')) > 1 and len(path[1:].split('/')[1]) > 0), # ignore bucket notification configuration requests parsed.query != 'notification' and parsed.query != 'lifecycle', ]) # get subscribers and send bucket notifications if should_send_notifications: # if we already have a good key, use it, otherwise examine the path if key: object_path = '/' + key elif bucket_name_in_host: object_path = parsed.path else: parts = parsed.path[1:].split('/', 1) object_path = parts[1] if parts[1][ 0] == '/' else '/%s' % parts[1] send_notifications(method, bucket_name, object_path) # publish event for creation/deletion of buckets: if method in ('PUT', 'DELETE') and ('/' not in path[1:] or len(path[1:].split('/')[1]) <= 0): event_type = (event_publisher.EVENT_S3_CREATE_BUCKET if method == 'PUT' else event_publisher.EVENT_S3_DELETE_BUCKET) event_publisher.fire_event( event_type, payload={'n': event_publisher.get_hash(bucket_name)}) # fix an upstream issue in moto S3 (see https://github.com/localstack/localstack/issues/382) if method == 'PUT' and parsed.query == 'policy': response._content = '' response.status_code = 204 return response if response: # append CORS headers to response append_cors_headers(bucket_name, request_method=method, request_headers=headers, response=response) if response._content: # default content type; possibly overwritten with "text/xml" for API calls further below response.headers['Content-Type'] = 'binary/octet-stream' response_content_str = None try: response_content_str = to_str(response._content) except Exception: pass # we need to un-pretty-print the XML, otherwise we run into this issue with Spark: # https://github.com/jserver/mock-s3/pull/9/files # https://github.com/localstack/localstack/issues/183 # Note: yet, we need to make sure we have a newline after the first line: <?xml ...>\n if response_content_str and response_content_str.startswith('<'): is_bytes = isinstance(response._content, six.binary_type) response._content = re.sub(r'([^\?])>\n\s*<', r'\1><', response_content_str, flags=re.MULTILINE) if is_bytes: response._content = to_bytes(response._content) # fix content-type: https://github.com/localstack/localstack/issues/549 if 'text/html' in response.headers.get('Content-Type', ''): response.headers[ 'Content-Type'] = 'text/xml; charset=utf-8' # update content-length headers (fix https://github.com/localstack/localstack/issues/541) if isinstance(response._content, (six.string_types, six.binary_type)): response.headers['content-length'] = len(response._content)