class AutoContainerBase(object):
BYPASS_QS = "bypass-autocontainer"
BYPASS_HEADER = "X-bypass-autocontainer"
def __init__(self, app, acct,
strip_v1=False, account_first=False, swift3_compat=False,
stop_at_first_match=True, skip_metadata=False):
self.app = app
self.account = acct
self.bypass_header_key = ("HTTP_" +
self.BYPASS_HEADER.upper().replace('-', '_'))
self.con_builder = ContainerBuilder()
self.account_first = account_first
self.swift3_compat = swift3_compat
self.strip_v1 = strip_v1
self.skip_metadata = skip_metadata
if (not stop_at_first_match and
not hasattr(self.con_builder, 'alternatives')):
raise ValueError("Disabling 'stop_at_first_match' parameter "
"is not supported with openio-sds < 4.2")
self.stop_at_first_match = stop_at_first_match
def should_bypass(self, env):
"""Should we bypass this filter?"""
header = env.get(self.bypass_header_key, "").lower()
query = parse_qs(env.get('QUERY_STRING', "")).get(self.BYPASS_QS, [""])
return config_true_value(header) or config_true_value(query[0])
def _extract_path(self, path):
account = self.account
# Remove leading '/' to be consistent with split_path()
obj = path[1:]
container = None
try:
if self.strip_v1:
version, tail = split_path('/' + obj, 1, 2, True)
if version in ('v1', 'v1.0'):
obj = tail
if obj is not None and self.account_first:
account, tail = split_path('/' + obj, 1, 2, True)
obj = tail
if obj is not None and self.swift3_compat:
container, tail = split_path('/' + obj, 1, 2, True)
obj = tail
# Do not yield an empty object name
if not obj:
obj = None
except ValueError:
raise HTTPBadRequest()
return account, container, obj
def _save_bucket_name(self, env):
req = Request(env)
account, container, obj = self._extract_path(req.path_info)
sys_meta_key = '%soio-bucket-name' % get_sys_meta_prefix('object')
req.headers[sys_meta_key] = container
def _convert_path(self, path):
account, container, obj = self._extract_path(path)
if obj is not None:
# TODO(FVE): when supported, also pass account to con_builder()
container = quote_plus(self.con_builder(obj))
return account, container, obj
def _alternatives(self, account, container, obj):
# put S3 parts in dedicated container
suffix = ("+segments" if container and container.endswith('+segments')
else "")
if obj is None:
yield account, container, obj
elif self.stop_at_first_match:
# TODO(FVE): when supported, also pass account to con_builder()
yield account, quote_plus(self.con_builder(obj)) + suffix, obj
else:
for alt_container in self.con_builder.alternatives(obj):
yield account, quote_plus(alt_container) + suffix, obj
raise StopIteration
@staticmethod
def is_copy(env):
"""Tell if `env` represents an object copy operation."""
return (env['REQUEST_METHOD'] == 'PUT' and
('HTTP_X_COPY_FROM' in env or 'HTTP_OIO_COPY_FROM' in env))
@staticmethod
def _save_response(env, status, headers, exc_info=None):
env['last_status'] = status
env['last_headers'] = headers
env['last_exc_info'] = exc_info
if 'first_status' not in env:
env['first_status'] = status
env['first_headers'] = headers
env['first_exc_info'] = exc_info
def _retry_loop(self, orig_env, start_response, path_to_modify,
env_modifier, alt_checker=None):
"""
:param env_modifier: function copies and modifies the env
dictionary, according to the alternative path parts
:param alt_checker: function that checks the alternative
path parts (may raise an exception)
"""
local_env = {}
query = parse_qs(orig_env.get('QUERY_STRING', ''), True)
account, container, obj = self._extract_path(path_to_modify)
is_container_req = container is not None and obj is None
is_service_req = (account is not None and
container is None and
obj is None)
if is_container_req and 'prefix' in query:
obj = query['prefix'][0]
elif is_service_req:
env = env_modifier(orig_env, (account, ))
return self.app(env, start_response)
for alt in self._alternatives(account, container, obj):
if alt_checker and not alt_checker(alt):
return self.app(orig_env, start_response)
env = env_modifier(orig_env, alt[:2] if is_container_req else alt)
resp = self.app(env, partial(self._save_response, local_env))
if isinstance(resp, collections.Iterable):
# TODO when a MPU is completed, resp is an iterator
# and we must fill last_status but it may break
# next test
resp = reiterate(resp)
if 'last_status' not in local_env:
# start_response() was not called. This happens when there is
# no 'proxy-logging' just after 'catch_errors' in the pipeline.
return resp
if not local_env['last_status'].startswith('404'):
# start_response() was called, and status is not 404.
start_response(local_env['last_status'],
local_env['last_headers'],
local_env['last_exc_info'])
return resp
if 'first_resp' not in local_env:
local_env['first_resp'] = resp
start_response(local_env['first_status'],
local_env['first_headers'],
local_env['first_exc_info'])
return local_env['first_resp']
def _call_copy(self, env, start_response):
"""
Run the retry loop (copy operations).
"""
account, container, obj = self._convert_path(env.get('PATH_INFO'))
if obj is None:
# This is probably an account request
return self.app(env, start_response)
env['PATH_INFO'] = "/v1/%s/%s/%s" % (account, container, obj)
for hdr in ("HTTP_OIO_COPY_FROM", "HTTP_X_COPY_FROM"):
if hdr in env:
from_value = env.get(hdr)
from_header = hdr
break
else:
raise HTTPBadRequest(body="Malformed copy-source header")
# HTTP_X_COPY_FROM_ACCOUNT will just pass through
if self.account_first:
src_path = "/fake_account" + from_value
else:
src_path = from_value
def modify_copy_from(orig_env, alternative):
env_ = orig_env.copy()
env_[from_header] = "/%s/%s" % (
quote_plus(alternative[1]), alternative[2])
return env_
def check_container_obj(alternative):
if not alternative[1] or not alternative[2]:
raise HTTPBadRequest(body="Malformed copy-source header")
return True
return self._retry_loop(
env, start_response, src_path,
env_modifier=modify_copy_from,
alt_checker=check_container_obj)
def _mock_infocache(self, env):
if not self.skip_metadata:
return
req = Request(env)
# don't fake obj metadata
account, container, _ = self._extract_path(req.path_info)
req.environ.setdefault('swift.infocache', {})
req.environ['swift.infocache'][get_cache_key(account)] = \
headers_to_account_info({}, 0)
if container:
key = get_cache_key(account, container)
req.environ['swift.infocache'][key] = \
headers_to_container_info({}, 0)
def _call(self, env, start_response):
"""
Run the retry loop (regular operations).
"""
def modify_path_info(orig_env, alternative):
env_ = orig_env.copy()
env_['PATH_INFO'] = '/'.join(('', 'v1') + alternative)
return env_
def check_obj(alternative):
return alternative[2] is not None
return self._retry_loop(
env, start_response, env['PATH_INFO'],
env_modifier=modify_path_info,
alt_checker=check_obj)
def __call__(self, env, start_response):
self._save_bucket_name(env)
self._mock_infocache(env)
if self.should_bypass(env):
return self.app(env, start_response)
if self.is_copy(env):
return self._call_copy(env, start_response)
else:
return self._call(env, start_response)
class AutoContainerBase(object):
BYPASS_QS = "bypass-autocontainer"
BYPASS_HEADER = "X-bypass-autocontainer"
def __init__(self,
app,
acct,
strip_v1=False,
account_first=False,
swift3_compat=False,
stop_at_first_match=True,
skip_metadata=False,
**_kwargs):
self.app = app
self.account = acct
self.bypass_header_key = ("HTTP_" +
self.BYPASS_HEADER.upper().replace('-', '_'))
self.con_builder = ContainerBuilder()
self.account_first = account_first
self.swift3_compat = swift3_compat
self.strip_v1 = strip_v1
self.skip_metadata = skip_metadata
if (not stop_at_first_match
and not hasattr(self.con_builder, 'alternatives')):
raise ValueError("Disabling 'stop_at_first_match' parameter "
"is not supported with openio-sds < 4.2")
self.stop_at_first_match = stop_at_first_match
def should_bypass(self, env):
"""Should we bypass this filter?"""
header = env.get(self.bypass_header_key, "").lower()
query = parse_qs(env.get('QUERY_STRING', "")).get(self.BYPASS_QS, [""])
return config_true_value(header) or config_true_value(query[0])
def _extract_path(self, path):
account = self.account
# Remove leading '/' to be consistent with split_path()
obj = path[1:]
container = None
try:
if self.strip_v1:
version, tail = split_path('/' + obj, 1, 2, True)
if version in ('v1', 'v1.0'):
obj = tail
if obj is not None and self.account_first:
account, tail = split_path('/' + obj, 1, 2, True)
obj = tail
if obj is not None and self.swift3_compat:
container, tail = split_path('/' + obj, 1, 2, True)
obj = tail
# Do not yield an empty object name
if not obj:
obj = None
except ValueError:
raise HTTPBadRequest()
return account, container, obj
def _save_bucket_name(self, env):
req = Request(env)
account, container, obj = self._extract_path(req.path_info)
sys_meta_key = '%soio-bucket-name' % get_sys_meta_prefix('object')
req.headers[sys_meta_key] = container
def _convert_path(self, path):
account, container, obj = self._extract_path(path)
if obj is not None:
# TODO(FVE): when supported, also pass account to con_builder()
container = quote_plus(self.con_builder(obj))
return account, container, obj
def _alternatives(self, account, container, obj):
# put S3 parts in dedicated container
suffix = ("+segments"
if container and container.endswith('+segments') else "")
if obj is None:
yield account, container, obj
elif self.stop_at_first_match:
# TODO(FVE): when supported, also pass account to con_builder()
yield account, quote_plus(self.con_builder(obj)) + suffix, obj
else:
for alt_container in self.con_builder.alternatives(obj):
yield account, quote_plus(alt_container) + suffix, obj
raise StopIteration
@staticmethod
def is_copy(env):
"""Tell if `env` represents an object copy operation."""
return (env['REQUEST_METHOD'] == 'PUT'
and ('HTTP_X_COPY_FROM' in env or 'HTTP_OIO_COPY_FROM' in env))
@staticmethod
def _save_response(env, status, headers, exc_info=None):
env['last_status'] = status
env['last_headers'] = headers
env['last_exc_info'] = exc_info
if 'first_status' not in env:
env['first_status'] = status
env['first_headers'] = headers
env['first_exc_info'] = exc_info
def _retry_loop(self,
orig_env,
start_response,
path_to_modify,
env_modifier,
alt_checker=None):
"""
:param env_modifier: function copies and modifies the env
dictionary, according to the alternative path parts
:param alt_checker: function that checks the alternative
path parts (may raise an exception)
"""
local_env = {}
query = parse_qs(orig_env.get('QUERY_STRING', ''), True)
account, container, obj = self._extract_path(path_to_modify)
is_container_req = container is not None and obj is None
is_service_req = (account is not None and container is None
and obj is None)
if is_container_req and 'prefix' in query:
obj = query['prefix'][0]
elif is_service_req:
env = env_modifier(orig_env, (account, ))
return self.app(env, start_response)
for alt in self._alternatives(account, container, obj):
if alt_checker and not alt_checker(alt):
return self.app(orig_env, start_response)
env = env_modifier(orig_env, alt[:2] if is_container_req else alt)
resp = self.app(env, partial(self._save_response, local_env))
if isinstance(resp, collections.Iterable):
# TODO when a MPU is completed, resp is an iterator
# and we must fill last_status but it may break
# next test
resp = reiterate(resp)
if 'last_status' not in local_env:
# start_response() was not called. This happens when there is
# no 'proxy-logging' just after 'catch_errors' in the pipeline.
return resp
if not local_env['last_status'].startswith('404'):
# start_response() was called, and status is not 404.
start_response(local_env['last_status'],
local_env['last_headers'],
local_env['last_exc_info'])
return resp
if 'first_resp' not in local_env:
local_env['first_resp'] = resp
start_response(local_env['first_status'], local_env['first_headers'],
local_env['first_exc_info'])
return local_env['first_resp']
def _call_copy(self, env, start_response):
"""
Run the retry loop (copy operations).
"""
account, container, obj = self._convert_path(env.get('PATH_INFO'))
if obj is None:
# This is probably an account request
return self.app(env, start_response)
env['PATH_INFO'] = "/v1/%s/%s/%s" % (account, container, obj)
for hdr in ("HTTP_OIO_COPY_FROM", "HTTP_X_COPY_FROM"):
if hdr in env:
from_value = env.get(hdr)
from_header = hdr
break
else:
raise HTTPBadRequest(body="Malformed copy-source header")
# HTTP_X_COPY_FROM_ACCOUNT will just pass through
if self.account_first:
src_path = "/fake_account" + from_value
else:
src_path = from_value
def modify_copy_from(orig_env, alternative):
env_ = orig_env.copy()
env_[from_header] = "/%s/%s" % (quote_plus(
alternative[1]), alternative[2])
return env_
def check_container_obj(alternative):
if not alternative[1] or not alternative[2]:
raise HTTPBadRequest(body="Malformed copy-source header")
return True
return self._retry_loop(env,
start_response,
src_path,
env_modifier=modify_copy_from,
alt_checker=check_container_obj)
def _mock_infocache(self, env):
if not self.skip_metadata:
return
req = Request(env)
# don't fake obj metadata
account, container, _ = self._extract_path(req.path_info)
req.environ.setdefault('swift.infocache', {})
req.environ['swift.infocache'][get_cache_key(account)] = \
headers_to_account_info({}, 0)
if container:
key = get_cache_key(account, container)
req.environ['swift.infocache'][key] = \
headers_to_container_info({}, 0)
def _call(self, env, start_response):
"""
Run the retry loop (regular operations).
"""
def modify_path_info(orig_env, alternative):
env_ = orig_env.copy()
env_['PATH_INFO'] = '/'.join(('', 'v1') + alternative)
return env_
def check_obj(alternative):
return alternative[2] is not None
return self._retry_loop(env,
start_response,
env['PATH_INFO'],
env_modifier=modify_path_info,
alt_checker=check_obj)
def __call__(self, env, start_response):
self._save_bucket_name(env)
self._mock_infocache(env)
if self.should_bypass(env):
return self.app(env, start_response)
if self.is_copy(env):
return self._call_copy(env, start_response)
else:
return self._call(env, start_response)
class AutoContainerBase(object):
BYPASS_QS = "bypass-autocontainer"
BYPASS_HEADER = "X-bypass-autocontainer"
def __init__(self,
app,
acct,
strip_v1=False,
account_first=False,
swift3_compat=False,
stop_at_first_match=True):
self.app = app
self.account = acct
self.bypass_header_key = ("HTTP_" +
self.BYPASS_HEADER.upper().replace('-', '_'))
self.con_builder = ContainerBuilder()
self.account_first = account_first
self.swift3_compat = swift3_compat
self.strip_v1 = strip_v1
if (not stop_at_first_match
and not hasattr(self.con_builder, 'alternatives')):
raise ValueError("Disabling 'stop_at_first_match' parameter "
"is not supported with openio-sds < 4.2")
self.stop_at_first_match = stop_at_first_match
def should_bypass(self, env):
"""Should we bypass this filter?"""
header = env.get(self.bypass_header_key, "").lower()
query = parse_qs(env.get('QUERY_STRING', "")).get(self.BYPASS_QS, [""])
return config_true_value(header) or config_true_value(query[0])
def _extract_path(self, path):
account = self.account
# Remove leading '/' to be consistent with split_path()
obj = path[1:]
container = None
if self.strip_v1:
version, tail = split_path('/' + obj, 1, 2, True)
if version in ('v1', 'v1.0'):
obj = tail
if self.account_first:
account, tail = split_path('/' + obj, 1, 2, True)
obj = tail
if obj is not None and self.swift3_compat:
container, tail = split_path('/' + obj, 1, 2, True)
obj = tail
return account, container, obj
def _convert_path(self, path):
account, container, obj = self._extract_path(path)
if obj is not None:
container = quote_plus(self.con_builder(obj))
return account, container, obj
def _alternatives(self, path):
account, container, obj = self._extract_path(path)
if obj is None:
yield account, container, obj
elif self.stop_at_first_match:
yield account, quote_plus(self.con_builder(obj)), obj
else:
for alt_container in self.con_builder.alternatives(obj):
yield account, quote_plus(alt_container), obj
raise StopIteration
@staticmethod
def is_copy(env):
"""Tell if `env` represents an object copy operation."""
return env['REQUEST_METHOD'] == 'PUT' and 'HTTP_X_COPY_FROM' in env
@staticmethod
def _save_response(env, status, headers, exc_info=None):
env['last_status'] = status
env['last_headers'] = headers
env['last_exc_info'] = exc_info
if 'first_status' not in env:
env['first_status'] = status
env['first_headers'] = headers
env['first_exc_info'] = exc_info
def _retry_loop(self,
orig_env,
start_response,
path_to_modify,
env_modifier,
alt_checker=None):
"""
:param env_modifier: function copies and modifies the env
dictionary, according to the alternative path parts
:param alt_checker: function that checks the alternative
path parts (may raise an exception)
"""
local_env = {}
for alt in self._alternatives(path_to_modify):
if alt_checker and not alt_checker(alt):
return self.app(orig_env, start_response)
env = env_modifier(orig_env, alt)
resp = self.app(env, partial(self._save_response, local_env))
if 'last_status' not in local_env:
# start_response() was not called. This happens when there is
# no 'proxy-logging' just after 'catch_errors' in the pipeline.
return resp
if not local_env['last_status'].startswith('404'):
# start_response() was called, and status is not 404.
start_response(local_env['last_status'],
local_env['last_headers'],
local_env['last_exc_info'])
return resp
if 'first_resp' not in local_env:
local_env['first_resp'] = resp
start_response(local_env['first_status'], local_env['first_headers'],
local_env['first_exc_info'])
return local_env['first_resp']
def _call_copy(self, env, start_response):
"""
Run the retry loop (copy operations).
"""
account, container, obj = self._convert_path(env.get('PATH_INFO'))
if obj is None:
# This is probably an account request
return self.app(env, start_response)
env['PATH_INFO'] = "/v1/%s/%s/%s" % (account, container, obj)
# HTTP_X_COPY_FROM_ACCOUNT will just pass through
if self.account_first:
src_path = "/fake_account" + env['HTTP_X_COPY_FROM']
else:
src_path = env['HTTP_X_COPY_FROM']
def modify_copy_from(orig_env, alternative):
env_ = orig_env.copy()
env_['HTTP_X_COPY_FROM'] = "/%s/%s" % (quote_plus(
alternative[1]), alternative[2])
return env_
def check_container_obj(alternative):
if not alternative[1] or not alternative[2]:
raise HTTPBadRequest(body="Malformed copy-source header")
return True
return self._retry_loop(env,
start_response,
src_path,
env_modifier=modify_copy_from,
alt_checker=check_container_obj)
def _call(self, env, start_response):
"""
Run the retry loop (regular operations).
"""
def modify_path_info(orig_env, alternative):
env_ = orig_env.copy()
env_['PATH_INFO'] = "/v1/%s/%s/%s" % alternative
return env_
def check_obj(alternative):
return alternative[2] is not None
return self._retry_loop(env,
start_response,
env['PATH_INFO'],
env_modifier=modify_path_info,
alt_checker=check_obj)
def __call__(self, env, start_response):
if self.should_bypass(env):
return self.app(env, start_response)
if self.is_copy(env):
return self._call_copy(env, start_response)
else:
return self._call(env, start_response)