def test_valid_api_version(self):
version = 'v1'
self.assertTrue(constraints.valid_api_version(version))
version = 'v1.0'
self.assertTrue(constraints.valid_api_version(version))
version = 'v2'
self.assertFalse(constraints.valid_api_version(version))
def __call__(self, env, start_response):
"""
WSGI entry point.
Wraps env in swob.Request object and passes it down.
:param env: WSGI environment dictionary
:param start_response: WSGI callable
"""
req = Request(env)
if self.memcache_client is None:
self.memcache_client = cache_from_env(env)
if not self.memcache_client:
self.logger.warning(
_('Warning: Cannot ratelimit without a memcached client'))
return self.app(env, start_response)
try:
version, account, container, obj = req.split_path(1, 4, True)
except ValueError:
return self.app(env, start_response)
if not valid_api_version(version):
return self.app(env, start_response)
ratelimit_resp = self.handle_ratelimit(req, account, container, obj)
if ratelimit_resp is None:
return self.app(env, start_response)
else:
return ratelimit_resp(env, start_response)
def get_controller(self, req):
"""
Get the controller to handle a request.
:param req: the request
:returns: tuple of (controller class, path dictionary)
:raises: ValueError (thrown by split_path) if given invalid path
"""
print 'req.path',req.path
if req.path == '/info':
d = dict(version=None,
expose_info=self.expose_info,
disallowed_sections=self.disallowed_sections,
admin_key=self.admin_key)
print 'd',d
return InfoController, d
version, account, container, obj = split_path(req.path, 1, 4, True)
d = dict(version=version,
account_name=account,
container_name=container,
object_name=obj)
print 'd',d
#print 'valid_api_version(version)',valid_api_version(version)
if account and not valid_api_version(version):
raise APIVersionError('Invalid path')
if obj and container and account:
info = get_container_info(req.environ, self)
print 'info of obj,Acc,Con',info
policy_index = req.headers.get('X-Backend-Storage-Policy-Index',
info['storage_policy'])
print 'policy_index',policy_index
policy = POLICIES.get_by_index(policy_index)
print 'policy',policy
if not policy:
# This indicates that a new policy has been created,
# with rings, deployed, released (i.e. deprecated =
# False), used by a client to create a container via
# another proxy that was restarted after the policy
# was released, and is now cached - all before this
# worker was HUPed to stop accepting new
# connections. There should never be an "unknown"
# index - but when there is - it's probably operator
# error and hopefully temporary.
raise HTTPServiceUnavailable('Unknown Storage Policy')
return self.obj_controller_router[policy], d
elif container and account:
print 'container & account, returning containercontroller',container,account
return ContainerController, d
elif account and not container and not obj:
print 'account, returning accountcontroller',account
return AccountController, d
return None, d
def get_controller(self, req):
"""
Get the controller to handle a request.
:param req: the request
:returns: tuple of (controller class, path dictionary)
:raises: ValueError (thrown by split_path) if given invalid path
"""
if req.path == '/info':
d = dict(version=None,
expose_info=self.expose_info,
disallowed_sections=self.disallowed_sections,
admin_key=self.admin_key)
return InfoController, d
#分割路径信息
version, account, container, obj = split_path(req.path, 1, 4, True)
#生成包含version、account、container、object的路径字典，用于返回
d = dict(version=version,
account_name=account,
container_name=container,
object_name=obj)
if account and not valid_api_version(version):
raise APIVersionError('Invalid path')
#如果是对象操作
if obj and container and account:
#获取container信息
info = get_container_info(req.environ, self)
policy_index = req.headers.get('X-Backend-Storage-Policy-Index',
info['storage_policy'])
#通过index获取存储策略对象
policy = POLICIES.get_by_index(policy_index)
if not policy:
# This indicates that a new policy has been created,
# with rings, deployed, released (i.e. deprecated =
# False), used by a client to create a container via
# another proxy that was restarted after the policy
# was released, and is now cached - all before this
# worker was HUPed to stop accepting new
# connections. There should never be an "unknown"
# index - but when there is - it's probably operator
# error and hopefully temporary.
raise HTTPServiceUnavailable('Unknown Storage Policy')
#返回对象操作的控制器对象，以及路径字典
return self.obj_controller_router[policy], d
#如果是container操作，返回container控制器，以及路径字典
elif container and account:
return ContainerController, d
#如果是account操作，返回account控制器，以及路径字典
elif account and not container and not obj:
return AccountController, d
return None, d
def get_controller(self, req):
"""
Get the controller to handle a request.
:param req: the request
:returns: tuple of (controller class, path dictionary)
:raises: ValueError (thrown by split_path) if given invalid path
"""
if req.path == '/info':
d = dict(version=None,
expose_info=self.expose_info,
disallowed_sections=self.disallowed_sections,
admin_key=self.admin_key)
return InfoController, d
version, account, container, obj = split_path(req.path, 1, 4, True)
d = dict(version=version,
account_name=account,
container_name=container,
object_name=obj)
if account and not valid_api_version(version):
raise APIVersionError('Invalid path')
if obj and container and account:
info = get_container_info(req.environ, self)
policy_index = req.headers.get('X-Backend-Storage-Policy-Index',
info['storage_policy'])
policy = POLICIES.get_by_index(policy_index)
if not policy:
# This indicates that a new policy has been created,
# with rings, deployed, released (i.e. deprecated =
# False), used by a client to create a container via
# another proxy that was restarted after the policy
# was released, and is now cached - all before this
# worker was HUPed to stop accepting new
# connections. There should never be an "unknown"
# index - but when there is - it's probably operator
# error and hopefully temporary.
raise HTTPServiceUnavailable('Unknown Storage Policy')
return self.obj_controller_router[policy], d
elif container and account:
return ContainerController, d
elif account and not container and not obj:
return AccountController, d
return None, d
def is_proxy_fs(self, env):
if 'pfs.is_bimodal' in env:
return env['pfs.is_bimodal']
try:
# Need at least an account
vers, a, c, o = utils.split_path(env['PATH_INFO'], 2, 4, True)
if not constraints.valid_api_version(vers):
raise ValueError
except ValueError:
# If it's not a swift request, it can't be a ProxyFS request
return False
account_info = get_account_info(env, self.base_app, swift_source='')
# ignore status; rely on something elsewhere in the pipeline
# to propagate the error
return utils.config_true_value(
account_info["sysmeta"].get('proxyfs-bimodal'))
def __call__(self, req):
vrs, acc, con, obj = utils.parse_path(req.path)
if not acc or not constraints.valid_api_version(vrs):
# could be a GET /info request or something made up by some
# other middleware; get out of the way.
return self.app
try:
is_bimodal, owner_addrinfo = self._fetch_owning_proxyfs(req, acc)
except (utils.RpcError, utils.RpcTimeout) as err:
return swob.HTTPServiceUnavailable(
request=req,
headers={'Content-Type': 'text/plain'},
body=str(err))
# Other middlewares will find and act on this
req.environ[utils.ENV_IS_BIMODAL] = is_bimodal
req.environ[utils.ENV_OWNING_PROXYFS] = owner_addrinfo
req.environ[utils.ENV_BIMODAL_CHECKER] = self
return self.app
def __call__(self, env, start_response):
req = Request(env)
try:
(api_version, account, container, obj) = req.split_path(2, 4, True)
bad_path = False
except ValueError:
bad_path = True
# use of bad_path bool is to avoid recursive tracebacks
if bad_path or not valid_api_version(api_version):
return self.app(env, start_response)
try:
if not container:
return self.account_request(req, api_version, account,
start_response)
if container and not obj:
return self.container_request(req, start_response)
else:
return self.object_request(req, api_version, account,
container, obj)(env, start_response)
except HTTPException as error_response:
return error_response(env, start_response)
def __call__(self, req):
try:
(version, account, container, obj) = \
split_path(req.path_info, 4, 4, True)
except ValueError:
return req.get_response(self.app)
# Only worry about data fetches, not uploads.
if not valid_api_version(version) or req.method not in ('GET', 'HEAD'):
return req.get_response(self.app)
# Get all roles that apply to the user making the request
roles = set()
if (req.environ.get('HTTP_X_IDENTITY_STATUS') == 'Confirmed' or \
req.environ.get('HTTP_X_SERVICE_IDENTITY_STATUS') in \
(None, "Confirmed")):
roles = set(list_from_csv(req.environ.get('HTTP_X_ROLES', '')))
# If we have one of the "nostrip" roles, then don't do any stripping
if roles.intersection(self.nostrip_roles):
return req.get_response(self.app)
# Perform the request and grab a response object that we can work
# with
resp = req.get_response(self.app)
# Check that the requested object is actually a CAIDA avro file
conttype = resp.headers.get("Content-Type", None)
if conttype is None:
return resp
if not conttype.startswith("application/vnd.caida."):
return resp
if not conttype.endswith(".avro"):
return resp
dtype = conttype.replace("application/vnd.caida.", "", 1)[:-5]
if dtype not in self.defaultstrip:
return resp
# Start by planning to strip all fields for this datatype that have
# been explicitly appeared in the config file. Then for each role that
# the user has, remove any fields from the strip set that should be
# retained for that role.
tostrip = self.defaultstrip[dtype]
for r in roles:
if r not in self.dontstrip:
# No specified config for this role, so leave strip set as is
continue
if dtype not in self.dontstrip[r]:
continue
tostrip = tostrip - self.dontstrip[r][dtype]
# Remove the Etag because otherwise swift clients get very upset
# about the md5sum of the response body not matching the md5sum
# in the Etag header :/
if 'Etag' in resp.headers:
del (resp.headers['Etag'])
# If we are going to be stripping fields, replace our response
# iterable with one that will parse the received Avro and remove
# the desired fields. The swift proxy should handle the rest.
x = GenericStrippingAvroParser(resp.app_iter, resp.body, tostrip)
resp.app_iter = x
return resp
def __call__(self, env, start_response):
req = Request(env)
try:
# account and container only
version, acct, cont = req.split_path(2, 3)
except ValueError:
return self.app(env, start_response)
if not valid_api_version(version) or req.method not in ('GET', 'HEAD'):
return self.app(env, start_response)
# OK, definitely have an account/container request.
# Get the desired content-type, then force it to a JSON request.
try:
out_content_type = get_listing_content_type(req)
except HTTPException as err:
return err(env, start_response)
params = req.params
params['format'] = 'json'
req.params = params
status, headers, resp_iter = req.call_application(self.app)
header_to_index = {}
resp_content_type = resp_length = None
for i, (header, value) in enumerate(headers):
header = header.lower()
if header == 'content-type':
header_to_index[header] = i
resp_content_type = value.partition(';')[0]
elif header == 'content-length':
header_to_index[header] = i
resp_length = int(value)
if not status.startswith('200 '):
start_response(status, headers)
return resp_iter
if resp_content_type != 'application/json':
start_response(status, headers)
return resp_iter
if resp_length is None or \
resp_length > MAX_CONTAINER_LISTING_CONTENT_LENGTH:
start_response(status, headers)
return resp_iter
def set_header(header, value):
if value is None:
del headers[header_to_index[header]]
else:
headers[header_to_index[header]] = (
headers[header_to_index[header]][0], str(value))
if req.method == 'HEAD':
set_header('content-type', out_content_type + '; charset=utf-8')
set_header('content-length', None) # don't know, can't determine
start_response(status, headers)
return resp_iter
body = b''.join(resp_iter)
try:
listing = json.loads(body)
# Do a couple sanity checks
if not isinstance(listing, list):
raise ValueError
if not all(isinstance(item, dict) for item in listing):
raise ValueError
except ValueError:
# Static web listing that's returning invalid JSON?
# Just pass it straight through; that's about all we *can* do.
start_response(status, headers)
return [body]
try:
if out_content_type.endswith('/xml'):
if cont:
body = container_to_xml(listing, cont)
else:
body = account_to_xml(listing, acct)
elif out_content_type == 'text/plain':
body = listing_to_text(listing)
# else, json -- we continue down here to be sure we set charset
except KeyError:
# listing was in a bad format -- funky static web listing??
start_response(status, headers)
return [body]
if not body:
status = '%s %s' % (HTTP_NO_CONTENT,
RESPONSE_REASONS[HTTP_NO_CONTENT][0])
set_header('content-type', out_content_type + '; charset=utf-8')
set_header('content-length', len(body))
start_response(status, headers)
return [body]
def __call__(self, env, start_response):
req = Request(env)
try:
# account and container only
version, acct, cont = req.split_path(2, 3)
except ValueError:
is_account_or_container_req = False
else:
is_account_or_container_req = True
if not is_account_or_container_req:
return self.app(env, start_response)
if not valid_api_version(version) or req.method not in ('GET', 'HEAD'):
return self.app(env, start_response)
# OK, definitely have an account/container request.
# Get the desired content-type, then force it to a JSON request.
try:
out_content_type = get_listing_content_type(req)
except HTTPException as err:
return err(env, start_response)
params = req.params
can_vary = 'format' not in params
params['format'] = 'json'
req.params = params
# Give other middlewares a chance to be in charge
env.setdefault('swift.format_listing', True)
status, headers, resp_iter = req.call_application(self.app)
if not env.get('swift.format_listing'):
start_response(status, headers)
return resp_iter
header_to_index = {}
resp_content_type = resp_length = None
for i, (header, value) in enumerate(headers):
header = header.lower()
if header == 'content-type':
header_to_index[header] = i
resp_content_type = value.partition(';')[0]
elif header == 'content-length':
header_to_index[header] = i
resp_length = int(value)
elif header == 'vary':
header_to_index[header] = i
if not status.startswith(('200 ', '204 ')):
start_response(status, headers)
return resp_iter
if can_vary:
if 'vary' in header_to_index:
value = headers[header_to_index['vary']][1]
if 'accept' not in list_from_csv(value.lower()):
headers[header_to_index['vary']] = ('Vary',
value + ', Accept')
else:
headers.append(('Vary', 'Accept'))
if resp_content_type != 'application/json':
start_response(status, headers)
return resp_iter
if resp_length is None or \
resp_length > MAX_CONTAINER_LISTING_CONTENT_LENGTH:
start_response(status, headers)
return resp_iter
def set_header(header, value):
if value is None:
del headers[header_to_index[header]]
else:
headers[header_to_index[header]] = (
headers[header_to_index[header]][0], str(value))
if req.method == 'HEAD':
set_header('content-type', out_content_type + '; charset=utf-8')
set_header('content-length', None) # don't know, can't determine
start_response(status, headers)
return resp_iter
body = b''.join(resp_iter)
try:
listing = json.loads(body)
# Do a couple sanity checks
if not isinstance(listing, list):
raise ValueError
if not all(isinstance(item, dict) for item in listing):
raise ValueError
except ValueError:
# Static web listing that's returning invalid JSON?
# Just pass it straight through; that's about all we *can* do.
start_response(status, headers)
return [body]
if not req.allow_reserved_names:
listing = self.filter_reserved(listing, acct, cont)
try:
if out_content_type.endswith('/xml'):
if cont:
body = container_to_xml(
listing,
wsgi_to_bytes(cont).decode('utf-8'))
else:
body = account_to_xml(listing,
wsgi_to_bytes(acct).decode('utf-8'))
elif out_content_type == 'text/plain':
body = listing_to_text(listing)
else:
body = json.dumps(listing).encode('ascii')
except KeyError:
# listing was in a bad format -- funky static web listing??
start_response(status, headers)
return [body]
if not body:
status = '%s %s' % (HTTP_NO_CONTENT,
RESPONSE_REASONS[HTTP_NO_CONTENT][0])
set_header('content-type', out_content_type + '; charset=utf-8')
set_header('content-length', len(body))
start_response(status, headers)
return [body]
def __call__(self, env, start_response):
if time() > self._rtime:
self._reload()
req = swob.Request(env)
try:
vers, acct, cont, obj = req.split_path(2, 4, True)
except ValueError:
return self.app(env, start_response)
if req.headers.get(SHUNT_BYPASS_HEADER, ''):
self.logger.debug('Bypassing shunt (%s header) for %r',
SHUNT_BYPASS_HEADER, req.path_info)
return self.app(env, start_response)
if not constraints.valid_api_version(vers):
return self.app(env, start_response)
if not cont:
sync_profile = self.sync_profiles.get((acct, '/*'))
if req.method == 'GET' and sync_profile and\
sync_profile.get('migration'):
# TODO: make the container an optional parameter
profile, _ = maybe_munge_profile_for_all_containers(
sync_profile, '.stub-container')
return self.handle_account(req, start_response, profile, acct)
return self.app(env, start_response)
sync_profile = next(
(self.sync_profiles[(acct, c)]
for c in (cont, '/*') if (acct, c) in self.sync_profiles), None)
if sync_profile is None:
return self.app(env, start_response)
sync_profile, per_account = maybe_munge_profile_for_all_containers(
sync_profile, cont)
if req.method == 'DELETE' and sync_profile.get('migration'):
return self.handle_delete(req, start_response, sync_profile, obj,
per_account)
if not obj:
if req.method == 'GET':
return self.handle_listing(req, start_response, sync_profile,
cont, per_account)
if req.method == 'HEAD' and sync_profile.get('migration'):
return self.handle_container_head(req, start_response,
sync_profile, cont,
per_account)
if obj and req.method in ('GET', 'HEAD'):
# TODO: think about what to do for POST, COPY
return self.handle_object(req, start_response, sync_profile, obj,
per_account)
if req.method == 'POST' and sync_profile.get('migration'):
return self.handle_post(req, start_response, sync_profile, obj,
per_account)
if obj and req.method == 'PUT' and sync_profile.get('migration'):
return self.handle_object_put(req, start_response, sync_profile,
per_account)
return self.app(env, start_response)
def __call__(self, env, start_response):
# a lot of this is cribbed from listing_formats / swob.Request
if env['REQUEST_METHOD'] != 'GET':
# Nothing to translate
return self.app(env, start_response)
try:
v, a, c = split_path(env.get('SCRIPT_NAME', '') +
env['PATH_INFO'], 3, 3)
if not valid_api_version(v):
raise ValueError
except ValueError:
# not a container request; pass through
return self.app(env, start_response)
ctx = WSGIContext(self.app)
resp_iter = ctx._app_call(env)
content_type = content_length = cl_index = None
for index, (header, value) in enumerate(ctx._response_headers):
header = header.lower()
if header == 'content-type':
content_type = value.split(';', 1)[0].strip()
if content_length:
break
elif header == 'content-length':
cl_index = index
try:
content_length = int(value)
except ValueError:
pass # ignore -- we'll bail later
if content_type:
break
if content_type != 'application/json' or content_length is None or \
content_length > MAX_CONTAINER_LISTING_CONTENT_LENGTH:
start_response(ctx._response_status, ctx._response_headers,
ctx._response_exc_info)
return resp_iter
# We've done our sanity checks, slurp the response into memory
with closing_if_possible(resp_iter):
body = b''.join(resp_iter)
try:
listing = json.loads(body)
for item in listing:
if 'subdir' in item:
continue
value, params = parse_header(item['hash'])
if 's3_etag' in params:
item['s3_etag'] = '"%s"' % params.pop('s3_etag')
item['hash'] = value + ''.join(
'; %s=%s' % kv for kv in params.items())
except (TypeError, KeyError, ValueError):
# If anything goes wrong above, drop back to original response
start_response(ctx._response_status, ctx._response_headers,
ctx._response_exc_info)
return [body]
body = json.dumps(listing).encode('ascii')
ctx._response_headers[cl_index] = (
ctx._response_headers[cl_index][0],
str(len(body)),
)
start_response(ctx._response_status, ctx._response_headers,
ctx._response_exc_info)
return [body]
def get_controller(self, req):
"""
Get the controller to handle a request.
获得一个请求的控制器
:param req: the request
:returns: tuple of (controller class, path dictionary)
:raises: ValueError (thrown by split_path) if given invalid path
"""
#如果请求为/info{?swiftinfo_sig,swiftinfo_expires}这种格式
#表示请求的是一些info,所以会调用InfoController
if req.path == '/info':
d = dict(version=None,
expose_info=self.expose_info,
disallowed_sections=self.disallowed_sections,
admin_key=self.admin_key)
return InfoController, d
#获得具体信息
version, account, container, obj = split_path(req.path, 1, 4, True)
d = dict(version=version,
account_name=account,
container_name=container,
object_name=obj)
#account存在但是version不对，目前version只能是v1或者是v1.0
if account and not valid_api_version(version):
raise APIVersionError('Invalid path')
#如果path中account, container和object都存在，返回的是object的controller
if obj and container and account:
info = get_container_info(req.environ, self)
#获得container的存储策略，共有三种策略，在配置文件swift.conf中有，可以去查看
policy_index = req.headers.get('X-Backend-Storage-Policy-Index',
info['storage_policy'])
policy = POLICIES.get_by_index(policy_index)
if not policy:
# This indicates that a new policy has been created,
# with rings, deployed, released (i.e. deprecated =
# False), used by a client to create a container via
# another proxy that was restarted after the policy
# was released, and is now cached - all before this
# worker was HUPed to stop accepting new
# connections. There should never be an "unknown"
# index - but when there is - it's probably operator
# error and hopefully temporary.
raise HTTPServiceUnavailable('Unknown Storage Policy')
return self.obj_controller_router[policy], d
#如果path中只包含account和container，则返回container的controller
elif container and account:
return ContainerController, d
#如果path只存在account，则返回account的controller
elif account and not container and not obj:
return AccountController, d
#都没有，返回None
return None, d
def __call__(self, env, start_response):
req = Request(env)
try:
version, account, container, obj = req.split_path(
2, 4, rest_with_last=True)
is_swifty_request = valid_api_version(version)
except ValueError:
is_swifty_request = False
if not is_swifty_request:
return self.app(env, start_response)
if not obj:
typ = 'Container' if container else 'Account'
client_header = 'X-%s-Rfc-Compliant-Etags' % typ
sysmeta_header = 'X-%s-Sysmeta-Rfc-Compliant-Etags' % typ
if client_header in req.headers:
if req.headers[client_header]:
req.headers[sysmeta_header] = config_true_value(
req.headers[client_header])
else:
req.headers[sysmeta_header] = ''
if req.headers.get(client_header.replace('X-', 'X-Remove-', 1)):
req.headers[sysmeta_header] = ''
def translating_start_response(status, headers, exc_info=None):
return start_response(
status,
[(client_header if h.title() == sysmeta_header else h, v)
for h, v in headers], exc_info)
return self.app(env, translating_start_response)
container_info = get_container_info(env, self.app, 'EQ')
if not container_info or not is_success(container_info['status']):
return self.app(env, start_response)
flag = container_info.get('sysmeta', {}).get('rfc-compliant-etags')
if flag is None:
account_info = get_account_info(env, self.app, 'EQ')
if not account_info or not is_success(account_info['status']):
return self.app(env, start_response)
flag = account_info.get('sysmeta', {}).get('rfc-compliant-etags')
if flag is None:
flag = self.conf.get('enable_by_default', 'false')
if not config_true_value(flag):
return self.app(env, start_response)
status, headers, resp_iter = req.call_application(self.app)
for i, (header, value) in enumerate(headers):
if header.lower() == 'etag':
if not value.startswith(('"', 'W/"')) or \
not value.endswith('"'):
headers[i] = (header, '"%s"' % value)
start_response(status, headers)
return resp_iter
def __call__(self, req):
if req.path == '/info':
# Ensure /info requests get the freshest results
self.register_info()
return self.app
try:
(version, acc, cont, obj) = req.split_path(3, 4, True)
bad_path = False
except ValueError:
bad_path = True
# use of bad_path bool is to avoid recursive tracebacks
if bad_path or not valid_api_version(version):
return self.app
# validate container-sync metdata update
info = get_container_info(req.environ, self.app, swift_source='CS')
sync_to = req.headers.get('x-container-sync-to')
if req.method in ('PUT', 'POST') and cont and not obj:
versions_cont = info.get('sysmeta', {}).get('versions-container')
if sync_to and versions_cont:
raise HTTPBadRequest(
'Cannot configure container sync on a container '
'with object versioning configured.',
request=req)
if not self.allow_full_urls:
if sync_to and not sync_to.startswith('//'):
raise HTTPBadRequest(
body='Full URLs are not allowed for X-Container-Sync-To '
'values. Only realm values of the format '
'//realm/cluster/account/container are allowed.\n',
request=req)
auth = req.headers.get('x-container-sync-auth')
if auth:
valid = False
auth = auth.split()
if len(auth) != 3:
req.environ.setdefault('swift.log_info',
[]).append('cs:not-3-args')
else:
realm, nonce, sig = auth
realm_key = self.realms_conf.key(realm)
realm_key2 = self.realms_conf.key2(realm)
if not realm_key:
req.environ.setdefault('swift.log_info',
[]).append('cs:no-local-realm-key')
else:
user_key = info.get('sync_key')
if not user_key:
req.environ.setdefault(
'swift.log_info',
[]).append('cs:no-local-user-key')
else:
# x-timestamp headers get shunted by gatekeeper
if 'x-backend-inbound-x-timestamp' in req.headers:
req.headers['x-timestamp'] = req.headers.pop(
'x-backend-inbound-x-timestamp')
expected = self.realms_conf.get_sig(
req.method, req.path,
req.headers.get('x-timestamp', '0'), nonce,
realm_key, user_key)
expected2 = self.realms_conf.get_sig(
req.method, req.path,
req.headers.get('x-timestamp', '0'), nonce,
realm_key2, user_key) if realm_key2 else expected
if not streq_const_time(sig, expected) and \
not streq_const_time(sig, expected2):
req.environ.setdefault('swift.log_info',
[]).append('cs:invalid-sig')
else:
req.environ.setdefault('swift.log_info',
[]).append('cs:valid')
valid = True
if not valid:
exc = HTTPUnauthorized(
body='X-Container-Sync-Auth header not valid; '
'contact cluster operator for support.',
headers={'content-type': 'text/plain'},
request=req)
exc.headers['www-authenticate'] = ' '.join([
'SwiftContainerSync',
exc.www_authenticate().split(None, 1)[1]
])
raise exc
else:
req.environ['swift.authorize_override'] = True
# An SLO manifest will already be in the internal manifest
# syntax and might be synced before its segments, so stop SLO
# middleware from performing the usual manifest validation.
req.environ['swift.slo_override'] = True
# Similar arguments for static symlinks
req.environ['swift.symlink_override'] = True
return self.app
def __call__(self, env, start_response):
# a lot of this is cribbed from listing_formats / swob.Request
if env['REQUEST_METHOD'] != 'GET':
# Nothing to translate
return self.app(env, start_response)
try:
v, a, c = split_path(
env.get('SCRIPT_NAME', '') + env['PATH_INFO'], 3, 3)
if not valid_api_version(v):
raise ValueError
except ValueError:
# not a container request; pass through
return self.app(env, start_response)
ctx = WSGIContext(self.app)
resp_iter = ctx._app_call(env)
content_type = content_length = cl_index = None
for index, (header, value) in enumerate(ctx._response_headers):
header = header.lower()
if header == 'content-type':
content_type = value.split(';', 1)[0].strip()
if content_length:
break
elif header == 'content-length':
cl_index = index
try:
content_length = int(value)
except ValueError:
pass # ignore -- we'll bail later
if content_type:
break
if content_type != 'application/json' or content_length is None or \
content_length > MAX_CONTAINER_LISTING_CONTENT_LENGTH:
start_response(ctx._response_status, ctx._response_headers,
ctx._response_exc_info)
return resp_iter
# We've done our sanity checks, slurp the response into memory
with closing_if_possible(resp_iter):
body = b''.join(resp_iter)
try:
listing = json.loads(body)
for item in listing:
if 'subdir' in item:
continue
value, params = parse_header(item['hash'])
if 's3_etag' in params:
item['s3_etag'] = '"%s"' % params.pop('s3_etag')
item['hash'] = value + ''.join('; %s=%s' % kv
for kv in params.items())
except (TypeError, KeyError, ValueError):
# If anything goes wrong above, drop back to original response
start_response(ctx._response_status, ctx._response_headers,
ctx._response_exc_info)
return [body]
body = json.dumps(listing)
ctx._response_headers[cl_index] = (
ctx._response_headers[cl_index][0],
str(len(body)),
)
start_response(ctx._response_status, ctx._response_headers,
ctx._response_exc_info)
return [body]
def __call__(self, env, start_response):
req = Request(env)
try:
# account and container only
version, acct, cont = req.split_path(2, 3)
except ValueError:
return self.app(env, start_response)
if not valid_api_version(version) or req.method not in ('GET', 'HEAD'):
return self.app(env, start_response)
# OK, definitely have an account/container request.
# Get the desired content-type, then force it to a JSON request.
try:
out_content_type = get_listing_content_type(req)
except HTTPException as err:
return err(env, start_response)
params = req.params
params['format'] = 'json'
req.params = params
status, headers, resp_iter = req.call_application(self.app)
header_to_index = {}
resp_content_type = resp_length = None
for i, (header, value) in enumerate(headers):
header = header.lower()
if header == 'content-type':
header_to_index[header] = i
resp_content_type = value.partition(';')[0]
elif header == 'content-length':
header_to_index[header] = i
resp_length = int(value)
if not status.startswith('200 '):
start_response(status, headers)
return resp_iter
if resp_content_type != 'application/json':
start_response(status, headers)
return resp_iter
if resp_length is None or \
resp_length > MAX_CONTAINER_LISTING_CONTENT_LENGTH:
start_response(status, headers)
return resp_iter
def set_header(header, value):
if value is None:
del headers[header_to_index[header]]
else:
headers[header_to_index[header]] = (
headers[header_to_index[header]][0], str(value))
if req.method == 'HEAD':
set_header('content-type', out_content_type + '; charset=utf-8')
set_header('content-length', None) # don't know, can't determine
start_response(status, headers)
return resp_iter
body = b''.join(resp_iter)
try:
listing = json.loads(body)
# Do a couple sanity checks
if not isinstance(listing, list):
raise ValueError
if not all(isinstance(item, dict) for item in listing):
raise ValueError
except ValueError:
# Static web listing that's returning invalid JSON?
# Just pass it straight through; that's about all we *can* do.
start_response(status, headers)
return [body]
try:
if out_content_type.endswith('/xml'):
if cont:
body = container_to_xml(listing, cont)
else:
body = account_to_xml(listing, acct)
elif out_content_type == 'text/plain':
body = listing_to_text(listing)
# else, json -- we continue down here to be sure we set charset
except KeyError:
# listing was in a bad format -- funky static web listing??
start_response(status, headers)
return [body]
if not body:
status = '%s %s' % (HTTP_NO_CONTENT,
RESPONSE_REASONS[HTTP_NO_CONTENT][0])
set_header('content-type', out_content_type + '; charset=utf-8')
set_header('content-length', len(body))
start_response(status, headers)
return [body]
