def __init__(self, env, path, swift_source=None):
self.environ = env
(version, account, container, obj) = split_path(path, 2, 4, True)
self.account = account
self.container = container
self.obj = obj
if obj:
stype = 'object'
self.headers = {'content-length': 5555,
'content-type': 'text/plain'}
else:
stype = container and 'container' or 'account'
self.headers = {'x-%s-object-count' % (stype): 1000,
'x-%s-bytes-used' % (stype): 6666}
if swift_source:
meta = 'x-%s-meta-fakerequest-swift-source' % stype
self.headers[meta] = swift_source
def split_and_validate_path(request,
minsegs=1,
maxsegs=None,
rest_with_last=False):
"""
Utility function to split and validate the request path.
:returns: result of split_path if everything's okay
:raises: HTTPBadRequest if something's not okay
"""
try:
segs = split_path(unquote(request.path), minsegs, maxsegs,
rest_with_last)
validate_device_partition(segs[0], segs[1])
return segs
except ValueError as err:
raise HTTPBadRequest(body=str(err),
request=request,
content_type='text/plain')
def check_copy_from_header(req):
"""
Validate that the value from x-copy-from header is
well formatted. We assume the caller ensures that
x-copy-from header is present in req.headers.
:param req: HTTP request object
:returns: A tuple with container name and object name
:raise: HTTPPreconditionFailed if x-copy-from value
is not well formatted.
"""
src_header = unquote(req.headers.get('X-Copy-From'))
if not src_header.startswith('/'):
src_header = '/' + src_header
try:
return split_path(src_header, 2, 2, True)
except ValueError:
raise HTTPPreconditionFailed(
request=req,
body='X-Copy-From header must be of the form'
'<container name>/<object name>')
def __get_service_details(self, metadata, host=''):
'''
Get container service details, filesystem, directory
of container update request.
:param metadata: object metadata
:returns: host: string containing IP and port of container service
directory: directory name
file_system:filesystem name
'''
account, container, _ = split_path(metadata['name'], 1, 3, True)
node, filesystem, directory, global_map_version, comp_no = \
self.__container_ring.get_node(account, container)
#TODO : Gl interface
#GET_GLOBAL_MAP
#component_name = get_path_name()
#import libraryutils
#ip,port = get_ip(),get_port()
#host = '%s:%s' % (ip,port)
if not host:
host = '%s:%s' % (node[0]['ip'], node[0]['port'])
return host, directory, filesystem
def split_path(self, minsegs=1, maxsegs=None, rest_with_last=False):
"""
Validate and split the Request's path.
**Examples**::
['a'] = split_path('/a')
['a', None] = split_path('/a', 1, 2)
['a', 'c'] = split_path('/a/c', 1, 2)
['a', 'c', 'o/r'] = split_path('/a/c/o/r', 1, 3, True)
:param minsegs: Minimum number of segments to be extracted
:param maxsegs: Maximum number of segments to be extracted
:param rest_with_last: If True, trailing data will be returned as part
of last segment. If False, and there is
trailing data, raises ValueError.
:returns: list of segments with a length of maxsegs (non-existant
segments will return as None)
:raises: ValueError if given an invalid path
"""
return split_path(
self.environ.get('SCRIPT_NAME', '') + self.environ['PATH_INFO'],
minsegs, maxsegs, rest_with_last)
def __get_target_path(self, metadata, file_name):
'''
Get targets path of object file and meta file storage.
:param tmp_dir: tmp directory
:param file_name: file name
:returns: data file target path and meta file target path
'''
data_file = file_name.split('_')[-1] + '.data'
meta_file = file_name.split('_')[-1] + '.meta'
path = metadata['name']
account, container, obj = split_path(path, 1, 3, True)
#_, file_system, directory, global_map_version, comp_no = \
file_system, directory = \
self.__object_ring.get_node(account, container, obj, only_fs_dir_flag=True)
acc_dir, cont_dir, obj_dir = directory.split('/')
data_target_path = join(EXPORT_PATH, file_system, acc_dir, hash_path(account), \
cont_dir, hash_path(account, container), \
obj_dir, DATADIR, data_file)
meta_target_path = join(EXPORT_PATH, file_system, acc_dir, hash_path(account), \
cont_dir, hash_path(account, container), \
obj_dir, METADIR, meta_file)
return data_target_path, meta_target_path
def get_controller(self, path, method):
"""
Get the controller to handle a request.
:param path: path from request
:returns: tuple of (controller class, path dictionary)
:raises: ValueError (thrown by split_path) if given invalid path
"""
version, account, container, obj = split_path(path, 1, 4, True)
d = dict(version=version,
account_name=account,
container_name=container,
object_name=obj)
if method == "STOP_SERVICE":
return StopController, d
if obj and container and account:
return ObjectController, d
elif container and account:
return ContainerController, d
elif account and not container and not obj:
return AccountController, d
return None, d
def handle_request(self, req):
"""
Entry point for proxy server.
Should return a WSGI-style callable (such as swob.Response).
:param req: swob.Request object
"""
try:
if 'swift.trans_id' not in req.environ:
# if this wasn't set by an earlier middleware, set it now
trans_id = generate_trans_id(self.trans_id_suffix)
req.environ['swift.trans_id'] = trans_id
self.logger.txn_id = trans_id
self.logger.info(
'Proxy service received request for %(method)s '
'%(path)s with %(headers)s (txn: %(trans_id)s)', {
'method': req.method,
'path': req.path,
'headers': req.headers,
'trans_id': self.logger.txn_id
})
req.headers['x-trans-id'] = req.environ['swift.trans_id']
self.logger.set_statsd_prefix('proxy-server')
#if self.stop_service_flag:
# self.logger.info('Proxy is going to stop therefore no more request')
# return HTTPForbidden(request=req, body='Proxy is going to stop\
# therefore no more request') #TODO would it be HTTPForbidden or something else
if req.content_length and req.content_length < 0:
self.logger.increment('errors')
return HTTPBadRequest(request=req,
body='Invalid Content-Length')
try:
if not check_utf8(req.path_info):
self.logger.increment('errors')
return HTTPPreconditionFailed(
request=req, body='Invalid UTF8 or contains NULL')
except UnicodeError:
self.logger.increment('errors')
return HTTPPreconditionFailed(
request=req, body='Invalid UTF8 or contains NULL')
try:
error_response = constraints.check_non_allowed_headers(req)
if error_response:
return error_response
obj_list = ''
if ('bulk-delete' in req.params or 'X-Bulk-Delete' in \
req.headers) and req.method in ['POST', 'DELETE']:
self.logger.debug("*** Bulk delete request ***")
cont, obj_list = get_objs_to_delete(req, \
self.max_bulk_delete_entries, self.logger)
self.logger.debug("Container: %s, Obj list: %s" \
% (cont, obj_list))
version, account, container = \
split_path(req.path, 1, 3, True)
if container:
container = container.strip('/')
if cont != container:
self.logger.error("Container in path is different")
return HTTPBadRequest(request=req, \
body='Container name mismatch')
req.path_info = req.path.rstrip('/')
else:
req.path_info = os.path.join(req.path, cont)
req.method = 'BULK_DELETE'
req.headers['Content-Length'] = len(str(obj_list))
req.body = str(obj_list)
controller, path_parts = self.get_controller(
req.path, req.method)
p = req.path_info
if isinstance(p, unicode):
p = p.encode('utf-8')
except ValueError:
self.logger.increment('errors')
return HTTPNotFound(request=req)
if not controller:
self.logger.increment('errors')
return HTTPPreconditionFailed(request=req, body='Bad URL')
if self.deny_host_headers and \
req.host.split(':')[0] in self.deny_host_headers:
return HTTPForbidden(request=req, body='Invalid host header')
self.logger.set_statsd_prefix('proxy-server.' +
controller.server_type.lower())
controller = controller(self, **path_parts)
controller.trans_id = req.environ['swift.trans_id']
#self.logger.client_ip = get_remote_client(req)
try:
handler = getattr(controller, req.method)
getattr(handler, 'publicly_accessible')
except AttributeError:
allowed_methods = getattr(controller, 'allowed_methods', set())
return HTTPMethodNotAllowed(
request=req, headers={'Allow': ', '.join(allowed_methods)})
if 'swift.authorize' in req.environ:
# We call authorize before the handler, always. If authorized,
# we remove the swift.authorize hook so isn't ever called
# again. If not authorized, we return the denial unless the
# controller's method indicates it'd like to gather more
# information and try again later.
resp = req.environ['swift.authorize'](req)
if not resp:
# No resp means authorized, no delayed recheck required.
del req.environ['swift.authorize']
else:
# Response indicates denial, but we might delay the denial
# and recheck later. If not delayed, return the error now.
if not getattr(handler, 'delay_denial', None):
return resp
# Save off original request method (GET, POST, etc.) in case it
# gets mutated during handling. This way logging can display the
# method the client actually sent.
req.environ['swift.orig_req_method'] = req.method
if not req.method == "STOP_SERVICE":
self.ongoing_operation_list.append(req)
resp = handler(req)
self.logger.info(
'Proxy returning response %(status)s for '
'%(method)s %(path)s with %(headers)s '
'(txn: %(trans_id)s)', {
'status': resp.status,
'method': req.method,
'path': req.path,
'headers': resp.headers,
'trans_id': req.headers['x-trans-id']
})
return resp
except HTTPException as error_response:
return error_response
except (Exception, Timeout):
self.logger.exception(_('ERROR Unhandled exception in request'))
return HTTPServerError(request=req)
def publish_sample(self, env, bytes_received, bytes_sent):
req = REQUEST.Request(env)
try:
version, account, container, obj = split_path(req.path, 2, 4, True)
except ValueError:
return
now = timeutils.utcnow().isoformat()
resource_metadata = {
"path": req.path,
"version": version,
"container": container,
"object": obj,
}
for header in self.metadata_headers:
if header.upper() in req.headers:
resource_metadata['http_header_%s' % header] = req.headers.get(
header.upper())
with self.pipeline_manager.publisher(
context.get_admin_context()) as publisher:
if bytes_received:
publisher([
sample.Sample(name='storage.objects.incoming.bytes',
type=sample.TYPE_DELTA,
unit='B',
volume=bytes_received,
user_id=env.get('HTTP_X_USER_ID'),
project_id=env.get('HTTP_X_TENANT_ID'),
resource_id=account.partition(
self.reseller_prefix)[2],
timestamp=now,
resource_metadata=resource_metadata)
])
if bytes_sent:
publisher([
sample.Sample(name='storage.objects.outgoing.bytes',
type=sample.TYPE_DELTA,
unit='B',
volume=bytes_sent,
user_id=env.get('HTTP_X_USER_ID'),
project_id=env.get('HTTP_X_TENANT_ID'),
resource_id=account.partition(
self.reseller_prefix)[2],
timestamp=now,
resource_metadata=resource_metadata)
])
# publish the event for each request
# request method will be recorded in the metadata
resource_metadata['method'] = req.method.lower()
publisher([
sample.Sample(name='storage.api.request',
type=sample.TYPE_DELTA,
unit='request',
volume=1,
user_id=env.get('HTTP_X_USER_ID'),
project_id=env.get('HTTP_X_TENANT_ID'),
resource_id=account.partition(
self.reseller_prefix)[2],
timestamp=now,
resource_metadata=resource_metadata)
])
def handle_get_token(self, req):
"""
Handles the various `request for token and service end point(s)` calls.
There are various formats to support the various auth servers in the
past. Examples::
GET <auth-prefix>/v1/<act>/auth
X-Auth-User: <act>:<usr> or X-Storage-User: <usr>
X-Auth-Key: <key> or X-Storage-Pass: <key>
GET <auth-prefix>/auth
X-Auth-User: <act>:<usr> or X-Storage-User: <act>:<usr>
X-Auth-Key: <key> or X-Storage-Pass: <key>
GET <auth-prefix>/v1.0
X-Auth-User: <act>:<usr> or X-Storage-User: <act>:<usr>
X-Auth-Key: <key> or X-Storage-Pass: <key>
On successful authentication, the response will have X-Auth-Token and
X-Storage-Token set to the token to use with Swift and X-Storage-URL
set to the URL to the default Swift cluster to use.
:param req: The swob.Request to process.
:returns: swob.Response, 2xx on success with data set as explained
above.
"""
# Validate the request info
try:
pathsegs = split_path(req.path_info, 1, 3, True)
except ValueError:
self.logger.increment('errors')
return HTTPNotFound(request=req)
if pathsegs[0] == 'v1' and pathsegs[2] == 'auth':
account = pathsegs[1]
user = req.headers.get('x-storage-user')
if not user:
user = req.headers.get('x-auth-user')
if not user or ':' not in user:
self.logger.increment('token_denied')
return HTTPUnauthorized(request=req,
headers={
'Www-Authenticate':
'Swift realm="%s"' % account
})
account2, user = user.split(':', 1)
if account != account2:
self.logger.increment('token_denied')
return HTTPUnauthorized(request=req,
headers={
'Www-Authenticate':
'Swift realm="%s"' % account
})
key = req.headers.get('x-storage-pass')
if not key:
key = req.headers.get('x-auth-key')
elif pathsegs[0] in ('auth', 'v1.0'):
user = req.headers.get('x-auth-user')
if not user:
user = req.headers.get('x-storage-user')
if not user or ':' not in user:
self.logger.increment('token_denied')
return HTTPUnauthorized(
request=req,
headers={'Www-Authenticate': 'Swift realm="unknown"'})
account, user = user.split(':', 1)
key = req.headers.get('x-auth-key')
if not key:
key = req.headers.get('x-storage-pass')
else:
return HTTPBadRequest(request=req)
if not all((account, user, key)):
self.logger.increment('token_denied')
realm = account or 'unknown'
return HTTPUnauthorized(
request=req,
headers={'Www-Authenticate': 'Swift realm="%s"' % realm})
# Authenticate user
account_user = account + ':' + user
if account_user not in self.users:
self.logger.increment('token_denied')
return HTTPUnauthorized(
request=req,
headers={'Www-Authenticate': 'Swift realm="%s"' % account})
if self.users[account_user]['key'] != key:
self.logger.increment('token_denied')
return HTTPUnauthorized(
request=req,
headers={'Www-Authenticate': 'Swift realm="unknown"'})
account_id = self.users[account_user]['url'].rsplit('/', 1)[-1]
# Get memcache client
memcache_client = cache_from_env(req.environ)
if not memcache_client:
raise Exception('Memcache required')
# See if a token already exists and hasn't expired
token = None
memcache_user_key = '%s/user/%s' % (self.reseller_prefix, account_user)
candidate_token = memcache_client.get(memcache_user_key)
if candidate_token:
memcache_token_key = \
'%s/token/%s' % (self.reseller_prefix, candidate_token)
cached_auth_data = memcache_client.get(memcache_token_key)
if cached_auth_data:
expires, old_groups = cached_auth_data
old_groups = old_groups.split(',')
new_groups = self._get_user_groups(account, account_user,
account_id)
if expires > time() and \
set(old_groups) == set(new_groups.split(',')):
token = candidate_token
# Create a new token if one didn't exist
if not token:
# Generate new token
token = '%stk%s' % (self.reseller_prefix, uuid4().hex)
expires = time() + self.token_life
groups = self._get_user_groups(account, account_user, account_id)
# Save token
memcache_token_key = '%s/token/%s' % (self.reseller_prefix, token)
memcache_client.set(memcache_token_key, (expires, groups),
time=float(expires - time()))
# Record the token with the user info for future use.
memcache_user_key = \
'%s/user/%s' % (self.reseller_prefix, account_user)
memcache_client.set(memcache_user_key,
token,
time=float(expires - time()))
resp = Response(request=req,
headers={
'x-auth-token': token,
'x-storage-token': token
})
url = self.users[account_user]['url'].replace('$HOST', resp.host_url)
if self.storage_url_scheme != 'default':
url = self.storage_url_scheme + ':' + url.split(':', 1)[1]
resp.headers['x-storage-url'] = url
return resp
def __call__(self, env, start_response):
"""
Accepts a standard WSGI application call, authenticating the request
and installing callback hooks for authorization and ACL header
validation. For an authenticated request, REMOTE_USER will be set to a
comma separated list of the user's groups.
With a non-empty reseller prefix, acts as the definitive auth service
for just tokens and accounts that begin with that prefix, but will deny
requests outside this prefix if no other auth middleware overrides it.
With an empty reseller prefix, acts as the definitive auth service only
for tokens that validate to a non-empty set of groups. For all other
requests, acts as the fallback auth service when no other auth
middleware overrides it.
Alternatively, if the request matches the self.auth_prefix, the request
will be routed through the internal auth request handler (self.handle).
This is to handle granting tokens, etc.
"""
if self.allow_overrides and env.get('swift.authorize_override', False):
return self.app(env, start_response)
if env.get('PATH_INFO', '').startswith(self.auth_prefix):
return self.handle(env, start_response)
s3 = env.get('HTTP_AUTHORIZATION')
token = env.get('HTTP_X_AUTH_TOKEN', env.get('HTTP_X_STORAGE_TOKEN'))
if s3 or (token and token.startswith(self.reseller_prefix)):
# Note: Empty reseller_prefix will match all tokens.
groups = self.get_groups(env, token)
if groups:
user = groups and groups.split(',', 1)[0] or ''
trans_id = env.get('swift.trans_id')
self.logger.debug('User: %s uses token %s (trans_id %s)' %
(user, 's3' if s3 else token, trans_id))
env['REMOTE_USER'] = groups
env['swift.authorize'] = self.authorize
env['swift.clean_acl'] = clean_acl
if '.reseller_admin' in groups:
env['reseller_request'] = True
else:
# Unauthorized token
if self.reseller_prefix and not s3:
# Because I know I'm the definitive auth for this token, I
# can deny it outright.
self.logger.increment('unauthorized')
try:
vrs, realm, rest = split_path(env['PATH_INFO'], 2, 3,
True)
except ValueError:
realm = 'unknown'
return HTTPUnauthorized(
headers={
'Www-Authenticate': 'Swift realm="%s"' % realm
})(env, start_response)
# Because I'm not certain if I'm the definitive auth for empty
# reseller_prefixed tokens, I won't overwrite swift.authorize.
elif 'swift.authorize' not in env:
env['swift.authorize'] = self.denied_response
else:
if self.reseller_prefix:
# With a non-empty reseller_prefix, I would like to be called
# back for anonymous access to accounts I know I'm the
# definitive auth for.
try:
version, rest = split_path(env.get('PATH_INFO', ''), 1, 2,
True)
except ValueError:
version, rest = None, None
self.logger.increment('errors')
if rest and rest.startswith(self.reseller_prefix):
# Handle anonymous access to accounts I'm the definitive
# auth for.
env['swift.authorize'] = self.authorize
env['swift.clean_acl'] = clean_acl
# Not my token, not my account, I can't authorize this request,
# deny all is a good idea if not already set...
elif 'swift.authorize' not in env:
env['swift.authorize'] = self.denied_response
# Because I'm not certain if I'm the definitive auth for empty
# reseller_prefixed accounts, I won't overwrite swift.authorize.
elif 'swift.authorize' not in env:
env['swift.authorize'] = self.authorize
env['swift.clean_acl'] = clean_acl
return self.app(env, start_response)