def __call__(self, env, start_response):
"""
这是 proxy server 的入口，上层 middleware 将参数传到这里进入 proxy server 的处理流程。
+ 最终返回的时候调用的是 swift.common.swob.Response。
+ 在 Response 里会调用 start_response 函数（返回 head），并返回（body）。
WSGI entry point.
Wraps env in swob.Request object and passes it down.
:param env: WSGI environment dictionary
:param start_response: WSGI callable
"""
try:
if self.memcache is None:
self.memcache = cache_from_env(env, True)
req = self.update_request(Request(env))
# 下列返回的值中，self.handle_request(req)为一个Response类的实例。
# Response类包含方法 __call__(evn,start_response)
return self.handle_request(req)(env, start_response)
except UnicodeError:
err = HTTPPreconditionFailed(
request=req, body='Invalid UTF8 or contains NULL')
return err(env, start_response)
except (Exception, Timeout):
start_response('500 Server Error',
[('Content-Type', 'text/plain')])
return ['Internal server error.\n']
def __init__(self, path, req):
# Database defination
#self.conn = MySQLdb.connect(host="127.0.0.1", user="******", passwd='root', db="auth", charset="utf8")
#self.cur = self.conn.cursor()
self.mc_meta = cache_from_env(req.environ)
obj = self.get_conobj_from_path(path)
#obj = self.get_objname_from_path(path)
if self.mc_meta.get(path):
self.object_id, self.object_name, self.parent_secl_id, self.seclevel, self.path, self.response = self.mc_meta.get(path).split(',')
else:
meta = self.get_metadata_from_objname(obj)
if meta:
self.object_id = meta['object_id']
self.object_name = meta['object_name'].encode("utf8")
self.parent_secl_id = meta['parent_secl_id']
self.seclevel = meta['obj_seclevel']
self.author = meta['author'].encode("utf8") if meta['author'] else None
self.path = meta['path'].encode("utf8")
self.subject = meta['subject'].encode("utf8") if meta['subject'] else None
self.description = meta['description'].encode("utf8") if meta['description'] else None
self.source = meta['source'].encode("utf8") if meta['source'] else None
self.response = 'True'
self.mc_meta.set(self.path, (('%s,%s,%s,%s,%s,%s') % (self.object_id, self.object_name, self.parent_secl_id, self.seclevel, self.path, self.response)))
elif not self.response:
self.response = ['Forbidden']
def __call__(self, env, start_response):
"""
WSGI entry point.
Wraps env in swob.Request object and passes it down.
:param env: WSGI environment dictionary
:param start_response: WSGI callable
"""
try:
if self.memcache is None:
self.memcache = cache_from_env(env)
# Remove any x-backend-* headers since those are reserved for use
# by backends communicating with each other; no end user should be
# able to send those into the cluster.
for key in list(k for k in env if k.startswith('HTTP_X_BACKEND_')):
del env[key]
req = self.update_request(Request(env))
return self.handle_request(req)(env, start_response)
except UnicodeError:
err = HTTPPreconditionFailed(
request=req, body='Invalid UTF8 or contains NULL')
return err(env, start_response)
except (Exception, Timeout):
start_response('500 Server Error',
[('Content-Type', 'text/plain')])
return ['Internal server error.\n']
def __call__(self, env, start_response):
"""
WSGI entry point.
Wraps env in swob.Request object and passes it down.
:param env: WSGI environment dictionary
:param start_response: WSGI callable
"""
try:
if self.memcache is None:
self.memcache = cache_from_env(env, True)
#更新headers的x-auth-token部分
req = self.update_request(Request(env))
#根据path的不同请求返回不同的controller
#最终会调用swob.Response.__call__方法处理这个wsig
return self.handle_request(req)(env, start_response)
except UnicodeError:
err = HTTPPreconditionFailed(
request=req, body='Invalid UTF8 or contains NULL')
return err(env, start_response)
except (Exception, Timeout):
start_response('500 Server Error',
[('Content-Type', 'text/plain')])
return ['Internal server error.\n']
def __init__(self, request, conf, app, logger):
super(VertigoProxyHandler, self).__init__(request, conf, app, logger)
self.mc_container = self.conf["mc_container"]
self.memcache = None
self.request.headers['mc-enabled'] = True
self.memcache = cache_from_env(self.request.environ)
def __call__(self, env, start_response):
"""
WSGI entry point.
Wraps env in swob.Request object and passes it down.
:param env: WSGI environment dictionary
:param start_response: WSGI callable
"""
#try:
# if self.memcache is None:
# self.memcache = cache_from_env(env, True)
# req = self.update_request(Request(env))
# if req.headers['Stop']:
# res = HTTPOk(request=req,body='')
# return res(env, start_response)
#except:
# pass
try:
if self.memcache is None:
self.memcache = cache_from_env(env, True)
req = self.update_request(Request(env))
return self.handle_request(req)(env, start_response)
except UnicodeError:
err = HTTPPreconditionFailed(
request=req, body='Invalid UTF8 or contains NULL')
return err(env, start_response)
except (Exception, Timeout):
start_response('500 Server Error',
[('Content-Type', 'text/plain')])
return ['Internal server error.\n']
def get_container_info(env, app, swift_source=None):
"""
Get the info structure for a container, based on env and app.
This is useful to middlewares.
"""
cache = cache_from_env(env)
if not cache:
return None
(version, account, container, obj) = \
split_path(env['PATH_INFO'], 2, 4, True)
cache_key = get_container_memcache_key(account, container)
# Use a unique environment cache key per container. If you copy this env
# to make a new request, it won't accidentally reuse the old container info
env_key = 'swift.%s' % cache_key
if env_key not in env:
container_info = cache.get(cache_key)
if not container_info:
resp = make_pre_authed_request(
env, 'HEAD', '/%s/%s/%s' % (version, account, container),
swift_source=swift_source,
).get_response(app)
container_info = headers_to_container_info(
resp.headers, resp.status_int)
env[env_key] = container_info
return env[env_key]
def _get_container_info(self, env):
"""
Retrieves x-container-meta-web-index, x-container-meta-web-error,
x-container-meta-web-listings, and x-container-meta-web-listings-css
from memcache or from the cluster and stores the result in memcache and
in self._index, self._error, self._listings, and self._listings_css.
:param env: The WSGI environment dict.
"""
self._index = self._error = self._listings = self._listings_css = None
memcache_client = cache_from_env(env)
if memcache_client:
memcache_key = "/staticweb/%s/%s/%s" % (self.version, self.account, self.container)
cached_data = memcache_client.get(memcache_key)
if cached_data:
(self._index, self._error, self._listings, self._listings_css) = cached_data
return
resp = make_pre_authed_request(
env, "HEAD", "/%s/%s/%s" % (self.version, self.account, self.container), agent=self.agent
).get_response(self.app)
if is_success(resp.status_int):
self._index = resp.headers.get("x-container-meta-web-index", "").strip()
self._error = resp.headers.get("x-container-meta-web-error", "").strip()
self._listings = resp.headers.get("x-container-meta-web-listings", "").strip()
self._listings_css = resp.headers.get("x-container-meta-web-listings-css", "").strip()
if memcache_client:
memcache_client.set(
memcache_key,
(self._index, self._error, self._listings, self._listings_css),
timeout=self.cache_timeout,
)
def __call__(self, env, start_response):
if self.enable_caching:
self.memcache = cache_from_env(env)
else:
self.memcache = None
start_time = time.time()
req = Request(env)
self.logger.txn_id = req.headers.get('x-trans-id', None)
if not check_utf8(req.path_info):
res = HTTPPreconditionFailed(body='Invalid UTF8')
else:
try:
if hasattr(self, req.method):
res = getattr(self, req.method)(req)
else:
res = HTTPMethodNotAllowed()
except Exception:
self.logger.exception(_('ERROR __call__ error with %(method)s'
' %(path)s '), {'method': req.method, 'path': req.path})
res = HTTPInternalServerError(body=traceback.format_exc())
trans_time = '%.4f' % (time.time() - start_time)
log_message = '%s - - [%s] "%s %s" %s %s "%s" "%s" "%s" %s' % (
req.remote_addr,
time.strftime('%d/%b/%Y:%H:%M:%S +0000',
time.gmtime()),
req.method, req.path,
res.status.split()[0], res.content_length or '-',
req.headers.get('x-trans-id', '-'),
req.referer or '-', req.user_agent or '-',
trans_time)
if req.method.upper() == 'REPLICATE':
self.logger.debug(log_message)
else:
self.logger.info(log_message)
return res(env, start_response)
def __call__(self, env, start_response):
"""
WSGI entry point.
Wraps env in swob.Request object and passes it down.
WSGI的入口点.
一个server在PasteDeplot中, 就是一个App, 并调用类的__call__方法.
:param env: WSGI environment dictionary
:param start_response: WSGI callable
"""
try:
if self.memcache is None:
self.memcache = cache_from_env(env)
req = self.update_request(Request(env))
# 开始处理请求
return self.handle_request(req)(env, start_response)
except UnicodeError:
err = HTTPPreconditionFailed(
request=req, body='Invalid UTF8 or contains NULL')
return err(env, start_response)
except (Exception, Timeout):
start_response('500 Server Error',
[('Content-Type', 'text/plain')])
return ['Internal server error.\n']
def __call__(self, env, start_response):
"""
Main hook into the WSGI paste.deploy filter/app pipeline.
:param env: The WSGI environment dict.
:param start_response: The WSGI start_response hook.
"""
env['staticweb.start_time'] = time.time()
try:
(self.version, self.account, self.container, self.obj) = \
split_path(env['PATH_INFO'], 2, 4, True)
except ValueError:
return self.app(env, start_response)
memcache_client = cache_from_env(env)
if memcache_client:
if env['REQUEST_METHOD'] in ('PUT', 'POST'):
if not self.obj and self.container:
memcache_key = '/staticweb/%s/%s/%s' % \
(self.version, self.account, self.container)
memcache_client.delete(memcache_key)
return self.app(env, start_response)
if (env['REQUEST_METHOD'] not in ('HEAD', 'GET') or
(env.get('REMOTE_USER') and
env.get('HTTP_X_WEB_MODE', 'f').lower() not in TRUE_VALUES) or
(not env.get('REMOTE_USER') and
env.get('HTTP_X_WEB_MODE', 't').lower() not in TRUE_VALUES)):
return self.app(env, start_response)
if self.obj:
return self._handle_object(env, start_response)
elif self.container:
return self._handle_container(env, start_response)
return self.app(env, start_response)
def _get_details(self, req, access_key):
"""Get access key details.
:return: (secret_key, account) as tuple or (None, None) if not found.
"""
memcache_client = cache_from_env(req.environ)
if memcache_client:
memcache_key = MEMCACHE_KEY_FORMAT % (self.reseller_prefix,
access_key)
data = memcache_client.get(memcache_key)
if data:
return data[0], data[1]
path = quote(self.akd_container_url + access_key)
resp = make_pre_authed_request(req.environ, 'GET',
path).get_response(self.app)
if resp.status_int // 100 == 2:
data = json.loads(resp.body)
secret_key, account = data['secret_key'], data['account']
if memcache_client:
memcache_client.set(memcache_key, (secret_key, account),
time=self.cache_time)
return secret_key, account
elif resp.status_int // 100 == 4:
return None, None
else:
raise Exception('Could not GET access key details: {} {}'.format(
path, resp.status_int))
def get_account_info(env, app, swift_source=None):
"""
Get the info structure for an account, based on env and app.
This is useful to middlewares.
Note: This call bypasses auth. Success does not imply that the
request has authorization to the account_info.
"""
cache = cache_from_env(env)
if not cache:
return None
(version, account, container, _) = \
split_path(env['PATH_INFO'], 2, 4, True)
cache_key = get_account_memcache_key(account)
# Use a unique environment cache key per account. If you copy this env
# to make a new request, it won't accidentally reuse the old account info
env_key = 'swift.%s' % cache_key
if env_key not in env:
account_info = cache.get(cache_key)
if not account_info:
resp = make_pre_authed_request(
env,
'HEAD',
'/%s/%s' % (version, account),
swift_source=swift_source,
).get_response(app)
account_info = headers_to_account_info(resp.headers,
resp.status_int)
env[env_key] = account_info
return env[env_key]
def __call__(self, env, start_response):
"""
Main hook into the WSGI paste.deploy filter/app pipeline.
:param env: The WSGI environment dict.
:param start_response: The WSGI start_response hook.
"""
env['staticweb.start_time'] = time.time()
try:
(self.version, self.account, self.container, self.obj) = \
split_path(env['PATH_INFO'], 2, 4, True)
except ValueError:
return self.app(env, start_response)
memcache_client = cache_from_env(env)
if memcache_client:
if env['REQUEST_METHOD'] in ('PUT', 'POST'):
if not self.obj and self.container:
memcache_key = '/staticweb/%s/%s/%s' % \
(self.version, self.account, self.container)
memcache_client.delete(memcache_key)
return self.app(env, start_response)
if (env['REQUEST_METHOD'] not in ('HEAD', 'GET') or
(env.get('REMOTE_USER')
and env.get('HTTP_X_WEB_MODE', 'f').lower() not in TRUE_VALUES) or
(not env.get('REMOTE_USER')
and env.get('HTTP_X_WEB_MODE', 't').lower() not in TRUE_VALUES)):
return self.app(env, start_response)
if self.obj:
return self._handle_object(env, start_response)
elif self.container:
return self._handle_container(env, start_response)
return self.app(env, start_response)
def __call__(self, env, start_response):
"""
Main hook into the WSGI paste.deploy filter/app pipeline.
:param env: The WSGI environment dict.
:param start_response: The WSGI start_response hook.
"""
env['staticweb.start_time'] = time.time()
try:
(version, account, container, obj) = \
split_path(env['PATH_INFO'], 2, 4, True)
except ValueError:
return self.app(env, start_response)
if env['REQUEST_METHOD'] in ('PUT', 'POST') and container and not obj:
memcache_client = cache_from_env(env)
if memcache_client:
memcache_key = \
'/staticweb/%s/%s/%s' % (version, account, container)
memcache_client.delete(memcache_key)
return self.app(env, start_response)
if env['REQUEST_METHOD'] not in ('HEAD', 'GET'):
return self.app(env, start_response)
if env.get('REMOTE_USER') and \
not config_true_value(env.get('HTTP_X_WEB_MODE', 'f')):
return self.app(env, start_response)
if not container:
return self.app(env, start_response)
context = _StaticWebContext(self, version, account, container, obj)
if obj:
return context.handle_object(env, start_response)
return context.handle_container(env, start_response)
def __call__(self, env, start_response):
"""
WSGI entry point.
Wraps env in swob.Request object and passes it down.
:param env: WSGI environment dictionary
:param start_response: WSGI callable
"""
req = Request(env)
if self.memcache_client is None:
# 在pipline中memcache middleware应该在前面
# memcache为每一个请求建立一个memcache client，并把这个client放入env中
self.memcache_client = cache_from_env(env)
if not self.memcache_client:
self.logger.warning(
_('Warning: Cannot ratelimit without a memcached client'))
return self.app(env, start_response)
try:
version, account, container, obj = req.split_path(1, 4, True)
except ValueError:
return self.app(env, start_response)
ratelimit_resp = self.handle_ratelimit(req, account, container, obj)
# ratelimit_resp 为 None 或者 Response 对象，Response 对象是可调用的。
# ratelimit 操作正常执行，或者不需要执行，返回 None，接着向下调用。
if ratelimit_resp is None:
return self.app(env, start_response)
else:
# ratelimit 执行失败，或者需要返回错误信息，返回 Response 对象。
# 不用向下调用，直接返回。
return ratelimit_resp(env, start_response)
def __call__(self, env, start_response):
"""
Main hook into the WSGI paste.deploy filter/app pipeline.
:param env: The WSGI environment dict.
:param start_response: The WSGI start_response hook.
"""
try:
(version, account, container, obj) = \
split_path(env['PATH_INFO'], 2, 4, True)
except ValueError:
return self.app(env, start_response)
if not self._cache:
self._cache = cache_from_env(env)
# Don't handle non-GET requests.
if env['REQUEST_METHOD'] not in ('HEAD', 'GET'):
# flush cache if we expect the container metadata being changed.
if container and not obj and self._cache:
memcache_key = 'better_static/%s/%s' % (account, container)
self._cache.delete(memcache_key)
return self.app(env, start_response)
# If non-html was explicitly requested, don't bother trying to format
# the html
params = urlparse.parse_qs(env.get('QUERY_STRING', ''))
if 'format' in params and params['format'] != ['html']:
return self.app(env, start_response)
context = Context(self, env, account, container, obj)
return context(env, start_response)
def handle_quota_object(self, env, start_response, version, account,
container, obj):
""" Handle quota of container usage and object count. """
memcache_client = cache_from_env(env)
object_size = int(env.get('CONTENT_LENGTH') or 0)
container_count, quota_level = self._get_account_meta(
env, version, account, memcache_client)
try:
container_usage_quota = self.container_usage[quota_level]
object_count_quota = self.object_count[quota_level]
except Exception:
self.logger.warn('Invalid quota_leve %s/%s quota_level[%s].' % (
account, container, quota_level))
container_usage_quota = None
object_count_quota = None
container_usage, object_count = self._get_container_meta(
env, version, account, container, memcache_client)
if container_usage_quota and container_usage >= container_usage_quota:
self.logger.notice("Container usage over quota, "
"request[PUT %s/%s/%s], container_usage[%s] "
"object_size[%s] quota[%s]" % (
account, container, obj, container_usage,
object_size, container_usage_quota))
return HTTPForbidden(body="The usage of container is over quota")(
env, start_response)
elif container_usage_quota and (container_usage + object_size >
container_usage_quota):
self.logger.notice("Container usage over quota, "
"request[PUT %s/%s/%s], container_usage[%s] "
"object_size[%s] quota[%s]" % (
account, container, obj, container_usage,
object_size, container_usage_quota))
return HTTPForbidden(body="The usage of container is over quota")(
env, start_response)
elif object_count_quota and object_count + 1 > object_count_quota:
self.logger.notice("Object count over quota, request[PUT %s/%s/%s],"
"object_count[%s] quota[%s]" % (
account, container, obj, object_count + 1,
object_count_quota))
return HTTPForbidden(body="The count of object is over quota")(
env, start_response)
elif self.precise_mode and memcache_client:
res = [None, None, None]
result_code = None
def _start_response(response_status, response_headers,
exc_info=None):
res[0] = response_status
res[1] = response_headers
res[2] = exc_info
resp = self.app(env, _start_response)
result_code = self._get_status_int(res[0])
if is_success(result_code):
memcache_client.delete(
get_container_memcache_key(account, container))
start_response(res[0], res[1], res[2])
return resp
else:
return self.app(env, start_response)
def get_account_info(env, app, swift_source=None):
"""
Get the info structure for an account, based on env and app.
This is useful to middlewares.
Note: This call bypasses auth. Success does not imply that the
request has authorization to the account_info.
"""
cache = cache_from_env(env)
if not cache:
return None
(version, account, _junk, _junk) = \
split_path(env['PATH_INFO'], 2, 4, True)
cache_key = get_account_memcache_key(account)
# Use a unique environment cache key per account. If you copy this env
# to make a new request, it won't accidentally reuse the old account info
env_key = 'swift.%s' % cache_key
if env_key not in env:
account_info = cache.get(cache_key)
if not account_info:
resp = make_pre_authed_request(
env, 'HEAD', '/%s/%s' % (version, account),
swift_source=swift_source,
).get_response(app)
account_info = headers_to_account_info(
resp.headers, resp.status_int)
env[env_key] = account_info
return env[env_key]
def handle_delete(self, env, start_response, version, account, container,
obj):
""" Handle delete request. """
memcache_client = cache_from_env(env)
if not memcache_client:
return self.app(env, start_response)
res = [None, None, None]
result_code = None
def _start_response(response_status, response_headers, exc_info=None):
res[0] = response_status
res[1] = response_headers
res[2] = exc_info
resp = self.app(env, _start_response)
result_code = self._get_status_int(res[0])
try:
if is_success(result_code):
if obj:
memcache_client.delete(
get_container_memcache_key(account, container))
else:
memcache_client.delete(get_account_memcache_key(account))
except Exception, err:
self.logger.error(
'Error in [Quota] delete cache: %s' % (err.message))
def __call__(self, env, start_response):
"""
Main hook into the WSGI paste.deploy filter/app pipeline.
:param env: The WSGI environment dict.
:param start_response: The WSGI start_response hook.
"""
env["staticweb.start_time"] = time.time()
try:
(version, account, container, obj) = split_path(env["PATH_INFO"], 2, 4, True)
except ValueError:
return self.app(env, start_response)
if env["REQUEST_METHOD"] in ("PUT", "POST") and container and not obj:
memcache_client = cache_from_env(env)
if memcache_client:
memcache_key = "/staticweb/%s/%s/%s" % (version, account, container)
memcache_client.delete(memcache_key)
return self.app(env, start_response)
if env["REQUEST_METHOD"] not in ("HEAD", "GET"):
return self.app(env, start_response)
if env.get("REMOTE_USER") and env.get("HTTP_X_WEB_MODE", "f").lower() not in TRUE_VALUES:
return self.app(env, start_response)
if not container:
return self.app(env, start_response)
context = _StaticWebContext(self, version, account, container, obj)
if obj:
return context.handle_object(env, start_response)
return context.handle_container(env, start_response)
def __init__(self, token=None, username=None, request=None):
# Database defination
self.mc_user = cache_from_env(request.environ)
# Initialze user information by token or usernames
if token is None and username is None:
self.denied_response('token')
# TO DO: get user from db (web update user, not read from cache , temparily )
if self.mc_user.get(username) and False:
self.tu_id, self.username, self.seclevel, self.login_name, self.response = self.mc_user.get(username).split(',')
else:
if token:
info = self.get_user_info_from_token(token=token)
elif username:
info = self.get_user_info_from_db(user_name=username)
if info:
# set each info
self.tu_id = info['tu_id']
self.username = info['username']
self.login_name = info['login_name']
self.seclevel = info['seclevel']
self.email = info['email']
self.password = info['password']
self.mobile = info['mobile']
self.response = 'True'
self.mc_user.set(self.username, (('%s,%s,%s,%s,%s')%(self.tu_id, self.username, self.seclevel,self.login_name,self.response)))
elif not self.response:
self.response = ['Forbidden']
def __call__(self, env, start_response):
"""
WSGI entry point.
Wraps env in swob.Request object and passes it down.
:param env: WSGI environment dictionary
:param start_response: WSGI callable
"""
try:
if self.memcache is None:
self.memcache = cache_from_env(env)
# Remove any x-backend-* headers since those are reserved for use
# by backends communicating with each other; no end user should be
# able to send those into the cluster.
for key in list(k for k in env if k.startswith('HTTP_X_BACKEND_')):
del env[key]
req = self.update_request(Request(env))
return self.handle_request(req)(env, start_response)
except UnicodeError:
err = HTTPPreconditionFailed(
request=req, body='Invalid UTF8 or contains NULL')
return err(env, start_response)
except (Exception, Timeout):
start_response('500 Server Error',
[('Content-Type', 'text/plain')])
return ['Internal server error.\n']
def __call__(self, env, start_response):
"""
WSGI entry point.
Wraps env in swob.Request object and passes it down.
:param env: WSGI environment dictionary
:param start_response: WSGI callable
"""
print '**** in __call__ of proxy server start *********'
print 'env',env
print 'start_response',start_response
try:
if self.memcache is None:
self.memcache = cache_from_env(env, True)
req = self.update_request(Request(env))
print '**** in __call__ of proxy server end by returning self.handle_request(req)(env, start_response) *********'
return self.handle_request(req)(env, start_response)
except UnicodeError:
err = HTTPPreconditionFailed(
request=req, body='Invalid UTF8 or contains NULL')
return err(env, start_response)
except (Exception, Timeout):
start_response('500 Server Error',
[('Content-Type', 'text/plain')])
return ['Internal server error.\n']
def __call__(self, env, start_response):
"""
WSGI entry point.
Wraps env in swob.Request object and passes it down.
:param env: WSGI environment dictionary
:param start_response: WSGI callable
"""
req = Request(env)
if self.memcache_client is None:
self.memcache_client = cache_from_env(env)
if not self.memcache_client:
self.logger.warning(
_('Warning: Cannot ratelimit without a memcached client'))
return self.app(env, start_response)
try:
version, account, container, obj = req.split_path(1, 4, True)
except ValueError:
return self.app(env, start_response)
if not valid_api_version(version):
return self.app(env, start_response)
ratelimit_resp = self.handle_ratelimit(req, account, container, obj)
if ratelimit_resp is None:
return self.app(env, start_response)
else:
return ratelimit_resp(env, start_response)
def __call__(self, env, start_response):
"""
WSGI entry point.
Wraps env in swob.Request object and passes it down.
:param env: WSGI environment dictionary
:param start_response: WSGI callable
"""
try:
if self.memcache is None:
self.memcache = cache_from_env(env)
req = self.update_request(Request(env))
return self.handle_request(req)(env, start_response)
except UnicodeError:
err = HTTPPreconditionFailed(
request=req, body='Invalid UTF8 or contains NULL')
return err(env, start_response)
except (Exception, Timeout):
# P3
fp = open("/tmp/dump","a")
fp.write("====== status 500 in __call__\n")
fp.write(traceback.format_exc())
fp.write("====== done\n")
fp.close()
start_response('500 Server Error',
[('Content-Type', 'text/plain')])
return ['Internal server error.\n']
def __call__(self, env, start_response):
if not self.storage_domain:
return self.app(env, start_response)
given_domain = env['HTTP_HOST']
port = ''
if ':' in given_domain:
given_domain, port = given_domain.rsplit(':', 1)
if given_domain == self.storage_domain[1:]: # strip initial '.'
return self.app(env, start_response)
a_domain = given_domain
if not a_domain.endswith(self.storage_domain):
if self.memcache is None:
self.memcache = cache_from_env(env)
error = True
for tries in xrange(self.lookup_depth):
found_domain = None
if self.memcache:
memcache_key = ''.join(['cname-', a_domain])
found_domain = self.memcache.get(memcache_key)
if not found_domain:
ttl, found_domain = lookup_cname(a_domain)
if self.memcache:
memcache_key = ''.join(['cname-', given_domain])
self.memcache.set(memcache_key, found_domain,
timeout=ttl)
if found_domain is None or found_domain == a_domain:
# no CNAME records or we're at the last lookup
error = True
found_domain = None
break
elif found_domain.endswith(self.storage_domain):
# Found it!
self.logger.info(
_('Mapped %(given_domain)s to %(found_domain)s') %
{'given_domain': given_domain,
'found_domain': found_domain})
if port:
env['HTTP_HOST'] = ':'.join([found_domain, port])
else:
env['HTTP_HOST'] = found_domain
error = False
break
else:
# try one more deep in the chain
self.logger.debug(
_('Following CNAME chain for '
'%(given_domain)s to %(found_domain)s') %
{'given_domain': given_domain,
'found_domain': found_domain})
a_domain = found_domain
if error:
if found_domain:
msg = 'CNAME lookup failed after %d tries' % \
self.lookup_depth
else:
msg = 'CNAME lookup failed to resolve to a valid domain'
resp = HTTPBadRequest(request=Request(env), body=msg,
content_type='text/plain')
return resp(env, start_response)
return self.app(env, start_response)
def __init__(self, token=None, username=None, request=None):
# Database defination
self.conn = MySQLdb.connect(host="127.0.0.1", user="******", passwd="root", db="auth", charset="utf8")
self.cur = self.conn.cursor()
self.mc_user = cache_from_env(request.environ)
# Initialze user information by token or usernames
if token is None and username is None:
self.denied_response("token")
# TO DO: get user from db (web update user, not read from cache , temparily )
if self.mc_user.get(username) and False:
self.tu_id, self.username, self.seclevel, self.login_name, self.response = self.mc_user.get(username).split(
","
)
else:
if token:
info = self.get_user_info_from_token(token=token)
elif username:
info = self.get_user_info_from_db(user_name=username)
if info:
# set each info
self.tu_id = info["tu_id"]
self.username = info["username"]
self.login_name = info["login_name"]
self.seclevel = info["seclevel"]
self.email = info["email"]
self.password = info["password"]
self.mobile = info["mobile"]
self.response = "True"
self.mc_user.set(
self.username,
(("%s,%s,%s,%s,%s") % (self.tu_id, self.username, self.seclevel, self.login_name, self.response)),
)
elif not self.response:
self.response = ["Forbidden"]
def __call__(self, env, start_response):
if not self.storage_domain:
return self.app(env, start_response)
given_domain = env['HTTP_HOST']
port = ''
if ':' in given_domain:
given_domain, port = given_domain.rsplit(':', 1)
if given_domain == self.storage_domain[1:]: # strip initial '.'
return self.app(env, start_response)
a_domain = given_domain
if not a_domain.endswith(self.storage_domain):
if self.memcache is None:
self.memcache = cache_from_env(env)
error = True
for tries in xrange(self.lookup_depth):
found_domain = None
if self.memcache:
memcache_key = ''.join(['cname-', a_domain])
found_domain = self.memcache.get(memcache_key)
if not found_domain:
ttl, found_domain = lookup_cname(a_domain)
if self.memcache:
memcache_key = ''.join(['cname-', given_domain])
self.memcache.set(memcache_key, found_domain,
timeout=ttl)
if found_domain is None or found_domain == a_domain:
# no CNAME records or we're at the last lookup
error = True
found_domain = None
break
elif found_domain.endswith(self.storage_domain):
# Found it!
self.logger.info(
_('Mapped %(given_domain)s to %(found_domain)s') %
{'given_domain': given_domain,
'found_domain': found_domain})
if port:
env['HTTP_HOST'] = ':'.join([found_domain, port])
else:
env['HTTP_HOST'] = found_domain
error = False
break
else:
# try one more deep in the chain
self.logger.debug(_('Following CNAME chain for ' \
'%(given_domain)s to %(found_domain)s') %
{'given_domain': given_domain,
'found_domain': found_domain})
a_domain = found_domain
if error:
if found_domain:
msg = 'CNAME lookup failed after %d tries' % \
self.lookup_depth
else:
msg = 'CNAME lookup failed to resolve to a valid domain'
resp = HTTPBadRequest(request=Request(env), body=msg,
content_type='text/plain')
return resp(env, start_response)
return self.app(env, start_response)
def __call__(self, env, start_response):
if not self.storage_domain:
return self.app(env, start_response)
if "HTTP_HOST" in env:
given_domain = env["HTTP_HOST"]
else:
given_domain = env["SERVER_NAME"]
port = ""
if ":" in given_domain:
given_domain, port = given_domain.rsplit(":", 1)
if is_ip(given_domain):
return self.app(env, start_response)
a_domain = given_domain
if not self._domain_endswith_in_storage_domain(a_domain):
if self.memcache is None:
self.memcache = cache_from_env(env)
error = True
for tries in xrange(self.lookup_depth):
found_domain = None
if self.memcache:
memcache_key = "".join(["cname-", a_domain])
found_domain = self.memcache.get(memcache_key)
if not found_domain:
ttl, found_domain = lookup_cname(a_domain)
if self.memcache:
memcache_key = "".join(["cname-", given_domain])
self.memcache.set(memcache_key, found_domain, time=ttl)
if found_domain is None or found_domain == a_domain:
# no CNAME records or we're at the last lookup
error = True
found_domain = None
break
elif self._domain_endswith_in_storage_domain(found_domain):
# Found it!
self.logger.info(
_("Mapped %(given_domain)s to %(found_domain)s")
% {"given_domain": given_domain, "found_domain": found_domain}
)
if port:
env["HTTP_HOST"] = ":".join([found_domain, port])
else:
env["HTTP_HOST"] = found_domain
error = False
break
else:
# try one more deep in the chain
self.logger.debug(
_("Following CNAME chain for " "%(given_domain)s to %(found_domain)s")
% {"given_domain": given_domain, "found_domain": found_domain}
)
a_domain = found_domain
if error:
if found_domain:
msg = "CNAME lookup failed after %d tries" % self.lookup_depth
else:
msg = "CNAME lookup failed to resolve to a valid domain"
resp = HTTPBadRequest(request=Request(env), body=msg, content_type="text/plain")
return resp(env, start_response)
return self.app(env, start_response)
def __init__(self, request, conf, app, logger):
super(VertigoProxyHandler, self).__init__(
request, conf, app, logger)
self.mc_container = self.conf["mc_container"]
self.memcache = None
self.request.headers['mc-enabled'] = True
self.memcache = cache_from_env(self.request.environ)
def __call__(self, environ, start_response):
self.logger.debug('Initialise keystone middleware')
req = Request(environ)
token = environ.get('HTTP_X_AUTH_TOKEN',
environ.get('HTTP_X_STORAGE_TOKEN'))
if not token:
self.logger.debug('No token: exiting')
environ['swift.authorize'] = self.denied_response
return self.app(environ, start_response)
self.logger.debug('Got token: %s' % (token))
identity = None
memcache_client = cache_from_env(environ)
memcache_key = 'tokens/%s' % (token)
candidate_cache = memcache_client.get(memcache_key)
if candidate_cache:
expires, _identity = candidate_cache
if expires > time():
self.logger.debug('getting identity info from memcache')
identity = _identity
if not identity:
if environ.get('HTTP_AUTHORIZATION'):
identity = self._keystone_validate_s3token(req, token)
if 'tenant' in identity:
print identity
environ['PATH_INFO'] = '/v1/AUTH_%s' % \
identity['tenant'][0]
else:
identity = self._keystone_validate_token(token)
if identity and memcache_client:
expires = identity['expires']
if expires:
memcache_client.set(memcache_key,
(expires, identity),
timeout=expires - time())
ts = str(datetime.fromtimestamp(expires))
self.logger.debug('setting memcache expiration to %s' % ts)
else: # if we didn't get identity it means there was an error.
return HTTPBadRequest(request=req)
self.logger.debug("Using identity: %r" % (identity))
if not identity:
#TODO: non authenticated access allow via refer
environ['swift.authorize'] = self.denied_response
return self.app(environ, start_response)
self.logger.debug("Using identity: %r" % (identity))
environ['keystone.identity'] = identity
environ['REMOTE_USER'] = identity.get('tenant')
environ['swift.authorize'] = self.authorize
environ['swift.clean_acl'] = clean_acl
return self.app(environ, start_response)
def _set_hash_data(self, env, cdn_obj_path, new_hash_data,
update_listings=True):
"""
Actually sets the data in the .origin account. If not successful on
any of the several updates this has to do, will raise a OriginDbFailure
"""
cdn_obj_data = new_hash_data.get_json_str()
cdn_obj_etag = md5(cdn_obj_data).hexdigest()
# this is always a PUT because a POST needs to update the file
cdn_obj_resp = make_pre_authed_request(
env, 'PUT', cdn_obj_path, body=cdn_obj_data,
headers={'Etag': cdn_obj_etag},
agent='SwiftOrigin', swift_source='SOS').get_response(self.app)
if cdn_obj_resp.status_int // 100 != 2:
raise OriginDbFailure(
'Could not PUT .hash obj in origin '
'db: %s %s' % (cdn_obj_path, cdn_obj_resp.status_int))
memcache_client = utils.cache_from_env(env)
if memcache_client:
memcache_key = self.cdn_data_memcache_key(cdn_obj_path)
memcache_client.delete(memcache_key)
if not update_listings:
return
listing_cont_path = quote('/v1/%s/%s' % (self.origin_account,
new_hash_data.account))
resp = make_pre_authed_request(
env, 'HEAD', listing_cont_path,
agent='SwiftOrigin', swift_source='SOS').get_response(self.app)
if resp.status_int == 404:
# create new container for listings
resp = make_pre_authed_request(
env, 'PUT', listing_cont_path,
agent='SwiftOrigin', swift_source='SOS').get_response(self.app)
if resp.status_int // 100 != 2:
raise OriginDbFailure(
'Could not create listing container '
'in origin db: %s %s' % (listing_cont_path, resp.status))
cdn_list_path = quote('/v1/%s/%s/%s' % (
self.origin_account, new_hash_data.account.encode('utf-8'),
new_hash_data.container.encode('utf-8')))
listing_content_type = new_hash_data.gen_listing_content_type()
cdn_list_resp = make_pre_authed_request(
env, 'PUT', cdn_list_path,
headers={'Content-Type': listing_content_type,
'Content-Length': 0},
agent='SwiftOrigin', swift_source='SOS').get_response(self.app)
if cdn_list_resp.status_int // 100 != 2:
raise OriginDbFailure(
'Could not PUT/POST to cdn listing in '
'origin db: %s %s' % (cdn_list_path, cdn_list_resp.status_int))
def origin_db_delete(self, env, req):
"""
Handles DELETEs in the Origin database.
This is not really a delete- it will remove the object from the
container listing and set cdn_enabled=false and a deleted flag in the
.hash_* obj that says that the obj is deleted. This way the container
won't show up in the listings, HEAD to the object will return 404s but
behind the scenes lookups to the object will be able to determine
the account and container from a container_hash.
"""
try:
vsn, account, container = split_path(req.path, 3, 3)
except ValueError:
return HTTPBadRequest(
'Invalid request. '
'URI format: /<api version>/<account>/<container>')
if self.extra_header_for_deletes and not req.headers.get(
self.extra_header_for_deletes, 'f').lower() in TRUE_VALUES:
# only do delete if header is set (assuming you want the header)
return HTTPMethodNotAllowed(request=req)
hsh = self.hash_path(account, container)
cdn_obj_path = self.get_hsh_obj_path(hsh)
# Remove memcache entry
memcache_client = utils.cache_from_env(env)
if memcache_client:
memcache_key = self.cdn_data_memcache_key(cdn_obj_path)
memcache_client.delete(memcache_key)
ref_hash_data = HashData(account,
container,
self.default_ttl,
False,
False,
deleted=True)
self._set_hash_data(env,
cdn_obj_path,
ref_hash_data,
update_listings=False)
cdn_list_path = quote('/v1/%s/%s/%s' %
(self.origin_account, account, container))
list_resp = make_pre_authed_request(env,
'DELETE',
cdn_list_path,
agent='SwiftOrigin',
swift_source='SOS').get_response(
self.app)
if list_resp.status_int // 100 != 2 and list_resp.status_int != 404:
raise OriginDbFailure('Could not DELETE listing path in origin '
'db: %s %s' %
(cdn_list_path, list_resp.status_int))
# Return 404 if container didn't exist
if list_resp.status_int == 404:
return HTTPNotFound(request=req)
return HTTPNoContent(request=req)
def __call__(self, environ, start_response):
self.logger.debug('Initialise keystone middleware')
req = Request(environ)
token = environ.get('HTTP_X_AUTH_TOKEN',
environ.get('HTTP_X_STORAGE_TOKEN'))
if not token:
self.logger.debug('No token: exiting')
environ['swift.authorize'] = self.denied_response
return self.app(environ, start_response)
self.logger.debug('Got token: %s' % (token))
identity = None
memcache_client = cache_from_env(environ)
memcache_key = 'tokens/%s' % (token)
candidate_cache = memcache_client.get(memcache_key)
if candidate_cache:
expires, _identity = candidate_cache
if expires > time():
self.logger.debug('getting identity info from memcache')
identity = _identity
if not identity:
if environ.get('HTTP_AUTHORIZATION'):
identity = self._keystone_validate_s3token(req, token)
if 'tenant' in identity:
print identity
environ['PATH_INFO'] = '/v1/AUTH_%s' % \
identity['tenant'][0]
else:
identity = self._keystone_validate_token(token)
if identity and memcache_client:
expires = identity['expires']
if expires:
memcache_client.set(memcache_key, (expires, identity),
timeout=expires - time())
ts = str(datetime.fromtimestamp(expires))
self.logger.debug('setting memcache expiration to %s' % ts)
else: # if we didn't get identity it means there was an error.
return HTTPBadRequest(request=req)
self.logger.debug("Using identity: %r" % (identity))
if not identity:
#TODO: non authenticated access allow via refer
environ['swift.authorize'] = self.denied_response
return self.app(environ, start_response)
self.logger.debug("Using identity: %r" % (identity))
environ['keystone.identity'] = identity
environ['REMOTE_USER'] = identity.get('tenant')
environ['swift.authorize'] = self.authorize
environ['swift.clean_acl'] = clean_acl
return self.app(environ, start_response)
def __init__(self, request, conf, app, logger):
super(VertigoProxyHandler, self).__init__(request, conf, app, logger)
self.mc_container = self.conf["mc_container"]
self.memcache = None
self.request.headers['mc-enabled'] = True
self.memcache = cache_from_env(self.request.environ)
self.redis = redis.StrictRedis(conf['redis_host'], conf['redis_port'],
conf['redis_db'])
def _get_container_info(self, env):
"""
Retrieves x-container-meta-web-index, x-container-meta-web-error,
x-container-meta-web-listings, x-container-meta-web-listings-css,
and x-container-meta-web-directory-type from memcache or from the
cluster and stores the result in memcache and in self._index,
self._error, self._listings, self._listings_css and self._dir_type.
:param env: The WSGI environment dict.
"""
self._index = self._error = self._listings = self._listings_css = \
self._dir_type = None
memcache_client = cache_from_env(env)
if memcache_client:
cached_data = memcache_client.get(
get_memcache_key(self.version, self.account, self.container))
if cached_data:
(self._index, self._error, self._listings, self._listings_css,
self._dir_type) = cached_data
return
else:
cached_data = memcache_client.get(
get_compat_memcache_key(
self.version, self.account, self.container))
if cached_data:
(self._index, self._error, self._listings,
self._listings_css) = cached_data
self._dir_type = ''
return
resp = make_pre_authed_request(
env, 'HEAD', '/%s/%s/%s' % (
self.version, self.account, self.container),
agent=self.agent, swift_source='SW').get_response(self.app)
if is_success(resp.status_int):
self._index = \
resp.headers.get('x-container-meta-web-index', '').strip()
self._error = \
resp.headers.get('x-container-meta-web-error', '').strip()
self._listings = \
resp.headers.get('x-container-meta-web-listings', '').strip()
self._listings_css = \
resp.headers.get('x-container-meta-web-listings-css',
'').strip()
self._dir_type = \
resp.headers.get('x-container-meta-web-directory-type',
'').strip()
if memcache_client:
memcache_client.set(
get_memcache_key(
self.version, self.account, self.container),
(self._index, self._error, self._listings,
self._listings_css, self._dir_type),
time=self.cache_timeout)
def list_containers_iter(self, *args, **kwargs):
""" Returns a list of containers the user has access to """
try:
version, account = self.request.split_path(2, 2)
except ValueError:
pass
path = "/%s/%s?format=json" % (version, account)
for key in ('limit', 'marker', 'end_marker', 'prefix', 'delimiter'):
value = self.request.params.get(key)
if value:
path+= '&%s=%s' % (key, value)
if self.memcache_client is None:
self.memcache_client = cache_from_env(self.request.environ)
memcache_key = 'containerlist%s%s' % (path, str(self.groups))
containers = self.memcache_client.get(memcache_key)
if containers is not None:
return containers
req = make_pre_authed_request(self.request.environ, 'GET', path)
resp = req.get_response(self.app)
tmp_containers = json.loads(resp.body)
# No cached result? -> ratelimit request to prevent abuse
memcache_key_sleep = 'containerlist_sleep/%s' % self.account
last_request_time = self.memcache_client.get(memcache_key_sleep)
if last_request_time and len(tmp_containers) > 0:
last_request = time.time() - last_request_time
if last_request < self.min_sleep:
eventlet.sleep(self.min_sleep - last_request)
containers = []
for container in tmp_containers:
tmp_env = copy.copy(self.request.environ)
container_name = container['name'].encode("utf8")
path_info = "/%s/%s/%s" % (version, account, container_name)
tmp_env['PATH_INFO'] = path_info
container_info = get_container_info(tmp_env, self.app)
acl = (container_info.get('read_acl') or '').split(',')
if (list(set(self.groups) & set(acl))):
containers.append((container['name'],
container['count'],
container['bytes'],
0))
self.memcache_client.set(memcache_key, containers, time=60)
self.memcache_client.set(memcache_key_sleep, time.time())
return containers
def get_cdn_data(self, env, cdn_obj_path):
"""
Retrieves HashData object from memcache or by doing a GET
of the cdn_obj_path which should be what is returned from
get_hsh_obj_path. Will return None if the HashData is "deleted"
:returns: HashData object or None.
"""
memcache_client = utils.cache_from_env(env)
memcache_key = self.cdn_data_memcache_key(cdn_obj_path)
if memcache_client:
cached_cdn_data = memcache_client.get(memcache_key)
if cached_cdn_data == '404':
return None
if cached_cdn_data:
try:
hash_data = HashData.create_from_json(cached_cdn_data)
if hash_data.deleted:
return None
else:
return hash_data
except ValueError:
pass
resp = make_pre_authed_request(env,
'GET',
cdn_obj_path,
agent='SwiftOrigin',
swift_source='SOS').get_response(
self.app)
if resp.status_int // 100 == 2:
try:
if memcache_client:
memcache_client.set(memcache_key,
resp.body,
serialize=False,
time=MEMCACHE_TIMEOUT)
hash_data = HashData.create_from_json(resp.body)
if not hash_data.deleted:
return hash_data
except ValueError:
self.logger.warn('Invalid HashData json: %s' % cdn_obj_path)
if resp.status_int == 404:
if memcache_client:
# only memcache for 30 secs in case adding container to swift
memcache_client.set(memcache_key,
'404',
serialize=False,
time=CACHE_404)
return None
def get_groups(self, env, token):
"""
Get groups for the given token.
:param env: The current WSGI environment dictionary.
:param token: Token to validate and return a group string for.
:returns: None if the token is invalid or a string containing a comma
separated list of groups the authenticated user is a member
of. The first group in the list is also considered a unique
identifier for that user.
"""
groups = None
memcache_client = cache_from_env(env)
if not memcache_client:
raise Exception('Memcache required')
memcache_token_key = '%s/token/%s' % (self.reseller_prefix, token)
cached_auth_data = memcache_client.get(memcache_token_key)
if cached_auth_data:
expires, groups = cached_auth_data
if expires < time():
groups = None
if env.get('HTTP_AUTHORIZATION'):
account_user, sign = \
env['HTTP_AUTHORIZATION'].split(' ')[1].rsplit(':', 1)
account, user = account_user.split(':', 1)
h_account = md5(account).hexdigest()
h_user = md5(user).hexdigest()
h_account_user = h_account +':'+h_user
if h_account_user not in self.users:
return None
# account_id = url.rsplit('/', 1)[-1]
account_id = self.reseller_prefix + account
path = env['PATH_INFO']
env['PATH_INFO'] = path.replace(account_user, account_id, 1)
msg = base64.urlsafe_b64decode(unquote(token))
key = self.users[account_user]['key']
s = base64.encodestring(hmac.new(key, msg, sha1).digest()).strip()
if s != sign:
return None
groups = [account, account_user]
groups.extend(self.users[h_account_user]['groups'])
if '.admin' in groups:
groups.remove('.admin')
groups.append(account_id)
groups = ','.join(groups)
return groups
def origin_db_delete(self, env, req):
""" Handles DELETEs in the Origin database """
if not self.delete_enabled:
return HTTPMethodNotAllowed(request=req)
try:
vsn, account, container = split_path(req.path, 3, 3)
except ValueError:
return HTTPBadRequest(
'Invalid request. '
'URI format: /<api version>/<account>/<container>')
hsh = self.hash_path(account, container)
cdn_obj_path = self.get_hsh_obj_path(hsh)
# Remove memcache entry
memcache_client = utils.cache_from_env(env)
if memcache_client:
memcache_key = self.cdn_data_memcache_key(cdn_obj_path)
memcache_client.delete(memcache_key)
resp = make_pre_authed_request(env,
'DELETE',
cdn_obj_path,
agent='SwiftOrigin',
swift_source='SOS').get_response(
self.app)
# A 404 means it's already deleted, which is okay
if resp.status_int // 100 != 2 and resp.status_int != 404:
raise OriginDbFailure('Could not DELETE .hash obj in origin '
'db: %s %s' %
(cdn_obj_path, resp.status_int))
cdn_list_path = quote('/v1/%s/%s/%s' %
(self.origin_account, account, container))
list_resp = make_pre_authed_request(env,
'DELETE',
cdn_list_path,
agent='SwiftOrigin',
swift_source='SOS').get_response(
self.app)
if list_resp.status_int // 100 != 2 and list_resp.status_int != 404:
raise OriginDbFailure('Could not DELETE listing path in origin '
'db: %s %s' %
(cdn_list_path, list_resp.status_int))
# Return 404 if container didn't exist
if resp.status_int == 404 and list_resp.status_int == 404:
return HTTPNotFound(request=req)
return HTTPNoContent(request=req)
def get_groups(self, env, token):
"""
Get groups for the given token.
:param env: The current WSGI environment dictionary.
:param token: Token to validate and return a group string for.
:returns: None if the token is invalid or a string containing a comma
separated list of groups the authenticated user is a member
of. The first group in the list is also considered a unique
identifier for that user.
"""
groups = None
memcache_client = cache_from_env(env)
if not memcache_client:
raise Exception('Memcache required')
memcache_token_key = '%s/token/%s' % (self.reseller_prefix, token)
cached_auth_data = memcache_client.get(memcache_token_key)
if cached_auth_data:
expires, groups = cached_auth_data
info = self.users.get(groups, {}).get('url')
if not info:
account_id = self.admins[groups]['url'].rsplit('/', 1)[-1]
else:
account_id = info.rsplit('/', 1)[-1]
# account_id = self.users.get(groups).get('url').rsplit('/', 1)[-1]
groups = groups + ',' + account_id + ',admin,.admin'
# groups=sun
if expires < time():
groups = None
if env.get('HTTP_AUTHORIZATION'):
account_user, sign = \
env['HTTP_AUTHORIZATION'].split(' ')[1].rsplit(':', 1)
if account_user not in self.users and account_user not in self.admins:
return None
account, user = account_user.split(':', 1)
account_id = self.users[account_user]['url'].rsplit('/', 1)[-1]
if not account_id:
account_id = self.admins[user]['url'].rsplit('/', 1)[-1]
path = env['PATH_INFO']
env['PATH_INFO'] = path.replace(account_user, account_id, 1)
msg = base64.urlsafe_b64decode(unquote(token))
key = self.users[account_user]['key']
s = base64.encodestring(hmac.new(key, msg, sha1).digest()).strip()
if s != sign:
return None
groups = self._get_user_groups(account, account_user, account_id)
# account_id = self.users[groups]['url'].rsplit('/', 1)[-1]
# groups=groups + ',' + account_id + ',admin,.admin'
return groups
def __call__(self, env, start_response):
request = Request(env)
if request.method not in ("POST", "PUT"):
return self.app(env, start_response)
try:
(version, account) = split_path(request.path_info, 1, 2, True)
except ValueError:
return self.app(env, start_response)
reseller = self.reseller_request(env)
# deny quota set for non-resellers
val = request.headers.get('X-Account-Meta-Quota-Bytes')
if val is not None and not reseller:
return HTTPForbidden()(env, start_response)
# verify new quota headers are properly formatted
if val and not val.isdigit():
return HTTPBadRequest()(env, start_response)
#Pass early if request is from reseller
if reseller:
return self.app(env, start_response)
memcache_client = cache_from_env(env)
quota_exceeded = None
if memcache_client:
memcache_key = "quota_exceeded_%s" % (account, )
quota_exceeded = memcache_client.get(memcache_key)
if quota_exceeded is None:
quota_exceeded = False
(used_bytes, quota) = self.get_quota(env, version, account)
if 0 <= quota < used_bytes:
quota_exceeded = True
if memcache_client:
memcache_client.set(
memcache_key,
quota_exceeded,
timeout=float(self.conf.get('cache_timeout', 60)))
if quota_exceeded:
return HTTPRequestEntityTooLarge()(env, start_response)
return self.app(env, start_response)
def _accession_by_auth_token(self, env, auth_token):
"""
add by colony.
"""
req = Request(env)
memcache_client = cache_from_env(env)
#get memcache
if memcache_client:
memcache_key = 'auth/%s' % auth_token
cached_auth_data = memcache_client.get(memcache_key)
if cached_auth_data:
expires, tenant, username, roles, storage_url = cached_auth_data
if expires > time():
if not self.across_account and not self.valid_account_owner(
req, tenant):
return None
return tenant, username, roles, storage_url
token = {'auth': {'token': {'id': auth_token}, 'tenantId': ''}}
req_headers = {
'Content-type': 'application/json',
'Accept': 'text/json'
}
connect = httplib.HTTPConnection if self.auth_protocol == 'http' else httplib.HTTPSConnection
conn = connect('%s' % self.auth_netloc, timeout=10)
conn.request('POST', '/v2.0/tokens', json.dumps(token), req_headers)
resp = conn.getresponse()
if resp.status == 200:
data = resp.read()
else:
return None
auth_resp = json.loads(data)
verified_auth_token, tenant, username, roles, storage_url = \
self._get_swift_info(auth_resp, self.region_name)
if auth_token != verified_auth_token:
return None
# set memcache
if memcache_client:
memcache_client.set(memcache_key,
(time() + self.memcache_expire, tenant,
username, roles, storage_url),
timeout=self.memcache_expire)
if not self.across_account and not self.valid_account_owner(
req, tenant):
return None
return tenant, username, roles, storage_url
def get_groups(self, env, token):
"""
Get groups for the given token.
:param env: The current WSGI environment dictionary.
:param token: Token to validate and return a group string for.
:returns: None if the token is invalid or a string containing a comma
separated list of groups the authenticated user is a member
of. The first group in the list is also considered a unique
identifier for that user.
"""
groups = None
memcache_client = cache_from_env(env)
if not memcache_client:
raise Exception('Memcache required')
memcache_token_key = '%s/token/%s' % (self.reseller_prefix, token)
cached_auth_data = memcache_client.get(memcache_token_key)
if cached_auth_data:
expires, groups = cached_auth_data
if expires < time():
groups = None
s3_auth_details = env.get('swift3.auth_details')
if s3_auth_details:
account_user = s3_auth_details['access_key']
signature_from_user = s3_auth_details['signature']
if account_user not in self.users:
return None
account, user = account_user.split(':', 1)
account_id = self.users[account_user]['url'].rsplit('/', 1)[-1]
path = env['PATH_INFO']
env['PATH_INFO'] = path.replace(account_user, account_id, 1)
if 'check_signature' in s3_auth_details:
if not s3_auth_details['check_signature'](
self.users[account_user]['key']):
return None
else:
valid_signature = base64.encodestring(hmac.new(
self.users[account_user]['key'],
s3_auth_details['string_to_sign'],
sha1).digest()).strip()
if signature_from_user != valid_signature:
return None
groups = self._get_user_groups(account, account_user, account_id)
return groups
def get_groups(self, env, token):
"""
Get groups for the given token.
:param env: The current WSGI environment dictionary.
:param token: Token to validate and return a group string for.
:returns: None if the token is invalid or a string containing a comma
separated list of groups the authenticated user is a member
of. The first group in the list is also considered a unique
identifier for that user.
"""
groups = None
memcache_client = cache_from_env(env)
if not memcache_client:
raise Exception('Memcache required')
memcache_token_key = '%s/token/%s' % (self.reseller_prefix, token)
cached_auth_data = memcache_client.get(memcache_token_key)
if cached_auth_data:
expires, groups = cached_auth_data
if expires < time():
groups = None
elif six.PY2:
groups = groups.encode('utf8')
s3_auth_details = env.get('s3api.auth_details') or\
env.get('swift3.auth_details')
if s3_auth_details:
if 'check_signature' not in s3_auth_details:
self.logger.warning(
'Swift3 did not provide a check_signature function; '
'upgrade Swift3 if you want to use it with tempauth')
return None
account_user = s3_auth_details['access_key']
if account_user not in self.users:
return None
user = self.users[account_user]
account = account_user.split(':', 1)[0]
account_id = user['url'].rsplit('/', 1)[-1]
if not s3_auth_details['check_signature'](user['key']):
return None
env['PATH_INFO'] = env['PATH_INFO'].replace(
account_user, account_id, 1)
groups = self._get_user_groups(account, account_user, account_id)
return groups
def set_container_metadata(vertigo, metadata):
"""
Sets the swift metadata to the container
:param metadata: metadata dictionary
"""
memcache = cache_from_env(vertigo.request.environ)
dest_path = os.path.join('/', vertigo.api_version, vertigo.account, vertigo.container)
for key in metadata.keys():
if not key.startswith(SYSMETA_CONTAINER_HEADER):
del metadata[key]
# We store the Vertigo metadata in the memcached server (only 10 minutes)
memcache.set("vertigo_"+dest_path, metadata, time=600)
new_env = dict(vertigo.request.environ)
auth_token = vertigo.request.headers.get('X-Auth-Token')
metadata.update({'X-Auth-Token': auth_token})
sub_req = make_subrequest(new_env, 'POST', dest_path,
headers=metadata,
swift_source='Vertigo')
sub_req.get_response(vertigo.app)
def _set_details(self, req, access_key, secret_key, account):
"""Set access key details."""
path = quote(self.akd_container_url + access_key)
resp = make_pre_authed_request(
env=req.environ,
method='PUT',
path=path,
body=json.dumps({'secret_key': secret_key, 'account': account})).\
get_response(self.app)
if resp.status_int // 100 == 2:
# Remove old data from cache.
memcache_client = cache_from_env(req.environ)
if memcache_client:
memcache_key = MEMCACHE_KEY_FORMAT % (self.reseller_prefix,
access_key)
memcache_client.delete(memcache_key)
else:
raise Exception('Could not PUT access key details: {} {}'.format(
path, resp.status_int))
def _get_container_info(self, env, start_response):
"""
Retrieves x-container-meta-web-index, x-container-meta-web-error,
x-container-meta-web-listings, and x-container-meta-web-listings-css
from memcache or from the cluster and stores the result in memcache and
in self._index, self._error, self._listings, and self._listings_css.
:param env: The WSGI environment dict.
:param start_response: The WSGI start_response hook.
"""
self._index = self._error = self._listings = self._listings_css = None
memcache_client = cache_from_env(env)
if memcache_client:
memcache_key = '/staticweb/%s/%s/%s' % (self.version, self.account,
self.container)
cached_data = memcache_client.get(memcache_key)
if cached_data:
(self._index, self._error, self._listings,
self._listings_css) = cached_data
return
tmp_env = self._get_escalated_env(env)
tmp_env['REQUEST_METHOD'] = 'HEAD'
req = Request.blank('/%s/%s/%s' %
(self.version, self.account, self.container),
environ=tmp_env)
resp = req.get_response(self.app)
if resp.status_int // 100 == 2:
self._index = \
resp.headers.get('x-container-meta-web-index', '').strip()
self._error = \
resp.headers.get('x-container-meta-web-error', '').strip()
self._listings = \
resp.headers.get('x-container-meta-web-listings', '').strip()
self._listings_css = \
resp.headers.get('x-container-meta-web-listings-css',
'').strip()
if memcache_client:
memcache_client.set(memcache_key,
(self._index, self._error, self._listings,
self._listings_css),
timeout=self.cache_timeout)
def _get_container_info(self, env):
"""
Retrieves x-container-meta-web-index, x-container-meta-web-error,
x-container-meta-web-listings, and x-container-meta-web-listings-css
from memcache or from the cluster and stores the result in memcache and
in self._index, self._error, self._listings, and self._listings_css.
:param env: The WSGI environment dict.
"""
self._index = self._error = self._listings = self._listings_css = None
memcache_client = cache_from_env(env)
if memcache_client:
memcache_key = '/staticweb/%s/%s/%s' % (self.version, self.account,
self.container)
cached_data = memcache_client.get(memcache_key)
if cached_data:
(self._index, self._error, self._listings,
self._listings_css) = cached_data
return
resp = make_pre_authed_request(
env,
'HEAD',
'/%s/%s/%s' % (self.version, self.account, self.container),
agent=self.agent,
swift_source='SW').get_response(self.app)
if is_success(resp.status_int):
self._index = \
resp.headers.get('x-container-meta-web-index', '').strip()
self._error = \
resp.headers.get('x-container-meta-web-error', '').strip()
self._listings = \
resp.headers.get('x-container-meta-web-listings', '').strip()
self._listings_css = \
resp.headers.get('x-container-meta-web-listings-css',
'').strip()
if memcache_client:
memcache_client.set(memcache_key,
(self._index, self._error, self._listings,
self._listings_css),
time=self.cache_timeout)
def set_container_metadata(self, container, metadata):
"""
Sets the swift metadata to the container
:param metadata: metadata dictionary
"""
for key in metadata.keys():
if not key.startswith(MICROCONTROLLERS_CONT_HEADER):
del metadata[key]
dest_path = os.path.join('/', self.api_version, self.account, container)
# We store the micro-controller execution list in a memcached server (only 10 minutes)
memcache = cache_from_env(self.request.environ)
memcache.set("vertigo_"+dest_path, metadata, time=600)
new_env = dict(self.request.environ)
auth_token = self.request.headers.get('X-Auth-Token')
metadata.update({'X-Auth-Token': auth_token})
sub_req = make_subrequest(new_env, 'POST', dest_path,
headers=metadata,
swift_source='micro-controllers_middleware')
response = sub_req.get_response(self.app)
if response.status_int != 202:
raise
def handle_delete_access_key(self, req, access_key):
"""Delete access key.
Required headers:
- `x-s3auth-admin-key`: admin key
"""
path = quote(self.akd_container_url + access_key)
resp = make_pre_authed_request(req.environ, 'DELETE',
path).get_response(self.app)
if resp.status_int // 100 == 2:
memcache_client = cache_from_env(req.environ)
if memcache_client:
memcache_key = MEMCACHE_KEY_FORMAT % (self.reseller_prefix,
access_key)
memcache_client.delete(memcache_key)
return HTTPNoContent(request=req)
elif resp.status_int // 100 == 4:
return HTTPNotFound(request=req)
else:
raise Exception(
'Could not DELETE access key details: {} {}'.format(
path, resp.status_int))
def __call__(self, env, start_response):
"""
WSGI entry point.
Wraps env in swob.Request object and passes it down.
:param env: WSGI environment dictionary
:param start_response: WSGI callable
"""
try:
if self.memcache is None:
self.memcache = cache_from_env(env, True)
#将env包装为一个swob的请求，并传递给请求处理方法
req = self.update_request(Request(env))
#调用处理请求的方法，处理请求
return self.handle_request(req)(env, start_response)
except UnicodeError:
err = HTTPPreconditionFailed(
request=req, body='Invalid UTF8 or contains NULL')
return err(env, start_response)
except (Exception, Timeout):
start_response('500 Server Error',
[('Content-Type', 'text/plain')])
return ['Internal server error.\n']
def get_groups(self, env, token):
"""
Get groups for the given token.
:param env: The current WSGI environment dictionary.
:param token: Token to validate and return a group string for.
:returns: None if the token is invalid or a string containing a comma
separated list of groups the authenticated user is a member
of. The first group in the list is also considered a unique
identifier for that user.
"""
groups = None
memcache_client = cache_from_env(env)
if not memcache_client:
raise Exception('Memcache required')
memcache_token_key = '%s/token/%s' % (self.reseller_prefix, token)
cached_auth_data = memcache_client.get(memcache_token_key)
if cached_auth_data:
expires, groups = cached_auth_data
if expires < time():
groups = None
return groups
def __call__(self, env, start_response):
"""
WSGI entry point.
Wraps env in swob.Request object and passes it down.
:param env: WSGI environment dictionary
:param start_response: WSGI callable
"""
req = Request(env)
if self.memcache_client is None:
self.memcache_client = cache_from_env(env)
if not self.memcache_client:
self.logger.warning(
_('Warning: Cannot ratelimit without a memcached client'))
return self.app(env, start_response)
try:
version, account, container, obj = req.split_path(1, 4, True)
except ValueError:
return self.app(env, start_response)
ratelimit_resp = self.handle_ratelimit(req, account, container, obj)
if ratelimit_resp is None:
return self.app(env, start_response)
else:
return ratelimit_resp(env, start_response)
def GETorHEAD(self, req):
"""Handler for HTTP GET/HEAD requests."""
ai = self.account_info(self.account_name, req)
auto_account = self.account_name.startswith(
self.app.auto_create_account_prefix)
if not (auto_account or ai[1]):
if 'swift.authorize' in req.environ:
aresp = req.environ['swift.authorize'](req)
if aresp:
# Don't cache this. It doesn't reflect the state of the
# container, just that the user can't access it.
return aresp
# Don't cache this. The lack of account will be cached, and that
# is sufficient.
return HTTPNotFound(request=req)
# The read-modify-write of params here is because the Request.params
# getter dynamically generates a dict of params from the query string;
# the setter must be called for new params to update the query string.
params = req.params
params['format'] = 'json'
# x-backend-record-type may be sent via internal client e.g. from
# the sharder or in probe tests
record_type = req.headers.get('X-Backend-Record-Type', '').lower()
if not record_type:
record_type = 'auto'
req.headers['X-Backend-Record-Type'] = 'auto'
params['states'] = 'listing'
req.params = params
memcache = cache_from_env(req.environ, True)
if (req.method == 'GET' and record_type != 'object'
and self.app.recheck_listing_shard_ranges > 0 and memcache
and get_param(req, 'states') == 'listing'
and not config_true_value(
req.headers.get('x-backend-include-deleted', False))):
# This GET might be served from cache or might populate cache.
# 'x-backend-include-deleted' is not usually expected in requests
# to the proxy (it is used from sharder to container servers) but
# it is included in the conditions just in case because we don't
# cache deleted shard ranges.
resp = self._GET_using_cache(req)
else:
resp = self._GETorHEAD_from_backend(req)
resp_record_type = resp.headers.get('X-Backend-Record-Type', '')
if all((req.method == "GET", record_type == 'auto',
resp_record_type.lower() == 'shard')):
resp = self._get_from_shards(req, resp)
if not config_true_value(resp.headers.get('X-Backend-Cached-Results')):
# Cache container metadata. We just made a request to a storage
# node and got up-to-date information for the container.
resp.headers['X-Backend-Recheck-Container-Existence'] = str(
self.app.recheck_container_existence)
set_info_cache(self.app, req.environ, self.account_name,
self.container_name, resp)
if 'swift.authorize' in req.environ:
req.acl = resp.headers.get('x-container-read')
aresp = req.environ['swift.authorize'](req)
if aresp:
# Don't cache this. It doesn't reflect the state of the
# container, just that the user can't access it.
return aresp
if not req.environ.get('swift_owner', False):
for key in self.app.swift_owner_headers:
if key in resp.headers:
del resp.headers[key]
# Expose sharding state in reseller requests
if req.environ.get('reseller_request', False):
resp.headers['X-Container-Sharding'] = config_true_value(
resp.headers.get(
get_sys_meta_prefix('container') + 'Sharding', 'False'))
return resp
def __call__(self, env, start_response):
self.logger.debug('In cs_auth middleware')
identity = None # the identity we are trying to populate
# Handle s3 connections first because s3 has a unique format/use for the 'HTTP_X_AUTH_TOKEN'.
s3 = env.get('HTTP_AUTHORIZATION', None)
if s3 and s3.startswith('AWS'):
s3_apikey, s3_signature = s3.split(' ')[1].rsplit(':', 1)[:]
if s3_apikey and s3_signature:
# check if we have cached data to validate this request instead of hitting cloudstack.
memcache_client = cache_from_env(env)
memcache_result = memcache_client.get('cs_s3_auth/%s' %
s3_apikey)
valid_cache = False
data = None
if memcache_result and self.cs_cache_timeout > 0:
expires, data = memcache_result
if expires > time():
valid_cache = True
if valid_cache:
self.logger.debug(
'Validating the S3 request via the cached identity')
s3_token = base64.urlsafe_b64decode(
env.get('HTTP_X_AUTH_TOKEN', '')).encode("utf-8")
if s3_signature == base64.b64encode(
hmac.new(data.get('secret', ''), s3_token,
hashlib.sha1).digest()):
self.logger.debug('Using cached S3 identity')
identity = data.get('identity', None)
token = identity.get(
'token', None
) # this just simplifies the logical flow, its not really used in this case.
# The swift3 middleware sets env['PATH_INFO'] to '/v1/<aws_secret_key>', we need to map it to the cloudstack account.
if self.reseller_prefix != '':
env['PATH_INFO'] = env['PATH_INFO'].replace(
s3_apikey,
'%s_%s' % (self.reseller_prefix,
identity.get('account', '')))
else:
env['PATH_INFO'] = env['PATH_INFO'].replace(
s3_apikey,
'%s' % (identity.get('account', '')))
else: # hit cloudstack and populate memcached if valid request
user_list = self.cs_api.request(
dict({'command': 'listUsers'}))
if user_list:
for user in user_list['user']:
if user['state'] == 'enabled' and 'apikey' in user and user[
'apikey'] == s3_apikey:
# At this point we have found a matching user. Authenticate them.
s3_token = base64.urlsafe_b64decode(
env.get('HTTP_X_AUTH_TOKEN',
'')).encode("utf-8")
if s3_signature == base64.b64encode(
hmac.new(user['secretkey'], s3_token,
hashlib.sha1).digest()):
expires = time() + self.cs_cache_timeout
timeout = self.cs_cache_timeout
token = hashlib.sha224(
'%s%s' % (user['secretkey'],
user['apikey'])).hexdigest()
if self.reseller_prefix != '':
account_url = '%s/v1/%s_%s' % (
self.storage_url,
self.reseller_prefix,
quote(user['account']))
else:
account_url = '%s/v1/%s' % (
self.storage_url,
quote(user['account']))
identity = dict({
'username':
user['username'],
'account':
user['account'],
'token':
token,
'account_url':
account_url,
'domain':
dict({
'id': user['domainid'],
'name': user['domain']
}),
'roles': [
self.cs_roles[user['accounttype']],
user['account']
],
'expires':
expires
})
self.logger.debug('Creating S3 identity')
# The swift3 middleware sets env['PATH_INFO'] to '/v1/<aws_secret_key>', we need to map it to the cloudstack account.
if self.reseller_prefix != '':
env['PATH_INFO'] = env[
'PATH_INFO'].replace(
s3_apikey, '%s_%s' %
(self.reseller_prefix,
user['account']))
else:
env['PATH_INFO'] = env[
'PATH_INFO'].replace(
s3_apikey,
'%s' % (user['account']))
memcache_client = cache_from_env(env)
if memcache_client:
memcache_client.set(
'cs_s3_auth/%s' % s3_apikey,
(expires,
dict({
'secret': user['secretkey'],
'identity': identity
})),
timeout=timeout)
memcache_client.set(
'cs_token/%s' % token,
(expires, identity),
timeout=timeout)
else:
self.logger.debug(
'S3 credentials are not valid')
env['swift.authorize'] = self.denied_response
return self.app(env, start_response)
else:
self.logger.debug('Errors: %s' % self.cs_api.errors)
env['swift.authorize'] = self.denied_response
return self.app(env, start_response)
else:
self.logger.debug('Invalid credential format')
env['swift.authorize'] = self.denied_response
return self.app(env, start_response)
# If it is not an S3 call, handle the request for authenication, otherwise, use the token.
req = Request(env)
if not s3:
try:
auth_url_piece, rest_of_url = split_path(req.path_info,
minsegs=1,
maxsegs=2,
rest_with_last=True)
except ValueError:
return HTTPNotFound(request=req)
# Check if the request is for authentication (to get a token).
if auth_url_piece in ('auth', 'v1.0'): # valid auth urls
auth_user = env.get('HTTP_X_AUTH_USER', None)
auth_key = env.get('HTTP_X_AUTH_KEY', None)
if auth_user and auth_key:
# check if we have this user and key cached.
memcache_client = cache_from_env(env)
memcache_result = memcache_client.get(
'cs_auth/%s/%s' % (auth_user, auth_key))
valid_cache = False
data = None
if memcache_result and self.cs_cache_timeout > 0 and env.get(
'HTTP_X_AUTH_TTL', 1) > 0:
expires, data = memcache_result
if expires > time():
valid_cache = True
if valid_cache:
self.logger.debug('Using cached identity via creds')
identity = data
self.logger.debug("Using identity: %r" % (identity))
token = identity.get('token', None)
req.response = Response(request=req,
headers={
'x-auth-token':
token,
'x-storage-token':
token,
'x-storage-url':
identity.get(
'account_url', None)
})
return req.response(env, start_response)
else: # hit cloudstack for the details.
user_list = self.cs_api.request(
dict({
'command': 'listUsers',
'username': auth_user
}))
if user_list:
for user in user_list['user']:
if user['state'] == 'enabled' and 'apikey' in user and user[
'apikey'] == auth_key:
token = hashlib.sha224(
'%s%s' % (user['secretkey'],
user['apikey'])).hexdigest()
if env.get('HTTP_X_AUTH_TTL', None):
expires = time() + int(
env.get('HTTP_X_AUTH_TTL'))
timeout = int(
env.get('HTTP_X_AUTH_TTL'))
else:
expires = time(
) + self.cs_cache_timeout
timeout = self.cs_cache_timeout
if self.reseller_prefix != '':
account_url = '%s/v1/%s_%s' % (
self.storage_url,
self.reseller_prefix,
quote(user['account']))
else:
account_url = '%s/v1/%s' % (
self.storage_url,
quote(user['account']))
identity = dict({
'username':
user['username'],
'account':
user['account'],
'token':
token,
'account_url':
account_url,
'domain':
dict({
'id': user['domainid'],
'name': user['domain']
}),
'roles': [
self.cs_roles[user['accounttype']],
user['account']
],
'expires':
expires
})
self.logger.debug('Created identity: %s' %
identity)
# add to memcache so it can be referenced later
memcache_client = cache_from_env(env)
if memcache_client:
memcache_client.set(
'cs_auth/%s/%s' %
(auth_user, auth_key),
(expires, identity),
timeout=timeout)
memcache_client.set(
'cs_token/%s' % token,
(expires, identity),
timeout=timeout)
req.response = Response(
request=req,
headers={
'x-auth-token': token,
'x-storage-token': token,
'x-storage-url': account_url
})
return req.response(env, start_response)
# if we get here the user was not valid, so fail...
self.logger.debug('Not a valid user and key pair')
env['swift.authorize'] = self.denied_response
return self.app(env, start_response)
else:
self.logger.debug('Errors: %s' %
self.cs_api.errors)
env['swift.authorize'] = self.denied_response
return self.app(env, start_response)
else:
self.logger.debug('Credentials missing')
env['swift.authorize'] = self.denied_response
return self.app(env, start_response)
else:
token = env.get('HTTP_X_AUTH_TOKEN',
env.get('HTTP_X_STORAGE_TOKEN'))
if not identity and not env.get('HTTP_X_AUTH_TOKEN',
env.get('HTTP_X_STORAGE_TOKEN', None)):
# this is an anonymous request. pass it through for authorize to verify.
self.logger.debug('Passing through anonymous request')
env['swift.authorize'] = self.authorize
env['swift.clean_acl'] = clean_acl
return self.app(env, start_response)
# setup a memcache client for the following.
memcache_client = cache_from_env(env)
if not identity:
memcache_result = memcache_client.get('cs_token/%s' % token)
if memcache_result and self.cs_cache_timeout > 0:
expires, _identity = memcache_result
if expires > time():
self.logger.debug('Using cached identity via token')
identity = _identity
if not identity:
self.logger.debug(
"No cached identity, validate token via cloudstack")
identity = self._cloudstack_validate_token(token)
if identity and memcache_client:
expires = identity['expires']
memcache_client.set('cs_token/%s' % token, (expires, identity),
timeout=expires - time())
ts = str(datetime.fromtimestamp(expires))
self.logger.debug('Setting memcache expiration to %s' % ts)
else: # if we didn't get identity it means there was an error.
self.logger.debug('No identity for this token')
env['swift.authorize'] = self.denied_response
return self.app(env, start_response)
if not identity:
env['swift.authorize'] = self.denied_response
return self.app(env, start_response)
self.logger.debug("Using identity: %r" % (identity))
env['cloudstack.identity'] = identity
env['REMOTE_USER'] = '******'.join(identity['roles'])
env['swift.authorize'] = self.authorize
env['swift.clean_acl'] = clean_acl
return self.app(env, start_response)
def __call__(self, req):
if not self.memcache:
self.memcache = cache_from_env(req.environ)
login_path = '%s/%s' % (self.page_path, 'login')
token = None
storage_url = None
# favicon
if req.path == '/favicon.ico':
return self.pass_file(req, 'images/favicon.ico',
'image/vnd.microsoft.icon')
# not taylor
if not req.path.startswith(self.page_path):
return self.app
# image
if req.path.startswith(join(self.page_path, 'image')):
return self.pass_file(req, join('images', basename(req.path)))
# css
if req.path.startswith(join(self.page_path, 'css')):
return self.pass_file(req, join('css', basename(req.path)))
# js
if req.path.startswith(join(self.page_path, 'js')):
return self.pass_file(req, join('js', basename(req.path)))
# get token from cookie and query memcache
token = req.cookies('_token')
if self.memcache and token:
cache_val = self.memcache.get('%s_%s' % (self.title, token))
if cache_val:
self.token_bank[token] = cache_val
status = self.token_bank.get(token, None)
if status:
storage_url = status.get('url', None)
# login page
if req.path == login_path:
return self.page_login(req)
if not token or not storage_url:
return HTTPFound(location=login_path)
self.token_bank[token].update({'last': time()})
# clean up token bank
for tok, val in self.token_bank.items():
last = val.get('last', 0)
if (time() - last) >= self.cookie_max_age:
del (self.token_bank[tok])
if 'X-PJAX' in req.headers:
return self.pass_file(req, 'images/test.html', 'text/html')
# return self.page_cont_list(req, storage_url, token,
# template_name='containers.tmpl')
# return self.page_obj_list(req, storage_url, token,
# template_name='objectss.tmpl')
# ajax action
if '_ajax' in req.params_alt():
if req.params_alt()['_action'].endswith('_meta_list'):
status, headers = self.action_routine(req, storage_url, token)
return Response(status=status, body=headers)
return Response(
status=self.action_routine(req, storage_url, token))
# after action
if '_action' in req.params_alt():
if req.params_alt()['_action'] == 'logout':
del self.token_bank[token]
self.memcache.delete('%s_%s' % (self.title, token))
return HTTPFound(location=login_path)
return self.page_after_action(req, storage_url, token)
# construct main pages
return self.page_main(req, storage_url, token)
def handle_get_token(self, req):
"""
Handles the various `request for token and service end point(s)` calls.
There are various formats to support the various auth servers in the
past. Examples::
GET <auth-prefix>/v1/<act>/auth
X-Auth-User: <act>:<usr> or X-Storage-User: <usr>
X-Auth-Key: <key> or X-Storage-Pass: <key>
GET <auth-prefix>/auth
X-Auth-User: <act>:<usr> or X-Storage-User: <act>:<usr>
X-Auth-Key: <key> or X-Storage-Pass: <key>
GET <auth-prefix>/v1.0
X-Auth-User: <act>:<usr> or X-Storage-User: <act>:<usr>
X-Auth-Key: <key> or X-Storage-Pass: <key>
On successful authentication, the response will have X-Auth-Token and
X-Storage-Token set to the token to use with Swift and X-Storage-URL
set to the URL to the default Swift cluster to use.
:param req: The swob.Request to process.
:returns: swob.Response, 2xx on success with data set as explained
above.
"""
# Validate the request info
try:
pathsegs = split_path(req.path_info, 1, 3, True)
except ValueError:
self.logger.increment('errors')
return HTTPNotFound(request=req)
if pathsegs[0] == 'v1' and pathsegs[2] == 'auth':
account = pathsegs[1]
user = req.headers.get('x-storage-user')
if not user:
user = req.headers.get('x-auth-user')
if not user or ':' not in user:
self.logger.increment('token_denied')
auth = 'Swift realm="%s"' % account
return HTTPUnauthorized(request=req,
headers={'Www-Authenticate': auth})
account2, user = user.split(':', 1)
if account != account2:
self.logger.increment('token_denied')
auth = 'Swift realm="%s"' % account
return HTTPUnauthorized(request=req,
headers={'Www-Authenticate': auth})
key = req.headers.get('x-storage-pass')
if not key:
key = req.headers.get('x-auth-key')
elif pathsegs[0] in ('auth', 'v1.0'):
user = req.headers.get('x-auth-user')
if not user:
user = req.headers.get('x-storage-user')
if not user or ':' not in user:
self.logger.increment('token_denied')
auth = 'Swift realm="unknown"'
return HTTPUnauthorized(request=req,
headers={'Www-Authenticate': auth})
account, user = user.split(':', 1)
key = req.headers.get('x-auth-key')
if not key:
key = req.headers.get('x-storage-pass')
else:
return HTTPBadRequest(request=req)
if not all((account, user, key)):
self.logger.increment('token_denied')
realm = account or 'unknown'
return HTTPUnauthorized(request=req, headers={'Www-Authenticate':
'Swift realm="%s"' %
realm})
# Authenticate user
account_user = account + ':' + user
if account_user not in self.users:
self.logger.increment('token_denied')
auth = 'Swift realm="%s"' % account
return HTTPUnauthorized(request=req,
headers={'Www-Authenticate': auth})
if self.users[account_user]['key'] != key:
self.logger.increment('token_denied')
auth = 'Swift realm="unknown"'
return HTTPUnauthorized(request=req,
headers={'Www-Authenticate': auth})
account_id = self.users[account_user]['url'].rsplit('/', 1)[-1]
# Get memcache client
memcache_client = cache_from_env(req.environ)
if not memcache_client:
raise Exception('Memcache required')
# See if a token already exists and hasn't expired
token = None
memcache_user_key = '%s/user/%s' % (self.reseller_prefix, account_user)
candidate_token = memcache_client.get(memcache_user_key)
if candidate_token:
memcache_token_key = \
'%s/token/%s' % (self.reseller_prefix, candidate_token)
cached_auth_data = memcache_client.get(memcache_token_key)
if cached_auth_data:
expires, old_groups = cached_auth_data
old_groups = old_groups.split(',')
new_groups = self._get_user_groups(account, account_user,
account_id)
if expires > time() and \
set(old_groups) == set(new_groups.split(',')):
token = candidate_token
# Create a new token if one didn't exist
if not token:
# Generate new token
token = '%stk%s' % (self.reseller_prefix, uuid4().hex)
expires = time() + self.token_life
groups = self._get_user_groups(account, account_user, account_id)
# Save token
memcache_token_key = '%s/token/%s' % (self.reseller_prefix, token)
memcache_client.set(memcache_token_key, (expires, groups),
time=float(expires - time()))
# Record the token with the user info for future use.
memcache_user_key = \
'%s/user/%s' % (self.reseller_prefix, account_user)
memcache_client.set(memcache_user_key, token,
time=float(expires - time()))
resp = Response(request=req, headers={
'x-auth-token': token, 'x-storage-token': token})
url = self.users[account_user]['url'].replace('$HOST', resp.host_url)
if self.storage_url_scheme != 'default':
url = self.storage_url_scheme + ':' + url.split(':', 1)[1]
resp.headers['x-storage-url'] = url
return resp
def _GET_using_cache(self, req):
# It may be possible to fulfil the request from cache: we only reach
# here if request record_type is 'shard' or 'auto', so if the container
# state is 'sharded' then look for cached shard ranges. However, if
# X-Newest is true then we always fetch from the backend servers.
get_newest = config_true_value(req.headers.get('x-newest', False))
if get_newest:
self.app.logger.debug(
'Skipping shard cache lookup (x-newest) for %s', req.path_qs)
info = None
else:
info = _get_info_from_caches(self.app, req.environ,
self.account_name,
self.container_name)
if (info and is_success(info['status'])
and info.get('sharding_state') == 'sharded'):
# container is sharded so we may have the shard ranges cached
headers = headers_from_container_info(info)
if headers:
# only use cached values if all required headers available
infocache = req.environ.setdefault('swift.infocache', {})
memcache = cache_from_env(req.environ, True)
cache_key = get_cache_key(self.account_name,
self.container_name,
shard='listing')
cached_ranges = infocache.get(cache_key)
if cached_ranges is None and memcache:
cached_ranges = memcache.get(cache_key)
if cached_ranges is not None:
infocache[cache_key] = tuple(cached_ranges)
# shard ranges can be returned from cache
self.app.logger.debug('Found %d shards in cache for %s',
len(cached_ranges), req.path_qs)
headers.update({
'x-backend-record-type': 'shard',
'x-backend-cached-results': 'true'
})
shard_range_body = self._filter_resp_shard_ranges(
req, cached_ranges)
# mimic GetOrHeadHandler.get_working_response...
# note: server sets charset with content_type but proxy
# GETorHEAD_base does not, so don't set it here either
resp = Response(request=req, body=shard_range_body)
update_headers(resp, headers)
resp.last_modified = math.ceil(
float(headers['x-put-timestamp']))
resp.environ['swift_x_timestamp'] = headers.get(
'x-timestamp')
resp.accept_ranges = 'bytes'
resp.content_type = 'application/json'
return resp
# The request was not fulfilled from cache so send to the backend
# server, but instruct the backend server to ignore name constraints in
# request params if returning shard ranges so that the response can
# potentially be cached. Only do this if the container state is
# 'sharded'. We don't attempt to cache shard ranges for a 'sharding'
# container as they may include the container itself as a 'gap filler'
# for shard ranges that have not yet cleaved; listings from 'gap
# filler' shard ranges are likely to become stale as the container
# continues to cleave objects to its shards and caching them is
# therefore more likely to result in stale or incomplete listings on
# subsequent container GETs.
req.headers['x-backend-override-shard-name-filter'] = 'sharded'
resp = self._GETorHEAD_from_backend(req)
sharding_state = resp.headers.get('x-backend-sharding-state',
'').lower()
resp_record_type = resp.headers.get('x-backend-record-type',
'').lower()
complete_listing = config_true_value(
resp.headers.pop('x-backend-override-shard-name-filter', False))
# given that we sent 'x-backend-override-shard-name-filter=sharded' we
# should only receive back 'x-backend-override-shard-name-filter=true'
# if the sharding state is 'sharded', but check them both anyway...
if (resp_record_type == 'shard' and sharding_state == 'sharded'
and complete_listing):
# backend returned unfiltered listing state shard ranges so parse
# them and replace response body with filtered listing
cache_key = get_cache_key(self.account_name,
self.container_name,
shard='listing')
data = self._parse_listing_response(req, resp)
backend_shard_ranges = self._parse_shard_ranges(req, data, resp)
if backend_shard_ranges is not None:
cached_ranges = [dict(sr) for sr in backend_shard_ranges]
if resp.headers.get('x-backend-sharding-state') == 'sharded':
# cache in infocache even if no shard ranges returned; this
# is unexpected but use that result for this request
infocache = req.environ.setdefault('swift.infocache', {})
infocache[cache_key] = tuple(cached_ranges)
memcache = cache_from_env(req.environ, True)
if memcache and cached_ranges:
# cache in memcache only if shard ranges as expected
self.app.logger.debug('Caching %d shards for %s',
len(cached_ranges), req.path_qs)
memcache.set(
cache_key,
cached_ranges,
time=self.app.recheck_listing_shard_ranges)
# filter returned shard ranges according to request constraints
resp.body = self._filter_resp_shard_ranges(req, cached_ranges)
return resp
