def POST(self, req):
"""HTTP POST request handler."""
if len(self.account_name) > MAX_ACCOUNT_NAME_LENGTH:
resp = HTTPBadRequest(request=req)
resp.body = 'Account name length of %d longer than %d' % \
(len(self.account_name), MAX_ACCOUNT_NAME_LENGTH)
return resp
error_response = check_metadata(req, 'account')
if error_response:
return error_response
account_partition, accounts = \
self.app.account_ring.get_nodes(self.account_name)
headers = self.generate_request_headers(req, transfer=True)
if self.app.memcache:
self.app.memcache.delete(
get_account_memcache_key(self.account_name))
resp = self.make_requests(req, self.app.account_ring,
account_partition, 'POST', req.path_info,
[headers] * len(accounts))
if resp.status_int == HTTP_NOT_FOUND and self.app.account_autocreate:
self.autocreate_account(self.account_name)
resp = self.make_requests(req, self.app.account_ring,
account_partition, 'POST', req.path_info,
[headers] * len(accounts))
return resp
def PUT(self, req):
"""HTTP PUT request handler."""
if not self.app.allow_account_management:
return HTTPMethodNotAllowed(
request=req,
headers={'Allow': ', '.join(self.allowed_methods)})
error_response = check_metadata(req, 'account')
if error_response:
return error_response
if len(self.account_name) > MAX_ACCOUNT_NAME_LENGTH:
resp = HTTPBadRequest(request=req)
resp.body = 'Account name length of %d longer than %d' % \
(len(self.account_name), MAX_ACCOUNT_NAME_LENGTH)
return resp
account_partition, accounts = \
self.app.account_ring.get_nodes(self.account_name)
headers = {
'X-Timestamp': normalize_timestamp(time.time()),
'x-trans-id': self.trans_id,
'Connection': 'close'
}
self.transfer_headers(req.headers, headers)
if self.app.memcache:
self.app.memcache.delete(
get_account_memcache_key(self.account_name))
resp = self.make_requests(req, self.app.account_ring,
account_partition, 'PUT', req.path_info,
[headers] * len(accounts))
return resp
def POST(self, req):
"""HTTP POST request handler."""
if len(self.account_name) > MAX_ACCOUNT_NAME_LENGTH:
resp = HTTPBadRequest(request=req)
resp.body = 'Account name length of %d longer than %d' % \
(len(self.account_name), MAX_ACCOUNT_NAME_LENGTH)
return resp
error_response = check_metadata(req, 'account')
if error_response:
return error_response
account_partition, accounts = \
self.app.account_ring.get_nodes(self.account_name)
headers = self.generate_request_headers(req, transfer=True)
if self.app.memcache:
self.app.memcache.delete(
get_account_memcache_key(self.account_name))
resp = self.make_requests(
req, self.app.account_ring, account_partition, 'POST',
req.path_info, [headers] * len(accounts))
if resp.status_int == HTTP_NOT_FOUND and self.app.account_autocreate:
self.autocreate_account(self.account_name)
resp = self.make_requests(
req, self.app.account_ring, account_partition, 'POST',
req.path_info, [headers] * len(accounts))
return resp
def DELETE(self, req):
"""HTTP DELETE request handler."""
# Extra safety in case someone typos a query string for an
# account-level DELETE request that was really meant to be caught by
# some middleware.
if req.query_string:
return HTTPBadRequest(request=req)
if not self.app.allow_account_management:
return HTTPMethodNotAllowed(
request=req,
headers={'Allow': ', '.join(self.allowed_methods)})
account_partition, accounts = \
self.app.account_ring.get_nodes(self.account_name)
headers = {
'X-Timestamp': normalize_timestamp(time.time()),
'X-Trans-Id': self.trans_id,
'Connection': 'close'
}
if self.app.memcache:
self.app.memcache.delete(
get_account_memcache_key(self.account_name))
resp = self.make_requests(req, self.app.account_ring,
account_partition, 'DELETE', req.path_info,
[headers] * len(accounts))
return resp
def handle_delete(self, env, start_response, version, account, container,
obj):
""" Handle delete request. """
memcache_client = cache_from_env(env)
if not memcache_client:
return self.app(env, start_response)
res = [None, None, None]
result_code = None
def _start_response(response_status, response_headers, exc_info=None):
res[0] = response_status
res[1] = response_headers
res[2] = exc_info
resp = self.app(env, _start_response)
result_code = self._get_status_int(res[0])
try:
if is_success(result_code):
if obj:
memcache_client.delete(
get_container_memcache_key(account, container))
else:
memcache_client.delete(get_account_memcache_key(account))
except Exception, err:
self.logger.error(
'Error in [Quota] delete cache: %s' % (err.message))
def POST(self, req):
"""HTTP POST request handler."""
error_response = check_metadata(req, 'account')
if error_response:
return error_response
account_partition, accounts = \
self.app.account_ring.get_nodes(self.account_name)
headers = {'X-Timestamp': normalize_timestamp(time.time()),
'X-Trans-Id': self.trans_id,
'Connection': 'close'}
self.transfer_headers(req.headers, headers)
if self.app.memcache:
self.app.memcache.delete(
get_account_memcache_key(self.account_name))
resp = self.make_requests(
req, self.app.account_ring, account_partition, 'POST',
req.path_info, [headers] * len(accounts))
if resp.status_int == HTTP_NOT_FOUND and self.app.account_autocreate:
if len(self.account_name) > MAX_ACCOUNT_NAME_LENGTH:
resp = HTTPBadRequest(request=req)
resp.body = 'Account name length of %d longer than %d' % \
(len(self.account_name), MAX_ACCOUNT_NAME_LENGTH)
return resp
resp = self.make_requests(
Request.blank('/v1/' + self.account_name),
self.app.account_ring, account_partition, 'PUT',
'/' + self.account_name, [headers] * len(accounts))
if not is_success(resp.status_int):
self.app.logger.warning('Could not autocreate account %r' %
self.account_name)
return resp
return resp
def PUT(self, req):
"""HTTP PUT request handler."""
if not self.app.allow_account_management:
return HTTPMethodNotAllowed(
request=req,
headers={'Allow': ', '.join(self.allowed_methods)})
error_response = check_metadata(req, 'account')
if error_response:
return error_response
if len(self.account_name) > MAX_ACCOUNT_NAME_LENGTH:
resp = HTTPBadRequest(request=req)
resp.body = 'Account name length of %d longer than %d' % \
(len(self.account_name), MAX_ACCOUNT_NAME_LENGTH)
return resp
account_partition, accounts = \
self.app.account_ring.get_nodes(self.account_name)
headers = {'X-Timestamp': normalize_timestamp(time.time()),
'x-trans-id': self.trans_id,
'Connection': 'close'}
self.transfer_headers(req.headers, headers)
if self.app.memcache:
self.app.memcache.delete(
get_account_memcache_key(self.account_name))
resp = self.make_requests(
req, self.app.account_ring, account_partition, 'PUT',
req.path_info, [headers] * len(accounts))
return resp
def POST(self, req):
"""HTTP POST request handler."""
error_response = check_metadata(req, 'account')
if error_response:
return error_response
account_partition, accounts = \
self.app.account_ring.get_nodes(self.account_name)
headers = {
'X-Timestamp': normalize_timestamp(time.time()),
'X-Trans-Id': self.trans_id,
'Connection': 'close'
}
self.transfer_headers(req.headers, headers)
if self.app.memcache:
self.app.memcache.delete(
get_account_memcache_key(self.account_name))
resp = self.make_requests(req, self.app.account_ring,
account_partition, 'POST', req.path_info,
[headers] * len(accounts))
if resp.status_int == HTTP_NOT_FOUND and self.app.account_autocreate:
if len(self.account_name) > MAX_ACCOUNT_NAME_LENGTH:
resp = HTTPBadRequest(request=req)
resp.body = 'Account name length of %d longer than %d' % \
(len(self.account_name), MAX_ACCOUNT_NAME_LENGTH)
return resp
resp = self.make_requests(
Request.blank('/v1/' + self.account_name),
self.app.account_ring, account_partition, 'PUT',
'/' + self.account_name, [headers] * len(accounts))
if not is_success(resp.status_int):
self.app.logger.warning('Could not autocreate account %r' %
self.account_name)
return resp
return resp
def test_get_account_info_env(self):
cache_key = get_account_memcache_key("account")
env_key = 'swift.%s' % cache_key
req = Request.blank("/v1/account",
environ={env_key: {'bytes': 3867},
'swift.cache': FakeCache({})})
resp = get_account_info(req.environ, 'xxx')
self.assertEquals(resp['bytes'], 3867)
def test_get_account_info_env(self):
cache_key = get_account_memcache_key("account")
env_key = 'swift.%s' % cache_key
req = Request.blank("/v1/account",
environ={env_key: {'bytes': 3867},
'swift.cache': FakeCache({})})
resp = get_account_info(req.environ, 'xxx')
self.assertEquals(resp['bytes'], 3867)
def test_allowed_without_cache(self):
accname = 'acc_allowed'
swproxy.make_pre_authed_request = FakeRequest
req = self._make_request(
environ={
'REQUEST_METHOD': 'POST',
'PATH_INFO': '/v1/%s' % (accname),
'swift.cache': FakeCache({}),
})
resp = req.get_response(self.test_default)
key = swproxy.get_account_memcache_key(accname)
memcache_key = resp.environ.get("swift.%s" % key)
self.assertTrue('meta' in memcache_key)
self.assertEquals(memcache_key['meta']['locked'],
'false')
self.assertTrue('swift.authorize' not in resp.environ)
def DELETE(self, req):
"""HTTP DELETE request handler."""
if not self.app.allow_account_management:
return HTTPMethodNotAllowed(request=req)
account_partition, accounts = \
self.app.account_ring.get_nodes(self.account_name)
headers = {'X-Timestamp': normalize_timestamp(time.time()),
'X-Trans-Id': self.trans_id,
'Connection': 'close'}
if self.app.memcache:
self.app.memcache.delete(
get_account_memcache_key(self.account_name))
resp = self.make_requests(req, self.app.account_ring,
account_partition, 'DELETE', req.path_info,
[headers] * len(accounts))
return resp
def DELETE(self, req):
"""HTTP DELETE request handler."""
if not self.app.allow_account_management:
return HTTPMethodNotAllowed(
request=req,
headers={'Allow': ', '.join(self.allowed_methods)})
account_partition, accounts = \
self.app.account_ring.get_nodes(self.account_name)
headers = {'X-Timestamp': normalize_timestamp(time.time()),
'X-Trans-Id': self.trans_id,
'Connection': 'close'}
if self.app.memcache:
self.app.memcache.delete(
get_account_memcache_key(self.account_name))
resp = self.make_requests(
req, self.app.account_ring, account_partition, 'DELETE',
req.path_info, [headers] * len(accounts))
return resp
def test_deny_other_header_without_cache(self):
accname = 'acc_denied_other'
other_header = 'other'
conf = {'locked_header': other_header}
self.test_default = middleware.filter_factory(conf)(FakeApp())
swproxy.make_pre_authed_request = FakeRequest
req = self._make_request(
environ={
'REQUEST_METHOD': 'POST',
'PATH_INFO': '/v1/%s' % (accname),
'swift.cache': FakeCache({}),
})
resp = req.get_response(self.test_default)
key = swproxy.get_account_memcache_key(accname)
memcache_key = resp.environ.get("swift.%s" % key)
self.assertTrue('meta' in memcache_key)
self.assertTrue(other_header in memcache_key['meta'])
self.assertEquals(memcache_key['meta'][other_header], 'true')
self.assertTrue('swift.authorize' in resp.environ)
self.assertEquals(resp.environ['swift.authorize'],
self.test_default.deny)
def DELETE(self, req):
"""HTTP DELETE request handler."""
# Extra safety in case someone typos a query string for an
# account-level DELETE request that was really meant to be caught by
# some middleware.
if req.query_string:
return HTTPBadRequest(request=req)
if not self.app.allow_account_management:
return HTTPMethodNotAllowed(
request=req,
headers={'Allow': ', '.join(self.allowed_methods)})
account_partition, accounts = \
self.app.account_ring.get_nodes(self.account_name)
headers = self.generate_request_headers(req)
if self.app.memcache:
self.app.memcache.delete(
get_account_memcache_key(self.account_name))
resp = self.make_requests(
req, self.app.account_ring, account_partition, 'DELETE',
req.path_info, [headers] * len(accounts))
return resp
def get_account_info(env, app, logger):
"""
Get the info structure for an account, based on env and app.
This is useful to middlewares.
"""
# TODO: memcachet
container_info = env.get("swift.container_info")
if not container_info:
cache = cache_from_env(env)
if not cache:
return None
(version, account, _, _) = split_path(env["PATH_INFO"], 2, 4, True)
cache_key = get_account_memcache_key(account)
account_info = cache.get(cache_key)
if account_info:
print "Using memcache."
if not account_info:
new_env = dict(env, REQUEST_METHOD="HEAD")
resp = Request.blank("/%s/%s" % (version, account), environ=new_env).get_response(app)
account_info = headers_to_account_info(resp.headers)
# TODO: use recheck_account_existence for timeout.
cache.set(cache_key, account_info, timeout=60)
return account_info
def handle_quota_container(self, env, start_response, version, account,
container):
""" Container Count Quota """
memcache_client = cache_from_env(env)
container_count, quota_level = self._get_account_meta(
env, version, account, memcache_client)
try:
quota = self.container_count[quota_level]
except Exception:
self.logger.warn('Invalid quota_leve %s/%s quota_level[%s].' % (
account, container, quota_level))
quota = None
if quota and container_count and container_count + 1 > quota:
self.logger.notice("Container count over quota, request[PUT %s/%s],"
" container_count[%s] quota[%s]" % (
account, container, container_count + 1,
quota))
return HTTPForbidden(body="The number of container is over quota")(
env, start_response)
elif self.precise_mode and memcache_client:
res = [None, None, None]
result_code = None
def _start_response(response_status, response_headers,
exc_info=None):
res[0] = response_status
res[1] = response_headers
res[2] = exc_info
resp = self.app(env, _start_response)
result_code = self._get_status_int(res[0])
if is_success(result_code):
memcache_client.delete(get_account_memcache_key(account))
start_response(res[0], res[1], res[2])
return resp
else:
return self.app(env, start_response)
def PUT(self, req):
"""HTTP PUT request handler."""
if not self.app.allow_account_management:
return HTTPMethodNotAllowed(
request=req,
headers={'Allow': ', '.join(self.allowed_methods)})
error_response = check_metadata(req, 'account')
if error_response:
return error_response
if len(self.account_name) > MAX_ACCOUNT_NAME_LENGTH:
resp = HTTPBadRequest(request=req)
resp.body = 'Account name length of %d longer than %d' % \
(len(self.account_name), MAX_ACCOUNT_NAME_LENGTH)
return resp
account_partition, accounts = \
self.app.account_ring.get_nodes(self.account_name)
headers = self.generate_request_headers(req, transfer=True)
if self.app.memcache:
self.app.memcache.delete(
get_account_memcache_key(self.account_name))
resp = self.make_requests(
req, self.app.account_ring, account_partition, 'PUT',
req.path_info, [headers] * len(accounts))
return resp
def PUT(self, req):
"""HTTP PUT request handler."""
if not self.app.allow_account_management:
return HTTPMethodNotAllowed(
request=req,
headers={'Allow': ', '.join(self.allowed_methods)})
error_response = check_metadata(req, 'account')
if error_response:
return error_response
if len(self.account_name) > MAX_ACCOUNT_NAME_LENGTH:
resp = HTTPBadRequest(request=req)
resp.body = 'Account name length of %d longer than %d' % \
(len(self.account_name), MAX_ACCOUNT_NAME_LENGTH)
return resp
account_partition, accounts = \
self.app.account_ring.get_nodes(self.account_name)
headers = self.generate_request_headers(req, transfer=True)
if self.app.memcache:
self.app.memcache.delete(
get_account_memcache_key(self.account_name))
resp = self.make_requests(req, self.app.account_ring,
account_partition, 'PUT', req.path_info,
[headers] * len(accounts))
return resp
def _get_account_meta(self, env, version, account, memcache_client):
"""
Get metadata of account.
:param env: The WSGI environment
:param version: The api version in PATH_INFO
:param account: The name of account
:return: tuple of (container_count, quota_level) or (None, None)
"""
value = None
quota_level = None
container_count = None
res = [None, None, None]
result_code = None
def _start_response(response_status, response_headers, exc_info=None):
res[0] = response_status
res[1] = response_headers
res[2] = exc_info
# get quota_level and container_count
account_key = get_account_memcache_key(account)
if memcache_client:
value = memcache_client.get(account_key)
if value:
self.logger.debug('value from mc: %s' % (value))
if not isinstance(value, dict):
result_code = value
else:
result_code = value.get('status')
if is_success(result_code):
# get from memcached
container_count = int(value.get('container_count') or 0)
quota_level = value.get('quota_level') or 'default'
return container_count, quota_level
else:
# get from account-server
temp_env = self._get_escalated_env(env)
temp_env['REQUEST_METHOD'] = 'HEAD'
temp_env['PATH_INFO'] = '/%s/%s' % (version, account)
resp = self.app(temp_env, _start_response)
self.logger.debug(
'value form account-server status[%s] header[%s]' % (res[0],
res[1]))
result_code = self._get_status_int(res[0])
if is_success(result_code):
headers = dict(res[1])
container_count = int(
headers.get('x-account-container-count') or 0)
quota_level = headers.get('x-account-meta-quota') or 'default'
if memcache_client:
memcache_client.set(
account_key,
{
'status': result_code,
'container_count': container_count,
'quota_level': quota_level
},
timeout=self.cache_timeout
)
return container_count, quota_level
else:
return None, None
def test_get_account_info_env(self):
cache_key = get_account_memcache_key("account")
env_key = "swift.%s" % cache_key
req = Request.blank("/v1/account", environ={env_key: {"bytes": 3867}, "swift.cache": FakeCache({})})
resp = get_account_info(req.environ, "xxx")
self.assertEquals(resp["bytes"], 3867)
