def test_get_account_info_cache(self):
# The original test that we prefer to preserve
cached = {'status': 404, 'bytes': 3333, 'total_object_count': 10}
req = Request.blank("/v1/account/cont",
environ={'swift.cache': FakeCache(cached)})
resp = get_account_info(req.environ, FakeApp())
self.assertEqual(resp['bytes'], 3333)
self.assertEqual(resp['total_object_count'], 10)
self.assertEqual(resp['status'], 404)
# Here is a more realistic test
cached = {
'status': 404,
'bytes': '3333',
'container_count': '234',
'total_object_count': '10',
'meta': {}
}
req = Request.blank("/v1/account/cont",
environ={'swift.cache': FakeCache(cached)})
resp = get_account_info(req.environ, FakeApp())
self.assertEqual(resp['status'], 404)
self.assertEqual(resp['bytes'], '3333')
self.assertEqual(resp['container_count'], 234)
self.assertEqual(resp['meta'], {})
self.assertEqual(resp['total_object_count'], '10')
def test_get_account_info_cache(self):
# The original test that we prefer to preserve
cached = {'status': 404, 'bytes': 3333, 'total_object_count': 10}
req = Request.blank("/v1/account/cont",
environ={'swift.cache': FakeCache(cached)})
with patch(
'swift.proxy.controllers.base.'
'_prepare_pre_auth_info_request', FakeRequest):
resp = get_account_info(req.environ, 'xxx')
self.assertEquals(resp['bytes'], 3333)
self.assertEquals(resp['total_object_count'], 10)
self.assertEquals(resp['status'], 404)
# Here is a more realistic test
cached = {
'status': 404,
'bytes': '3333',
'container_count': '234',
'total_object_count': '10',
'meta': {}
}
req = Request.blank("/v1/account/cont",
environ={'swift.cache': FakeCache(cached)})
with patch(
'swift.proxy.controllers.base.'
'_prepare_pre_auth_info_request', FakeRequest):
resp = get_account_info(req.environ, 'xxx')
self.assertEquals(resp['status'], 404)
self.assertEquals(resp['bytes'], '3333')
self.assertEquals(resp['container_count'], 234)
self.assertEquals(resp['meta'], {})
self.assertEquals(resp['total_object_count'], '10')
def test_get_account_info_cache(self):
# The original test that we prefer to preserve
cached = {'status': 404,
'bytes': 3333,
'total_object_count': 10}
req = Request.blank("/v1/account/cont",
environ={'swift.cache': FakeCache(cached)})
with patch('swift.proxy.controllers.base.'
'_prepare_pre_auth_info_request', FakeRequest):
resp = get_account_info(req.environ, 'xxx')
self.assertEquals(resp['bytes'], 3333)
self.assertEquals(resp['total_object_count'], 10)
self.assertEquals(resp['status'], 404)
# Here is a more realistic test
cached = {'status': 404,
'bytes': '3333',
'container_count': '234',
'total_object_count': '10',
'meta': {}}
req = Request.blank("/v1/account/cont",
environ={'swift.cache': FakeCache(cached)})
with patch('swift.proxy.controllers.base.'
'_prepare_pre_auth_info_request', FakeRequest):
resp = get_account_info(req.environ, 'xxx')
self.assertEquals(resp['status'], 404)
self.assertEquals(resp['bytes'], '3333')
self.assertEquals(resp['container_count'], 234)
self.assertEquals(resp['meta'], {})
self.assertEquals(resp['total_object_count'], '10')
def test_get_account_info_cache(self):
# Works with fake apps that return ints in the headers
cached = {'status': 404,
'bytes': 3333,
'total_object_count': 10}
req = Request.blank("/v1/account/cont",
environ={'swift.cache': FakeCache(cached)})
resp = get_account_info(req.environ, FakeApp())
self.assertEqual(resp['bytes'], 3333)
self.assertEqual(resp['total_object_count'], 10)
self.assertEqual(resp['status'], 404)
# Works with strings too, like you get when parsing HTTP headers
# that came in through a socket from the account server
cached = {'status': 404,
'bytes': '3333',
'container_count': '234',
'total_object_count': '10',
'meta': {}}
req = Request.blank("/v1/account/cont",
environ={'swift.cache': FakeCache(cached)})
resp = get_account_info(req.environ, FakeApp())
self.assertEqual(resp['status'], 404)
self.assertEqual(resp['bytes'], 3333)
self.assertEqual(resp['container_count'], 234)
self.assertEqual(resp['meta'], {})
self.assertEqual(resp['total_object_count'], 10)
def test_get_account_info_cache(self):
# The original test that we prefer to preserve
cached = {'status': 404,
'bytes': 3333,
'total_object_count': 10}
req = Request.blank("/v1/account/cont",
environ={'swift.cache': FakeCache(cached)})
resp = get_account_info(req.environ, FakeApp())
self.assertEqual(resp['bytes'], 3333)
self.assertEqual(resp['total_object_count'], 10)
self.assertEqual(resp['status'], 404)
# Here is a more realistic test
cached = {'status': 404,
'bytes': '3333',
'container_count': '234',
'total_object_count': '10',
'meta': {}}
req = Request.blank("/v1/account/cont",
environ={'swift.cache': FakeCache(cached)})
resp = get_account_info(req.environ, FakeApp())
self.assertEqual(resp['status'], 404)
self.assertEqual(resp['bytes'], '3333')
self.assertEqual(resp['container_count'], 234)
self.assertEqual(resp['meta'], {})
self.assertEqual(resp['total_object_count'], '10')
def test_get_account_info_cache(self):
# Works with fake apps that return ints in the headers
cached = {'status': 404, 'bytes': 3333, 'total_object_count': 10}
req = Request.blank("/v1/account/cont",
environ={'swift.cache': FakeCache(cached)})
resp = get_account_info(req.environ, FakeApp())
self.assertEqual(resp['bytes'], 3333)
self.assertEqual(resp['total_object_count'], 10)
self.assertEqual(resp['status'], 404)
# Works with strings too, like you get when parsing HTTP headers
# that came in through a socket from the account server
cached = {
'status': 404,
'bytes': '3333',
'container_count': '234',
'total_object_count': '10',
'meta': {}
}
req = Request.blank("/v1/account/cont",
environ={'swift.cache': FakeCache(cached)})
resp = get_account_info(req.environ, FakeApp())
self.assertEqual(resp['status'], 404)
self.assertEqual(resp['bytes'], 3333)
self.assertEqual(resp['container_count'], 234)
self.assertEqual(resp['meta'], {})
self.assertEqual(resp['total_object_count'], 10)
def test_get_account_info_infocache(self):
app = FakeApp()
ic = {}
req = Request.blank("/v1/a", environ={'swift.cache': FakeCache(),
'swift.infocache': ic})
get_account_info(req.environ, app)
got_infocaches = [e['swift.infocache'] for e in app.captured_envs]
self.assertEqual(1, len(got_infocaches))
self.assertIs(ic, got_infocaches[0])
def test_get_account_info_infocache(self):
app = FakeApp()
ic = {}
req = Request.blank("/v1/a", environ={'swift.cache': FakeCache(),
'swift.infocache': ic})
get_account_info(req.environ, app)
got_infocaches = [e['swift.infocache'] for e in app.captured_envs]
self.assertEqual(1, len(got_infocaches))
self.assertIs(ic, got_infocaches[0])
def __init__(self, request, conf, app, logger):
super(StorletProxyHandler, self).__init__(
request, conf, app, logger)
self.storlet_container = conf.get('storlet_container')
self.storlet_dependency = conf.get('storlet_dependency')
self.storlet_containers = [self.storlet_container,
self.storlet_dependency]
if not self.is_storlet_request:
# This is not storlet-related request, so pass it
raise NotStorletRequest()
# In proxy server, storlet handler validate if storlet enabled
# at the account, anyway
account_meta = get_account_info(self.request.environ,
self.app)['meta']
storlets_enabled = account_meta.get('storlet-enabled',
'False')
if not config_true_value(storlets_enabled):
self.logger.debug('Account disabled for storlets')
raise HTTPBadRequest('Account disabled for storlets',
request=self.request)
if self.is_storlet_object_update:
# TODO(takashi): We have to validate metadata in COPY case
self._validate_registration(self.request)
raise NotStorletExecution()
else:
# if self.is_storlet_execution
self._setup_gateway()
def __call__(self, env, start_response):
req = Request(env)
# We want to check POST here
# it can possibly have content_length > 0
if not self.enforce_quota or req.method not in ("POST", "PUT", "COPY"):
return self.app(env, start_response)
account_info = get_account_info(req.environ, self.app,
swift_source='litequota')
if not account_info:
return self.app(env, start_response)
service_plan = assemble_from_partial(self.metadata_key,
account_info['meta'])
try:
ver, account, container, obj = \
req.split_path(2, 4, rest_with_last=True)
except ValueError:
return self.app(env, start_response)
if not service_plan and req.method == 'PUT' and not obj:
service_plan = self.set_serviceplan(req, account)
if not service_plan:
return self.app(env, start_response)
try:
service_plan = json.loads(service_plan)
except ValueError:
return self.app(env, start_response)
if service_plan.get('storage', None):
resp = self.apply_storage_quota(req, service_plan['storage'],
account_info,
ver, account, container, obj)
if resp:
return resp(env, start_response)
return self.app(env, start_response)
def __init__(self, request, conf, gateway_conf, app, logger):
super(StorletProxyHandler, self).__init__(
request, conf, gateway_conf, app, logger)
self.storlet_containers = [self.storlet_container,
self.storlet_dependency]
self.agent = 'ST'
self.extra_sources = []
# A very initial hook for blocking requests
self._should_block(request)
if not self.is_storlet_request:
# This is not storlet-related request, so pass it
raise NotStorletRequest()
# In proxy server, storlet handler validate if storlet enabled
# at the account, anyway
account_meta = get_account_info(self.request.environ,
self.app)['meta']
storlets_enabled = account_meta.get('storlet-enabled', 'False')
if not config_true_value(storlets_enabled):
msg = 'Account disabled for storlets'
self.logger.debug(msg)
raise HTTPBadRequest(msg.encode('utf8'), request=self.request)
if self.is_storlet_acl_update:
self.acl_string = self._validate_acl_update(self.request)
elif self.is_storlet_object_update:
# TODO(takashi): We have to validate metadata in COPY case
self._validate_registration(self.request)
raise NotStorletExecution()
elif self.is_storlet_execution:
self._setup_gateway()
else:
raise NotStorletExecution()
def __call__(self, request):
if request.method not in ("POST", "PUT"):
return self.app
try:
request.split_path(2, 4, rest_with_last=True)
except ValueError:
return self.app
new_quota = request.headers.get('X-Account-Meta-Quota-Bytes')
remove_quota = request.headers.get('X-Remove-Account-Meta-Quota-Bytes')
if remove_quota:
new_quota = 0 # X-Remove dominates if both are present
if request.environ.get('reseller_request') is True:
if new_quota and not new_quota.isdigit():
return HTTPBadRequest()
return self.app
# deny quota set for non-reseller
if new_quota is not None:
return HTTPForbidden()
account_info = get_account_info(request.environ, self.app)
if not account_info or not account_info['bytes']:
return self.app
new_size = int(account_info['bytes']) + (request.content_length or 0)
quota = int(account_info['meta'].get('quota-bytes', -1))
if 0 <= quota < new_size:
return HTTPRequestEntityTooLarge()
return self.app
def get_defaults(self, req, req_type, format_args):
acct_sysmeta = get_account_info(req.environ, self.app)['sysmeta']
if req_type == 'object':
cont_sysmeta = get_container_info(req.environ, self.app)['sysmeta']
else:
cont_sysmeta = {}
defaults = {}
prefix = 'default-%s-' % req_type
for src in (self.conf, acct_sysmeta, cont_sysmeta):
for key, value in src.items():
if not key.lower().startswith(prefix):
continue
header_to_default = key[len(prefix):].lower()
if header_to_default.startswith(BLACKLIST_PREFIXES):
continue
if header_to_default in BLACKLIST:
continue
if self.conf['use_formatting']:
try:
value = value.format(**format_args)
except KeyError:
# This user may not have specified the default;
# don't fail because of someone else
pass
defaults[header_to_default] = value
return defaults
def __call__(self, env, start_response):
req = Request(env)
# TOREMOVE: when merged
# if we are doing a post to update the locked value then alow it.
if req.method == 'POST':
for header in req.headers:
if header.lower() == "x-account-meta-%s" % \
self.locked_header.lower():
return self.app(env, start_response)
# check if we are in a method we want to disallow.
if not req.method in self.denied_methods:
return self.app(env, start_response)
account_info = get_account_info(env,
self.app,
swift_source="LCK")
if not account_info:
return self.app(env, start_response)
if 'meta' in account_info and self.locked_header in \
account_info['meta'] and config_true_value(
account_info['meta'][self.locked_header]):
self.logger.debug(
"[account_access] account locked for %s" %
(str(req.remote_user)))
env['swift.authorize'] = self.deny
return self.app(env, start_response)
def __init__(self, request, conf, gateway_conf, app, logger):
super(StorletProxyHandler, self).__init__(request, conf, gateway_conf, app, logger)
self.storlet_containers = [self.storlet_container, self.storlet_dependency]
self.agent = "ST"
self.extra_sources = []
# A very initial hook for blocking requests
self._should_block(request)
if not self.is_storlet_request:
# This is not storlet-related request, so pass it
raise NotStorletRequest()
# In proxy server, storlet handler validate if storlet enabled
# at the account, anyway
account_meta = get_account_info(self.request.environ, self.app)["meta"]
storlets_enabled = account_meta.get("storlet-enabled", "False")
if not config_true_value(storlets_enabled):
self.logger.debug("Account disabled for storlets")
raise HTTPBadRequest("Account disabled for storlets", request=self.request)
if self.is_storlet_acl_update:
self.acl_string = self._validate_acl_update(self.request)
elif self.is_storlet_object_update:
# TODO(takashi): We have to validate metadata in COPY case
self._validate_registration(self.request)
raise NotStorletExecution()
elif self.is_storlet_execution:
self._setup_gateway()
else:
raise NotStorletExecution()
def POST(self, env):
"""Handle posts dealing with metadata alteration"""
req = Request(env)
conn = HTTPConnection('%s:%s' % (self.mds_ip, self.mds_port))
headers = req.params
version, acc, con, obj = split_path(req.path, 1, 4, True)
if not con:
try:
info = get_account_info(env, self.app)
if info:
stor_policy = info['storage_policy']
headers['storage_policy'] = stor_policy
except:
pass
else:
try:
info = get_container_info(env, self.app)
if info:
stor_policy = info['storage_policy']
headers['storage_policy'] = stor_policy
except:
pass
conn.request('POST', req.path, headers=headers)
resp = conn.getresponse()
#confirm response then pass along the request
return self.app
def POST(self, env):
"""Handle posts dealing with metadata alteration"""
req = Request(env)
conn = HTTPConnection('%s:%s' % (self.mds_ip, self.mds_port))
headers = req.params
version, acc, con, obj = split_path(req.path, 1, 4, True)
if not con:
try:
info = get_account_info(env, self.app)
if info:
stor_policy = info['storage_policy']
headers['storage_policy'] = stor_policy
except:
pass
else:
try:
info = get_container_info(env, self.app)
if info:
stor_policy = info['storage_policy']
headers['storage_policy'] = stor_policy
except:
pass
conn.request('POST', req.path, headers=headers)
resp = conn.getresponse()
#confirm response then pass along the request
return self.app
def get_defaults(self, req, req_type, format_args):
acct_sysmeta = get_account_info(req.environ, self.app)['sysmeta']
if req_type == 'object':
cont_sysmeta = get_container_info(req.environ, self.app)['sysmeta']
else:
cont_sysmeta = {}
defaults = {}
prefix = 'default-%s-' % req_type
for src in (self.conf, acct_sysmeta, cont_sysmeta):
for key, value in src.items():
if not key.lower().startswith(prefix):
continue
header_to_default = key[len(prefix):].lower()
if header_to_default.startswith(BLACKLIST_PREFIXES):
continue
if header_to_default in BLACKLIST:
continue
if self.conf['use_formatting']:
try:
value = value.format(**format_args)
except KeyError:
# This user may not have specified the default;
# don't fail because of someone else
pass
defaults[header_to_default] = value
return defaults
def __call__(self, env, start_response):
req = Request(env)
try:
version, account, container, obj = req.split_path(
2, 4, rest_with_last=True)
is_swifty_request = valid_api_version(version)
except ValueError:
is_swifty_request = False
if not is_swifty_request:
return self.app(env, start_response)
if not obj:
typ = 'Container' if container else 'Account'
client_header = 'X-%s-Rfc-Compliant-Etags' % typ
sysmeta_header = 'X-%s-Sysmeta-Rfc-Compliant-Etags' % typ
if client_header in req.headers:
if req.headers[client_header]:
req.headers[sysmeta_header] = config_true_value(
req.headers[client_header])
else:
req.headers[sysmeta_header] = ''
if req.headers.get(client_header.replace('X-', 'X-Remove-', 1)):
req.headers[sysmeta_header] = ''
def translating_start_response(status, headers, exc_info=None):
return start_response(
status,
[(client_header if h.title() == sysmeta_header else h, v)
for h, v in headers], exc_info)
return self.app(env, translating_start_response)
container_info = get_container_info(env, self.app, 'EQ')
if not container_info or not is_success(container_info['status']):
return self.app(env, start_response)
flag = container_info.get('sysmeta', {}).get('rfc-compliant-etags')
if flag is None:
account_info = get_account_info(env, self.app, 'EQ')
if not account_info or not is_success(account_info['status']):
return self.app(env, start_response)
flag = account_info.get('sysmeta', {}).get('rfc-compliant-etags')
if flag is None:
flag = self.conf.get('enable_by_default', 'false')
if not config_true_value(flag):
return self.app(env, start_response)
status, headers, resp_iter = req.call_application(self.app)
headers = [(header,
value) if header.lower() != 'etag' or (value.startswith(
('"', 'W/"')) and value.endswith('"')) else
(header, '"%s"' % value) for header, value in headers]
start_response(status, headers)
return resp_iter
def test_get_account_info_swift_source(self):
req = Request.blank("/v1/a", environ={'swift.cache': FakeCache({})})
with patch(
'swift.proxy.controllers.base.'
'_prepare_pre_auth_info_request', FakeRequest):
resp = get_account_info(req.environ, 'a', swift_source='MC')
self.assertEquals(resp['meta']['fakerequest-swift-source'], 'MC')
def _get_keys(self, env):
"""
Returns the X-[Account|Container]-Meta-Temp-URL-Key[-2] header values
for the account or container, or an empty list if none are set. Each
value comes as a 2-tuple (key, scope), where scope is either
CONTAINER_SCOPE or ACCOUNT_SCOPE.
Returns 0-4 elements depending on how many keys are set in the
account's or container's metadata.
:param env: The WSGI environment for the request.
:returns: [
(X-Account-Meta-Temp-URL-Key str value, ACCOUNT_SCOPE) if set,
(X-Account-Meta-Temp-URL-Key-2 str value, ACCOUNT_SCOPE if set,
(X-Container-Meta-Temp-URL-Key str value, CONTAINER_SCOPE) if set,
(X-Container-Meta-Temp-URL-Key-2 str value, CONTAINER_SCOPE if set,
]
"""
account_info = get_account_info(env, self.app, swift_source='TU')
account_keys = get_tempurl_keys_from_metadata(account_info['meta'])
container_info = get_container_info(env, self.app, swift_source='TU')
container_keys = get_tempurl_keys_from_metadata(
container_info.get('meta', []))
return ([(ak, ACCOUNT_SCOPE) for ak in account_keys] +
[(ck, CONTAINER_SCOPE) for ck in container_keys])
def account_acls(self, req):
"""
Return a dict of ACL data from the account server via get_account_info.
Auth systems may define their own format, serialization, structure,
and capabilities implemented in the ACL headers and persisted in the
sysmeta data. However, auth systems are strongly encouraged to be
interoperable with Tempauth.
Account ACLs are set and retrieved via the header
X-Account-Access-Control
For header format and syntax, see:
* :func:`swift.common.middleware.acl.parse_acl()`
* :func:`swift.common.middleware.acl.format_acl()`
"""
info = get_account_info(req.environ, self.app, swift_source='TA')
try:
acls = acls_from_account_info(info)
except ValueError as e1:
self.logger.warning("Invalid ACL stored in metadata: %r" % e1)
return None
except NotImplementedError as e2:
self.logger.warning(
"ACL version exceeds middleware version: %r"
% e2)
return None
return acls
def _get_keys(self, env):
"""
Returns the X-[Account|Container]-Meta-Temp-URL-Key[-2] header values
for the account or container, or an empty list if none are set.
Returns 0-4 elements depending on how many keys are set in the
account's or container's metadata.
Also validate that the request
path indicates a valid container; if not, no keys will be returned.
:param env: The WSGI environment for the request.
:returns: list of tempurl keys
"""
parts = env['PATH_INFO'].split('/', 4)
if len(parts) < 4 or parts[0] or not valid_api_version(parts[1]) \
or not parts[2] or not parts[3]:
return []
account_info = get_account_info(env, self.app, swift_source='FP')
account_keys = get_tempurl_keys_from_metadata(account_info['meta'])
container_info = get_container_info(env, self.app, swift_source='FP')
container_keys = get_tempurl_keys_from_metadata(
container_info.get('meta', []))
return account_keys + container_keys
def _get_keys(self, env):
"""
Returns the X-[Account|Container]-Meta-Temp-URL-Key[-2] header values
for the account or container, or an empty list if none are set.
Returns 0-4 elements depending on how many keys are set in the
account's or container's metadata.
Also validate that the request
path indicates a valid container; if not, no keys will be returned.
:param env: The WSGI environment for the request.
:returns: list of tempurl keys
"""
parts = env["PATH_INFO"].split("/", 4)
if len(parts) < 4 or parts[0] or parts[1] != "v1" or not parts[2] or not parts[3]:
return []
account_info = get_account_info(env, self.app, swift_source="FP")
account_keys = get_tempurl_keys_from_metadata(account_info["meta"])
container_info = get_container_info(env, self.app, swift_source="FP")
container_keys = get_tempurl_keys_from_metadata(container_info.get("meta", []))
return account_keys + container_keys
def test_get_account_info_no_cache(self):
app = FakeApp()
req = Request.blank("/v1/AUTH_account",
environ={'swift.cache': FakeCache({})})
resp = get_account_info(req.environ, app)
self.assertEqual(resp['bytes'], 6666)
self.assertEqual(resp['total_object_count'], 1000)
def test_get_account_info_no_cache(self):
app = FakeApp()
req = Request.blank("/v1/AUTH_account",
environ={'swift.cache': FakeCache({})})
resp = get_account_info(req.environ, app)
self.assertEqual(resp['bytes'], 6666)
self.assertEqual(resp['total_object_count'], 1000)
def __call__(self, request):
if request.method not in ("PUT","COPY"):
return self.app
try:
split_path(request.path,2, 4, rest_with_last=True)
except ValueError:
return self.app
new_quota = request.headers.get('X-Account-Meta-Quota-Bytes')
if new_quota:
if not new_quota.isdigit():
return jresponse('-1', 'bad request', request, 400)
return self.app
account_info = get_account_info(request.environ, self.app)
new_size = int(account_info['bytes']) + (request.content_length or 0)
quota = int(account_info['meta'].get('quota-bytes', -1))
if 0 <= quota < new_size:
respbody='Your request is too large.'
return jresponse('-1', respbody, request,413)
return self.app
def account_acls(self, req):
"""
Return a dict of ACL data from the account server via get_account_info.
Auth systems may define their own format, serialization, structure,
and capabilities implemented in the ACL headers and persisted in the
sysmeta data. However, auth systems are strongly encouraged to be
interoperable with Tempauth.
Account ACLs are set and retrieved via the header
X-Account-Access-Control
For header format and syntax, see:
* :func:`swift.common.middleware.acl.parse_acl()`
* :func:`swift.common.middleware.acl.format_acl()`
"""
info = get_account_info(req.environ, self.app, swift_source='TA')
try:
acls = acls_from_account_info(info)
except ValueError as e1:
self.logger.warn("Invalid ACL stored in metadata: %r" % e1)
return None
except NotImplementedError as e2:
self.logger.warn("ACL version exceeds middleware version: %r" % e2)
return None
return acls
def __call__(self, request):
if request.method not in ("POST", "PUT"):
return self.app
try:
request.split_path(2, 4, rest_with_last=True)
except ValueError:
return self.app
new_quota = request.headers.get('X-Account-Meta-Quota-Bytes')
if request.environ.get('reseller_request') is True:
if new_quota and not new_quota.isdigit():
return HTTPBadRequest()
return self.app
# deny quota set for non-reseller
if new_quota is not None:
return HTTPForbidden()
account_info = get_account_info(request.environ, self.app)
if not account_info or not account_info['bytes']:
return self.app
new_size = int(account_info['bytes']) + (request.content_length or 0)
quota = int(account_info['meta'].get('quota-bytes', -1))
if 0 <= quota < new_size:
return HTTPRequestEntityTooLarge()
return self.app
def is_account_storlet_enabled(self):
account_meta = get_account_info(self.request.environ, self.app)['meta']
storlets_enabled = account_meta.get('storlet-enabled', 'False')
if not config_true_value(storlets_enabled):
self.logger.debug('Vertigo - Account disabled for storlets')
raise HTTPBadRequest('Vertigo - Error: Account disabled for'
' storlets.\n', request=self.request)
return True
def test_get_deleted_account_410(self):
resp_headers = {'x-account-status': 'deleted'}
req = Request.blank('/v1/a')
with mock.patch('swift.proxy.controllers.base.http_connect',
fake_http_connect(404, headers=resp_headers)):
info = get_account_info(req.environ, self.app)
self.assertEqual(410, info.get('status'))
def test_get_account_info_no_cache(self):
swift.proxy.controllers.base.make_pre_authed_request = FakeRequest
req = Request.blank("/v1/AUTH_account",
environ={'swift.cache': FakeCache({})})
resp = get_account_info(req.environ, 'xxx')
print resp
self.assertEquals(resp['bytes'], 6666)
self.assertEquals(resp['total_object_count'], 1000)
def is_account_storlet_enabled(self):
account_meta = get_account_info(self.request.environ, self.app)['meta']
storlets_enabled = account_meta.get('storlet-enabled', 'False')
if not config_true_value(storlets_enabled):
self.logger.debug('Vertigo - Account disabled for storlets')
raise HTTPBadRequest('Vertigo - Error: Account disabled for'
' storlets.\n', request=self.request)
return True
def test_get_account_info_env(self):
cache_key = get_cache_key("account")
req = Request.blank(
"/v1/account",
environ={'swift.infocache': {cache_key: {'bytes': 3867}},
'swift.cache': FakeCache({})})
resp = get_account_info(req.environ, 'xxx')
self.assertEqual(resp['bytes'], 3867)
def test_get_account_info_env(self):
cache_key = get_account_memcache_key("account")
env_key = 'swift.%s' % cache_key
req = Request.blank("/v1/account",
environ={env_key: {'bytes': 3867},
'swift.cache': FakeCache({})})
resp = get_account_info(req.environ, 'xxx')
self.assertEquals(resp['bytes'], 3867)
def test_get_account_info_no_cache(self):
req = Request.blank("/v1/AUTH_account",
environ={'swift.cache': FakeCache({})})
with patch('swift.proxy.controllers.base.'
'_prepare_pre_auth_info_request', FakeRequest):
resp = get_account_info(req.environ, 'xxx')
self.assertEquals(resp['bytes'], 6666)
self.assertEquals(resp['total_object_count'], 1000)
def test_get_deleted_account_410(self):
resp_headers = {'x-account-status': 'deleted'}
req = Request.blank('/v1/a')
with mock.patch('swift.proxy.controllers.base.http_connect',
fake_http_connect(404, headers=resp_headers)):
info = get_account_info(req.environ, self.app)
self.assertEqual(410, info.get('status'))
def test_get_account_info_env(self):
cache_key = get_cache_key("account")
req = Request.blank(
"/v1/account",
environ={'swift.infocache': {cache_key: {'bytes': 3867}},
'swift.cache': FakeCache({})})
resp = get_account_info(req.environ, 'xxx')
self.assertEqual(resp['bytes'], 3867)
def test_get_account_info_env(self):
cache_key = get_account_memcache_key("account")
env_key = 'swift.%s' % cache_key
req = Request.blank("/v1/account",
environ={env_key: {'bytes': 3867},
'swift.cache': FakeCache({})})
resp = get_account_info(req.environ, 'xxx')
self.assertEquals(resp['bytes'], 3867)
def test_get_account_info_no_cache(self):
swift.proxy.controllers.base.make_pre_authed_request = FakeRequest
req = Request.blank("/v1/AUTH_account",
environ={'swift.cache': FakeCache({})})
resp = get_account_info(req.environ, 'xxx')
print resp
self.assertEquals(resp['bytes'], 6666)
self.assertEquals(resp['total_object_count'], 1000)
def handle_ratelimit(self, req, account_name, container_name, obj_name):
"""
Performs rate limiting and account white/black listing. Sleeps
if necessary. If self.memcache_client is not set, immediately returns
None.
:param account_name: account name from path
:param container_name: container name from path
:param obj_name: object name from path
"""
if not self.memcache_client:
return None
try:
account_info = get_account_info(req.environ, self.app,
swift_source='RL')
account_global_ratelimit = \
account_info.get('sysmeta', {}).get('global-write-ratelimit')
except ValueError:
account_global_ratelimit = None
if account_name in self.ratelimit_whitelist or \
account_global_ratelimit == 'WHITELIST':
return None
if account_name in self.ratelimit_blacklist or \
account_global_ratelimit == 'BLACKLIST':
self.logger.error(_('Returning 497 because of blacklisting: %s'),
account_name)
eventlet.sleep(self.BLACK_LIST_SLEEP)
return Response(status='497 Blacklisted',
body='Your account has been blacklisted',
request=req)
for key, max_rate in self.get_ratelimitable_key_tuples(
req, account_name, container_name=container_name,
obj_name=obj_name, global_ratelimit=account_global_ratelimit):
try:
need_to_sleep = self._get_sleep_time(key, max_rate)
if self.log_sleep_time_seconds and \
need_to_sleep > self.log_sleep_time_seconds:
self.logger.warning(
_("Ratelimit sleep log: %(sleep)s for "
"%(account)s/%(container)s/%(object)s"),
{'sleep': need_to_sleep, 'account': account_name,
'container': container_name, 'object': obj_name})
if need_to_sleep > 0:
eventlet.sleep(need_to_sleep)
except MaxSleepTimeHitError as e:
self.logger.error(
_('Returning 498 for %(meth)s to %(acc)s/%(cont)s/%(obj)s '
'. Ratelimit (Max Sleep) %(e)s'),
{'meth': req.method, 'acc': account_name,
'cont': container_name, 'obj': obj_name, 'e': str(e)})
error_resp = Response(status='498 Rate Limited',
body='Slow down', request=req)
return error_resp
return None
def handle_ratelimit(self, req, account_name, container_name, obj_name):
"""
Performs rate limiting and account white/black listing. Sleeps
if necessary. If self.memcache_client is not set, immediately returns
None.
:param account_name: account name from path
:param container_name: container name from path
:param obj_name: object name from path
"""
if not self.memcache_client:
return None
try:
account_info = get_account_info(req.environ, self.app,
swift_source='RL')
account_global_ratelimit = \
account_info.get('sysmeta', {}).get('global-write-ratelimit')
except ValueError:
account_global_ratelimit = None
if account_name in self.ratelimit_whitelist or \
account_global_ratelimit == 'WHITELIST':
return None
if account_name in self.ratelimit_blacklist or \
account_global_ratelimit == 'BLACKLIST':
self.logger.error(_('Returning 497 because of blacklisting: %s'),
account_name)
eventlet.sleep(self.BLACK_LIST_SLEEP)
return Response(status='497 Blacklisted',
body='Your account has been blacklisted',
request=req)
for key, max_rate in self.get_ratelimitable_key_tuples(
req, account_name, container_name=container_name,
obj_name=obj_name, global_ratelimit=account_global_ratelimit):
try:
need_to_sleep = self._get_sleep_time(key, max_rate)
if self.log_sleep_time_seconds and \
need_to_sleep > self.log_sleep_time_seconds:
self.logger.warning(
_("Ratelimit sleep log: %(sleep)s for "
"%(account)s/%(container)s/%(object)s"),
{'sleep': need_to_sleep, 'account': account_name,
'container': container_name, 'object': obj_name})
if need_to_sleep > 0:
eventlet.sleep(need_to_sleep)
except MaxSleepTimeHitError as e:
self.logger.error(
_('Returning 498 for %(meth)s to %(acc)s/%(cont)s/%(obj)s '
'. Ratelimit (Max Sleep) %(e)s'),
{'meth': req.method, 'acc': account_name,
'cont': container_name, 'obj': obj_name, 'e': str(e)})
error_resp = Response(status='498 Rate Limited',
body='Slow down', request=req)
return error_resp
return None
def __call__(self, request):
if request.method not in ("POST", "PUT"):
return self.app
try:
ver, account, container, obj = request.split_path(
2, 4, rest_with_last=True)
except ValueError:
return self.app
if not container:
# account request, so we pay attention to the quotas
new_quota = request.headers.get('X-Account-Meta-Quota-Bytes')
remove_quota = request.headers.get(
'X-Remove-Account-Meta-Quota-Bytes')
else:
# container or object request; even if the quota headers are set
# in the request, they're meaningless
new_quota = remove_quota = None
if remove_quota:
new_quota = 0 # X-Remove dominates if both are present
if request.environ.get('reseller_request') is True:
if new_quota and not new_quota.isdigit():
return HTTPBadRequest()
return self.app
# deny quota set for non-reseller
if new_quota is not None:
return HTTPForbidden()
if obj and request.method == "POST" or not obj:
return self.app
copy_from = request.headers.get('X-Copy-From')
content_length = (request.content_length or 0)
if obj and copy_from:
path = '/' + ver + '/' + account + '/' + copy_from.lstrip('/')
object_info = get_object_info(request.environ, self.app, path)
if not object_info or not object_info['length']:
content_length = 0
else:
content_length = int(object_info['length'])
account_info = get_account_info(request.environ, self.app)
if not account_info or not account_info['bytes']:
return self.app
new_size = int(account_info['bytes']) + content_length
quota = int(account_info['meta'].get('quota-bytes', -1))
if 0 <= quota < new_size:
return HTTPRequestEntityTooLarge()
return self.app
def test_get_account_info_no_cache(self):
req = Request.blank("/v1/AUTH_account",
environ={'swift.cache': FakeCache({})})
with patch(
'swift.proxy.controllers.base.'
'_prepare_pre_auth_info_request', FakeRequest):
resp = get_account_info(req.environ, 'xxx')
self.assertEquals(resp['bytes'], 6666)
self.assertEquals(resp['total_object_count'], 1000)
def test_get_account_info_cache(self):
swift.proxy.controllers.base.make_pre_authed_request = FakeRequest
cached = {'status': 404, 'bytes': 3333, 'total_object_count': 10}
req = Request.blank("/v1/account/cont",
environ={'swift.cache': FakeCache(cached)})
resp = get_account_info(req.environ, 'xxx')
self.assertEquals(resp['bytes'], 3333)
self.assertEquals(resp['total_object_count'], 10)
self.assertEquals(resp['status'], 404)
def test_get_account_info_cache(self):
# The original test that we prefer to preserve
cached = {"status": 404, "bytes": 3333, "total_object_count": 10}
req = Request.blank("/v1/account/cont", environ={"swift.cache": FakeCache(cached)})
resp = get_account_info(req.environ, FakeApp())
self.assertEquals(resp["bytes"], 3333)
self.assertEquals(resp["total_object_count"], 10)
self.assertEquals(resp["status"], 404)
# Here is a more realistic test
cached = {"status": 404, "bytes": "3333", "container_count": "234", "total_object_count": "10", "meta": {}}
req = Request.blank("/v1/account/cont", environ={"swift.cache": FakeCache(cached)})
resp = get_account_info(req.environ, FakeApp())
self.assertEquals(resp["status"], 404)
self.assertEquals(resp["bytes"], "3333")
self.assertEquals(resp["container_count"], 234)
self.assertEquals(resp["meta"], {})
self.assertEquals(resp["total_object_count"], "10")
def get_ratelimitable_key_tuples(self, req, account_name,
container_name=None, obj_name=None):
"""
Returns a list of key (used in memcache), ratelimit tuples. Keys
should be checked in order.
:param req: swob request
:param account_name: account name from path
:param container_name: container name from path
:param obj_name: object name from path
"""
keys = []
# COPYs are not limited
if self.account_ratelimit and \
account_name and container_name and not obj_name and \
req.method in ('PUT', 'DELETE'):
keys.append(("ratelimit/%s" % account_name,
self.account_ratelimit))
if account_name and container_name and obj_name and \
req.method in ('PUT', 'DELETE', 'POST', 'COPY'):
container_size = self.get_container_size(
account_name, container_name)
container_rate = get_maxrate(
self.container_ratelimits, container_size)
if container_rate:
keys.append((
"ratelimit/%s/%s" % (account_name, container_name),
container_rate))
if account_name and container_name and not obj_name and \
req.method == 'GET':
container_size = self.get_container_size(
account_name, container_name)
container_rate = get_maxrate(
self.container_listing_ratelimits, container_size)
if container_rate:
keys.append((
"ratelimit_listing/%s/%s" % (account_name, container_name),
container_rate))
if account_name and req.method in ('PUT', 'DELETE', 'POST', 'COPY'):
account_info = get_account_info(req.environ, self.app)
account_global_ratelimit = \
account_info.get('sysmeta', {}).get('global-write-ratelimit')
if account_global_ratelimit:
try:
account_global_ratelimit = float(account_global_ratelimit)
if account_global_ratelimit > 0:
keys.append((
"ratelimit/global-write/%s" % account_name,
account_global_ratelimit))
except ValueError:
pass
return keys
def __call__(self, request):
if request.method not in ("POST", "PUT"):
return self.app
try:
ver, account, container, obj = request.split_path(2, 4, rest_with_last=True)
except ValueError:
return self.app
if not container:
# account request, so we pay attention to the quotas
new_quota = request.headers.get("X-Account-Meta-Quota-Bytes")
remove_quota = request.headers.get("X-Remove-Account-Meta-Quota-Bytes")
else:
# container or object request; even if the quota headers are set
# in the request, they're meaningless
new_quota = remove_quota = None
if remove_quota:
new_quota = 0 # X-Remove dominates if both are present
if request.environ.get("reseller_request") is True:
if new_quota and not new_quota.isdigit():
return HTTPBadRequest()
return self.app
# deny quota set for non-reseller
if new_quota is not None:
return HTTPForbidden()
if obj and request.method == "POST" or not obj:
return self.app
copy_from = request.headers.get("X-Copy-From")
content_length = request.content_length or 0
if obj and copy_from:
path = "/" + ver + "/" + account + "/" + copy_from.lstrip("/")
object_info = get_object_info(request.environ, self.app, path)
if not object_info or not object_info["length"]:
content_length = 0
else:
content_length = int(object_info["length"])
account_info = get_account_info(request.environ, self.app)
if not account_info or not account_info["bytes"]:
return self.app
new_size = int(account_info["bytes"]) + content_length
quota = int(account_info["meta"].get("quota-bytes", -1))
if 0 <= quota < new_size:
return HTTPRequestEntityTooLarge()
return self.app
def is_account_crystal_enabled(self):
account_meta = get_account_info(self.request.environ,
self.app)['meta']
crystal_enabled = account_meta.get('crystal-enabled',
'False')
if not config_true_value(crystal_enabled):
return False
return True
def is_account_storlet_enabled(self):
account_meta = get_account_info(self.request.environ,
self.app)['meta']
storlets_enabled = account_meta.get('storlet-enabled',
'False')
if not config_true_value(storlets_enabled):
return False
return True
def is_account_storlet_enabled(self):
account_meta = get_account_info(self.request.environ,
self.app)['meta']
storlets_enabled = account_meta.get('storlet-enabled',
'False')
if not config_true_value(storlets_enabled):
return True # TODO: CHANGE TO FALSE
return True
def test_get_account_info_cache(self):
# The original test that we prefer to preserve
cached = {"status": 404, "bytes": 3333, "total_object_count": 10}
req = Request.blank("/v1/account/cont", environ={"swift.cache": FakeCache(cached)})
with patch("swift.proxy.controllers.base." "_prepare_pre_auth_info_request", FakeRequest):
resp = get_account_info(req.environ, "xxx")
self.assertEquals(resp["bytes"], 3333)
self.assertEquals(resp["total_object_count"], 10)
self.assertEquals(resp["status"], 404)
# Here is a more realistic test
cached = {"status": 404, "bytes": "3333", "container_count": "234", "total_object_count": "10", "meta": {}}
req = Request.blank("/v1/account/cont", environ={"swift.cache": FakeCache(cached)})
with patch("swift.proxy.controllers.base." "_prepare_pre_auth_info_request", FakeRequest):
resp = get_account_info(req.environ, "xxx")
self.assertEquals(resp["status"], 404)
self.assertEquals(resp["bytes"], "3333")
self.assertEquals(resp["container_count"], 234)
self.assertEquals(resp["meta"], {})
self.assertEquals(resp["total_object_count"], "10")
def test_get_account_info_cache(self):
swift.proxy.controllers.base.make_pre_authed_request = FakeRequest
cached = {'status': 404,
'bytes': 3333,
'total_object_count': 10}
req = Request.blank("/v1/account/cont",
environ={'swift.cache': FakeCache(cached)})
resp = get_account_info(req.environ, 'xxx')
self.assertEquals(resp['bytes'], 3333)
self.assertEquals(resp['total_object_count'], 10)
self.assertEquals(resp['status'], 404)
def is_proxy_fs(self, env):
if 'pfs.is_bimodal' in env:
return env['pfs.is_bimodal']
parts = env['PATH_INFO'].split('/', 3)
if len(parts) < 3 or parts[1] not in ('v1', 'v1.0'):
# If it's not a swift request, it can't be a ProxyFS request
return False
account_info = get_account_info(env, self.base_app, swift_source='')
# ignore status; rely on something elsewhere in the pipeline
# to propagate the error
return utils.config_true_value(account_info["sysmeta"].get(
'proxyfs-bimodal'))
def is_enabled_for(self, env):
"""
Whether an account or container has meta-data to opt out of undelete
protection
"""
sysmeta_c = get_container_info(env, self.app)['sysmeta']
# Container info gets & caches account info, so this is basically free
sysmeta_a = get_account_info(env, self.app)['sysmeta']
enabled = sysmeta_c.get(SYSMETA_UNDELETE_ENABLED)
if enabled is None:
enabled = sysmeta_a.get(SYSMETA_UNDELETE_ENABLED,
self.enable_by_default)
return utils.config_true_value(enabled)
def _get_keys(self, env):
"""
Fetch the tempurl keys for the account. Also validate that the request
path indicates a valid container; if not, no keys will be returned.
:param env: The WSGI environment for the request.
:returns: list of tempurl keys
"""
parts = env['PATH_INFO'].split('/', 4)
if len(parts) < 4 or parts[0] or parts[1] != 'v1' or not parts[2] or \
not parts[3]:
return []
account_info = get_account_info(env, self.app, swift_source='FP')
return get_tempurl_keys_from_metadata(account_info['meta'])
def _get_keys(self, env, account):
"""
Returns the X-Account-Meta-Temp-URL-Key[-2] header values for the
account, or an empty list if none is set.
Returns 0, 1, or 2 elements depending on how many keys are set
in the account's metadata.
:param env: The WSGI environment for the request.
:param account: Account str.
:returns: [X-Account-Meta-Temp-URL-Key str value if set,
X-Account-Meta-Temp-URL-Key-2 str value if set]
"""
account_info = get_account_info(env, self.app, swift_source='TU')
return get_tempurl_keys_from_metadata(account_info['meta'])
