def object_request(self, req, version, account, container, obj, allow_versioned_writes): account_name = unquote(account) container_name = unquote(container) object_name = unquote(obj) container_info = None resp = None is_enabled = config_true_value(allow_versioned_writes) if req.method in ('PUT', 'DELETE'): container_info = get_container_info( req.environ, self.app) elif req.method == 'COPY' and 'Destination' in req.headers: if 'Destination-Account' in req.headers: account_name = req.headers.get('Destination-Account') account_name = check_account_format(req, account_name) container_name, object_name = check_destination_header(req) req.environ['PATH_INFO'] = "/%s/%s/%s/%s" % ( version, account_name, container_name, object_name) container_info = get_container_info( req.environ, self.app) if not container_info: return self.app # To maintain backwards compatibility, container version # location could be stored as sysmeta or not, need to check both. # If stored as sysmeta, check if middleware is enabled. If sysmeta # is not set, but versions property is set in container_info, then # for backwards compatibility feature is enabled. object_versions = container_info.get( 'sysmeta', {}).get('versions-location') if object_versions and isinstance(object_versions, unicode): object_versions = object_versions.encode('utf-8') elif not object_versions: object_versions = container_info.get('versions') # if allow_versioned_writes is not set in the configuration files # but 'versions' is configured, enable feature to maintain # backwards compatibility if not allow_versioned_writes and object_versions: is_enabled = True if is_enabled and object_versions: object_versions = unquote(object_versions) vw_ctx = VersionedWritesContext(self.app, self.logger) if req.method in ('PUT', 'COPY'): policy_idx = req.headers.get( 'X-Backend-Storage-Policy-Index', container_info['storage_policy']) resp = vw_ctx.handle_obj_versions_put( req, object_versions, object_name, policy_idx) else: # handle DELETE resp = vw_ctx.handle_obj_versions_delete( req, object_versions, account_name, container_name, object_name) if resp: return resp else: return self.app
def object_request(self, req, version, account, container, obj, allow_versioned_writes): account_name = unquote(account) container_name = unquote(container) object_name = unquote(obj) container_info = None resp = None is_enabled = config_true_value(allow_versioned_writes) if req.method in ('PUT', 'DELETE'): container_info = get_container_info( req.environ, self.app) elif req.method == 'COPY' and 'Destination' in req.headers: if 'Destination-Account' in req.headers: account_name = req.headers.get('Destination-Account') account_name = check_account_format(req, account_name) container_name, object_name = check_destination_header(req) req.environ['PATH_INFO'] = "/%s/%s/%s/%s" % ( version, account_name, container_name, object_name) container_info = get_container_info( req.environ, self.app) if not container_info: return self.app # To maintain backwards compatibility, container version # location could be stored as sysmeta or not, need to check both. # If stored as sysmeta, check if middleware is enabled. If sysmeta # is not set, but versions property is set in container_info, then # for backwards compatibility feature is enabled. object_versions = container_info.get( 'sysmeta', {}).get('versions-location') if object_versions and isinstance(object_versions, six.text_type): object_versions = object_versions.encode('utf-8') elif not object_versions: object_versions = container_info.get('versions') # if allow_versioned_writes is not set in the configuration files # but 'versions' is configured, enable feature to maintain # backwards compatibility if not allow_versioned_writes and object_versions: is_enabled = True if is_enabled and object_versions: object_versions = unquote(object_versions) vw_ctx = VersionedWritesContext(self.app, self.logger) if req.method in ('PUT', 'COPY'): policy_idx = req.headers.get( 'X-Backend-Storage-Policy-Index', container_info['storage_policy']) resp = vw_ctx.handle_obj_versions_put( req, object_versions, object_name, policy_idx) else: # handle DELETE resp = vw_ctx.handle_obj_versions_delete( req, object_versions, account_name, container_name, object_name) if resp: return resp else: return self.app
def object_request(self, req, api_version, account, container, obj, allow_versioned_writes): account_name = unquote(account) container_name = unquote(container) object_name = unquote(obj) container_info = None resp = None is_enabled = config_true_value(allow_versioned_writes) if req.method in ('PUT', 'DELETE'): container_info = get_container_info(req.environ, self.app) elif req.method == 'COPY' and 'Destination' in req.headers: if 'Destination-Account' in req.headers: account_name = req.headers.get('Destination-Account') account_name = check_account_format(req, account_name) container_name, object_name = check_destination_header(req) req.environ['PATH_INFO'] = "/%s/%s/%s/%s" % ( api_version, account_name, container_name, object_name) container_info = get_container_info(req.environ, self.app) if not container_info: return self.app # To maintain backwards compatibility, container version # location could be stored as sysmeta or not, need to check both. # If stored as sysmeta, check if middleware is enabled. If sysmeta # is not set, but versions property is set in container_info, then # for backwards compatibility feature is enabled. versions_cont = container_info.get('sysmeta', {}).get('versions-location') if not versions_cont: versions_cont = container_info.get('versions') # if allow_versioned_writes is not set in the configuration files # but 'versions' is configured, enable feature to maintain # backwards compatibility if not allow_versioned_writes and versions_cont: is_enabled = True if is_enabled and versions_cont: versions_cont = unquote(versions_cont).split('/')[0] vw_ctx = VersionedWritesContext(self.app, self.logger) if req.method in ('PUT', 'COPY'): resp = vw_ctx.handle_obj_versions_put(req, versions_cont, api_version, account_name, object_name) else: # handle DELETE resp = vw_ctx.handle_obj_versions_delete( req, versions_cont, api_version, account_name, container_name, object_name) if resp: return resp else: return self.app
def __call__(self, request): try: (version, account, container, objname) = split_path( request.path_info, 1, 4, True) except ValueError: return self.app(environ, start_response) if container and not objname: if request.method in ('DELETE', 'HEAD'): return self.app if request.method == 'POST': # Deny setting if there are any objects in base container # Otherwise these objects won't be visible if request.headers.get('X-Container-Meta-Storage-Path'): container_info = get_container_info(request.environ, self.app) objects = container_info.get('object_count') if objects and int(objects) > 0: return HTTPBadRequest() # ACL set groups = (request.remote_user or '').split(',') account_name = groups[0].split(':')[0] read_acl = request.environ.get('HTTP_X_CONTAINER_READ', '') for target_account in read_acl.split(','): target_account = target_account.split(':')[0] target_storage_path = self._get_storage_path(request, target_account) if not target_storage_path or account_name == target_account: continue container_path = "/%s/%s/%s" % (version, account, container) headers = {'X-Container-Meta-Storage-Path': container_path} request_path = "%s/%s%s_%s" % (target_storage_path, self.prefix, account_name, container) req = make_pre_authed_request(request.environ, 'PUT', request_path, headers=headers) req.get_response(self.app) if container: container_info = get_container_info(request.environ, self.app) meta = container_info.get('meta', {}) storage_path = meta.get('storage-path') if storage_path: if objname: storage_path += '/' + objname request.environ['PATH_INFO'] = storage_path request.environ['RAW_PATH_INFO'] = storage_path return self.app
def apply_storage_quota(self, req, service_plan, account_info, ver, account, container, obj): if not obj: quota = service_plan['containers'] # If "number of containers" = (quota + 1): deny PUT # We don't want to deny overwrite of the last container new_size = int(account_info['container_count']) if 0 <= quota < new_size: return bad_response( req, None, 'Over quota: containers') return None content_length = (req.content_length or 0) if req.method == 'COPY': copy_from = container + '/' + obj else: copy_from = req.headers.get('X-Copy-From') container_info = None if copy_from: copy_account = req.headers.get('X-Copy-From-Account', account) path = '/' + ver + '/' + copy_account + '/' + copy_from.lstrip('/') # We are copying from another account # Let's not leak the existence of the remote object # to the unauthorized user if copy_account != account: container_info = get_container_info(req.environ, self.app, swift_source='litequota') aresp = check_acl(req, container_info, 'read_acl') if aresp: return aresp object_info = get_object_info(req.environ, self.app, path) if not object_info or not object_info['length']: content_length = 0 else: content_length = int(object_info['length']) new_size = int(account_info['bytes']) + content_length quota = service_plan['bytes'] if 0 <= quota < new_size: if not container_info: container_info = get_container_info(req.environ, self.app, swift_source='litequota') return bad_response(req, container_info, 'Over quota: bytes') # If "number of objects" == (quota + 1): deny PUT # We don't want to deny overwrite of the last object new_size = int(account_info['total_object_count']) quota = service_plan['objects'] if 0 <= quota < new_size: if not container_info: container_info = get_container_info(req.environ, self.app, swift_source='litequota') return bad_response(req, container_info, 'Over quota: objects')
def __call__(self, env, start_response): ctx = WSGIContext(self.app) app_iter = ctx._app_call(env) try: split_path(env['PATH_INFO'], 4, 4, True) except ValueError: pass # not an object request; don't care else: if env['REQUEST_METHOD'] == 'DELETE' and \ ctx._response_status[:3] == '404': # Should be a cache hit if is_success( get_container_info(env, self.app, swift_source='S3').get('status')): # Convert to a successful response ctx._response_status = '204 No Content' ctx._response_headers = [ (h, '0' if h.lower() == 'content-length' else v) for h, v in ctx._response_headers ] with closing_if_possible(app_iter): for chunk in app_iter: pass # should be short; just drop it on the floor app_iter = [''] start_response(ctx._response_status, ctx._response_headers) return app_iter
def test_get_container_info_no_cache(self): req = Request.blank("/v1/AUTH_account/cont", environ={'swift.cache': FakeCache({})}) resp = get_container_info(req.environ, FakeApp()) self.assertEqual(resp['storage_policy'], 0) self.assertEqual(resp['bytes'], 6666) self.assertEqual(resp['object_count'], 1000)
def test_get_container_info_no_auto_account(self): app = FakeApp(statuses=[200]) req = Request.blank("/v1/.system_account/cont") info = get_container_info(req.environ, app) self.assertEqual(info['status'], 200) self.assertEqual(info['bytes'], 6666) self.assertEqual(info['object_count'], 1000)
def __call__(self, env, start_response): req = Request(env) try: version, account, container, obj = req.split_path( 2, 4, rest_with_last=True) is_swifty_request = valid_api_version(version) except ValueError: is_swifty_request = False if not is_swifty_request: return self.app(env, start_response) if not obj: typ = 'Container' if container else 'Account' client_header = 'X-%s-Rfc-Compliant-Etags' % typ sysmeta_header = 'X-%s-Sysmeta-Rfc-Compliant-Etags' % typ if client_header in req.headers: if req.headers[client_header]: req.headers[sysmeta_header] = config_true_value( req.headers[client_header]) else: req.headers[sysmeta_header] = '' if req.headers.get(client_header.replace('X-', 'X-Remove-', 1)): req.headers[sysmeta_header] = '' def translating_start_response(status, headers, exc_info=None): return start_response( status, [(client_header if h.title() == sysmeta_header else h, v) for h, v in headers], exc_info) return self.app(env, translating_start_response) container_info = get_container_info(env, self.app, 'EQ') if not container_info or not is_success(container_info['status']): return self.app(env, start_response) flag = container_info.get('sysmeta', {}).get('rfc-compliant-etags') if flag is None: account_info = get_account_info(env, self.app, 'EQ') if not account_info or not is_success(account_info['status']): return self.app(env, start_response) flag = account_info.get('sysmeta', {}).get('rfc-compliant-etags') if flag is None: flag = self.conf.get('enable_by_default', 'false') if not config_true_value(flag): return self.app(env, start_response) status, headers, resp_iter = req.call_application(self.app) headers = [(header, value) if header.lower() != 'etag' or (value.startswith( ('"', 'W/"')) and value.endswith('"')) else (header, '"%s"' % value) for header, value in headers] start_response(status, headers) return resp_iter
def __call__(self, env, start_response): request = Request(env) if not request.path.startswith(self.endpoints_path): return self.app(env, start_response) if request.method != 'GET': return HTTPMethodNotAllowed(req=request, headers={"Allow": "GET"})(env, start_response) try: version, account, container, obj = self._parse_path(request) except ValueError as err: return HTTPBadRequest(str(err))(env, start_response) if account is not None: account = unquote(account) if container is not None: container = unquote(container) if obj is not None: obj = unquote(obj) storage_policy_index = None if obj is not None: container_info = get_container_info( {'PATH_INFO': '/v1/%s/%s' % (account, container)}, self.app, swift_source='LE') storage_policy_index = container_info['storage_policy'] obj_ring = self.get_object_ring(storage_policy_index) partition, nodes = obj_ring.get_nodes(account, container, obj) endpoint_template = 'http://{ip}:{port}/{device}/{partition}/' + \ '{account}/{container}/{obj}' elif container is not None: partition, nodes = self.container_ring.get_nodes( account, container) endpoint_template = 'http://{ip}:{port}/{device}/{partition}/' + \ '{account}/{container}' else: partition, nodes = self.account_ring.get_nodes(account) endpoint_template = 'http://{ip}:{port}/{device}/{partition}/' + \ '{account}' endpoints = [] for node in nodes: endpoint = endpoint_template.format(ip=node['ip'], port=node['port'], device=node['device'], partition=partition, account=quote(account), container=quote(container or ''), obj=quote(obj or '')) endpoints.append(endpoint) resp = self.response_map[version]( request, endpoints=endpoints, storage_policy_index=storage_policy_index) return resp(env, start_response)
def POST(self, env): """Handle posts dealing with metadata alteration""" req = Request(env) conn = HTTPConnection('%s:%s' % (self.mds_ip, self.mds_port)) headers = req.params version, acc, con, obj = split_path(req.path, 1, 4, True) if not con: try: info = get_account_info(env, self.app) if info: stor_policy = info['storage_policy'] headers['storage_policy'] = stor_policy except: pass else: try: info = get_container_info(env, self.app) if info: stor_policy = info['storage_policy'] headers['storage_policy'] = stor_policy except: pass conn.request('POST', req.path, headers=headers) resp = conn.getresponse() #confirm response then pass along the request return self.app
def __call__(self, env, start_response): request = Request(env) if request.method == "PUT": # Ensure a possibly cached CONTENT_TYPE will be cleared if env.get('CONTENT_TYPE'): del env['CONTENT_TYPE'] container_info = get_container_info( request.environ, self.app, swift_source='AM') if not container_info or not is_success(container_info['status']): return self.app(env, start_response) meta = container_info.get('meta', {}) enabled = meta.get('automime') if not enabled: return self.app(env, start_response) _type, encoding = mimetypes.guess_type(request.path) if _type: env['HTTP_CONTENT_TYPE'] = _type if encoding: env['HTTP_CONTENT_ENCODING'] = encoding return self.app(env, start_response)
def get_defaults(self, req, req_type, format_args): acct_sysmeta = get_account_info(req.environ, self.app)['sysmeta'] if req_type == 'object': cont_sysmeta = get_container_info(req.environ, self.app)['sysmeta'] else: cont_sysmeta = {} defaults = {} prefix = 'default-%s-' % req_type for src in (self.conf, acct_sysmeta, cont_sysmeta): for key, value in src.items(): if not key.lower().startswith(prefix): continue header_to_default = key[len(prefix):].lower() if header_to_default.startswith(BLACKLIST_PREFIXES): continue if header_to_default in BLACKLIST: continue if self.conf['use_formatting']: try: value = value.format(**format_args) except KeyError: # This user may not have specified the default; # don't fail because of someone else pass defaults[header_to_default] = value return defaults
def _get_keys(self, env): """ Returns the X-[Account|Container]-Meta-Temp-URL-Key[-2] header values for the account or container, or an empty list if none are set. Each value comes as a 2-tuple (key, scope), where scope is either CONTAINER_SCOPE or ACCOUNT_SCOPE. Returns 0-4 elements depending on how many keys are set in the account's or container's metadata. :param env: The WSGI environment for the request. :returns: [ (X-Account-Meta-Temp-URL-Key str value, ACCOUNT_SCOPE) if set, (X-Account-Meta-Temp-URL-Key-2 str value, ACCOUNT_SCOPE if set, (X-Container-Meta-Temp-URL-Key str value, CONTAINER_SCOPE) if set, (X-Container-Meta-Temp-URL-Key-2 str value, CONTAINER_SCOPE if set, ] """ account_info = get_account_info(env, self.app, swift_source='TU') account_keys = get_tempurl_keys_from_metadata(account_info['meta']) container_info = get_container_info(env, self.app, swift_source='TU') container_keys = get_tempurl_keys_from_metadata( container_info.get('meta', [])) return ([(ak, ACCOUNT_SCOPE) for ak in account_keys] + [(ck, CONTAINER_SCOPE) for ck in container_keys])
def test_get_container_info_no_cache(self): req = Request.blank("/v1/AUTH_account/cont", environ={'swift.cache': FakeCache({})}) resp = get_container_info(req.environ, FakeApp()) self.assertEqual(resp['storage_policy'], '0') self.assertEqual(resp['bytes'], 6666) self.assertEqual(resp['object_count'], 1000)
def test_get_container_info_no_cache(self): swift.proxy.controllers.base.make_pre_authed_request = FakeRequest req = Request.blank("/v1/AUTH_account/cont", environ={'swift.cache': FakeCache({})}) resp = get_container_info(req.environ, 'xxx') self.assertEquals(resp['bytes'], 6666) self.assertEquals(resp['object_count'], 1000)
def get_defaults(self, req, req_type, format_args): acct_sysmeta = get_account_info(req.environ, self.app)['sysmeta'] if req_type == 'object': cont_sysmeta = get_container_info(req.environ, self.app)['sysmeta'] else: cont_sysmeta = {} defaults = {} prefix = 'default-%s-' % req_type for src in (self.conf, acct_sysmeta, cont_sysmeta): for key, value in src.items(): if not key.lower().startswith(prefix): continue header_to_default = key[len(prefix):].lower() if header_to_default.startswith(BLACKLIST_PREFIXES): continue if header_to_default in BLACKLIST: continue if self.conf['use_formatting']: try: value = value.format(**format_args) except KeyError: # This user may not have specified the default; # don't fail because of someone else pass defaults[header_to_default] = value return defaults
def _get_keys(self, env): """ Returns the X-[Account|Container]-Meta-Temp-URL-Key[-2] header values for the account or container, or an empty list if none are set. Returns 0-4 elements depending on how many keys are set in the account's or container's metadata. Also validate that the request path indicates a valid container; if not, no keys will be returned. :param env: The WSGI environment for the request. :returns: list of tempurl keys """ parts = env['PATH_INFO'].split('/', 4) if len(parts) < 4 or parts[0] or not valid_api_version(parts[1]) \ or not parts[2] or not parts[3]: return [] account_info = get_account_info(env, self.app, swift_source='FP') account_keys = get_tempurl_keys_from_metadata(account_info['meta']) container_info = get_container_info(env, self.app, swift_source='FP') container_keys = get_tempurl_keys_from_metadata( container_info.get('meta', [])) return account_keys + container_keys
def POST(self, env): """Handle posts dealing with metadata alteration""" req = Request(env) conn = HTTPConnection('%s:%s' % (self.mds_ip, self.mds_port)) headers = req.params version, acc, con, obj = split_path(req.path, 1, 4, True) if not con: try: info = get_account_info(env, self.app) if info: stor_policy = info['storage_policy'] headers['storage_policy'] = stor_policy except: pass else: try: info = get_container_info(env, self.app) if info: stor_policy = info['storage_policy'] headers['storage_policy'] = stor_policy except: pass conn.request('POST', req.path, headers=headers) resp = conn.getresponse() #confirm response then pass along the request return self.app
def test_get_container_info_no_cache(self): swift.proxy.controllers.base.make_pre_authed_request = FakeRequest req = Request.blank("/v1/AUTH_account/cont", environ={'swift.cache': FakeCache({})}) resp = get_container_info(req.environ, 'xxx') self.assertEquals(resp['bytes'], 6666) self.assertEquals(resp['object_count'], 1000)
def get_container_size(self, env): rv = 0 container_info = get_container_info(env, self.app, swift_source='RL') if isinstance(container_info, dict): rv = container_info.get('object_count', container_info.get('container_size', 0)) return rv
def __call__(self, env, start_response): request = Request(env) if request.method == "PUT": # Ensure a possibly cached CONTENT_TYPE will be cleared if env.get('CONTENT_TYPE'): del env['CONTENT_TYPE'] container_info = get_container_info(request.environ, self.app, swift_source='AM') if not container_info or not is_success(container_info['status']): return self.app(env, start_response) meta = container_info.get('meta', {}) enabled = meta.get('automime') if not enabled: return self.app(env, start_response) _type, encoding = mimetypes.guess_type(request.path) if _type: env['HTTP_CONTENT_TYPE'] = _type if encoding: env['HTTP_CONTENT_ENCODING'] = encoding return self.app(env, start_response)
def test_get_container_info_swift_source(self): req = Request.blank("/v1/a/c", environ={'swift.cache': FakeCache({})}) with patch( 'swift.proxy.controllers.base.' '_prepare_pre_auth_info_request', FakeRequest): resp = get_container_info(req.environ, 'app', swift_source='MC') self.assertEquals(resp['meta']['fakerequest-swift-source'], 'MC')
def test_get_container_info_no_auto_account(self): app = FakeApp(statuses=[200]) req = Request.blank("/v1/.system_account/cont") info = get_container_info(req.environ, app) self.assertEqual(info['status'], 200) self.assertEqual(info['bytes'], 6666) self.assertEqual(info['object_count'], 1000)
def get_container_info(self, app): """ get_container_info will return a result dict of get_container_info from the backend Swift. :returns: a dictionary of container info from swift.controllers.base.get_container_info :raises: NoSuchBucket when the container doesn't exist :raises: InternalError when the request failed without 404 """ if self.is_authenticated: # if we have already authenticated, yes we can use the account # name like as AUTH_xxx for performance efficiency sw_req = self.to_swift_req(app, self.container_name, None) info = get_container_info(sw_req.environ, app) if is_success(info['status']): return info elif info['status'] == 404: raise NoSuchBucket(self.container_name) else: raise InternalError( 'unexpected status code %d' % info['status']) else: # otherwise we do naive HEAD request with the authentication resp = self.get_response(app, 'HEAD', self.container_name, '') return headers_to_container_info( resp.sw_headers, resp.status_int) # pylint: disable-msg=E1101
def _get_keys(self, env): """ Returns the X-[Account|Container]-Meta-Temp-URL-Key[-2] header values for the account or container, or an empty list if none are set. Returns 0-4 elements depending on how many keys are set in the account's or container's metadata. Also validate that the request path indicates a valid container; if not, no keys will be returned. :param env: The WSGI environment for the request. :returns: list of tempurl keys """ parts = env["PATH_INFO"].split("/", 4) if len(parts) < 4 or parts[0] or parts[1] != "v1" or not parts[2] or not parts[3]: return [] account_info = get_account_info(env, self.app, swift_source="FP") account_keys = get_tempurl_keys_from_metadata(account_info["meta"]) container_info = get_container_info(env, self.app, swift_source="FP") container_keys = get_tempurl_keys_from_metadata(container_info.get("meta", [])) return account_keys + container_keys
def __call__(self, env, start_response): request = Request(env) if not request.path.startswith(self.endpoints_path): return self.app(env, start_response) if request.method != 'GET': return HTTPMethodNotAllowed( req=request, headers={"Allow": "GET"})(env, start_response) try: version, account, container, obj = self._parse_path(request) except ValueError as err: return HTTPBadRequest(str(err))(env, start_response) if account is not None: account = unquote(account) if container is not None: container = unquote(container) if obj is not None: obj = unquote(obj) storage_policy_index = None if obj is not None: container_info = get_container_info( {'PATH_INFO': '/v1/%s/%s' % (account, container)}, self.app, swift_source='LE') storage_policy_index = container_info['storage_policy'] obj_ring = self.get_object_ring(storage_policy_index) partition, nodes = obj_ring.get_nodes( account, container, obj) endpoint_template = 'http://{ip}:{port}/{device}/{partition}/' + \ '{account}/{container}/{obj}' elif container is not None: partition, nodes = self.container_ring.get_nodes( account, container) endpoint_template = 'http://{ip}:{port}/{device}/{partition}/' + \ '{account}/{container}' else: partition, nodes = self.account_ring.get_nodes( account) endpoint_template = 'http://{ip}:{port}/{device}/{partition}/' + \ '{account}' endpoints = [] for node in nodes: endpoint = endpoint_template.format( ip=node['ip'], port=node['port'], device=node['device'], partition=partition, account=quote(account), container=quote(container or ''), obj=quote(obj or '')) endpoints.append(endpoint) resp = self.response_map[version]( request, endpoints=endpoints, storage_policy_index=storage_policy_index) return resp(env, start_response)
def test_get_container_info_env(self): cache_key = get_container_memcache_key("account", "cont") env_key = 'swift.%s' % cache_key req = Request.blank("/v1/account/cont", environ={env_key: {'bytes': 3867}, 'swift.cache': FakeCache({})}) resp = get_container_info(req.environ, 'xxx') self.assertEquals(resp['bytes'], 3867)
def test_get_container_info_no_auto_account(self): responses = DynamicResponseFactory(404, 200) app = FakeApp(responses) req = Request.blank("/v1/.system_account/cont") info = get_container_info(req.environ, app) self.assertEqual(info['status'], 200) self.assertEqual(info['bytes'], 6666) self.assertEqual(info['object_count'], 1000)
def test_get_container_info_env(self): cache_key = get_cache_key("account", "cont") req = Request.blank( "/v1/account/cont", environ={'swift.infocache': {cache_key: {'bytes': 3867}}, 'swift.cache': FakeCache({})}) resp = get_container_info(req.environ, 'xxx') self.assertEqual(resp['bytes'], 3867)
def get_container_size(self, env): rv = 0 container_info = get_container_info( env, self.app, swift_source='RL') if isinstance(container_info, dict): rv = container_info.get( 'object_count', container_info.get('container_size', 0)) return rv
def test_get_container_info_cache(self): cached = {"status": 404, "bytes": 3333, "object_count": 10} req = Request.blank("/v1/account/cont", environ={"swift.cache": FakeCache(cached)}) with patch("swift.proxy.controllers.base." "_prepare_pre_auth_info_request", FakeRequest): resp = get_container_info(req.environ, "xxx") self.assertEquals(resp["bytes"], 3333) self.assertEquals(resp["object_count"], 10) self.assertEquals(resp["status"], 404)
def test_get_container_info_no_auto_account(self): responses = DynamicResponseFactory(200) app = FakeApp(responses) req = Request.blank("/v1/.system_account/cont") info = get_container_info(req.environ, app) self.assertEqual(info['status'], 200) self.assertEqual(info['bytes'], 6666) self.assertEqual(info['object_count'], 1000)
def object_request(self, req, api_version, account, container, obj, allow_versioned_writes): """ Handle request for object resource. Note that account, container, obj should be unquoted by caller if the url path is under url encoding (e.g. %FF) :param req: swift.common.swob.Request instance :param api_version: should be v1 unless swift bumps api version :param account: account name string :param container: container name string :param object: object name string """ resp = None is_enabled = config_true_value(allow_versioned_writes) container_info = get_container_info( req.environ, self.app) # To maintain backwards compatibility, container version # location could be stored as sysmeta or not, need to check both. # If stored as sysmeta, check if middleware is enabled. If sysmeta # is not set, but versions property is set in container_info, then # for backwards compatibility feature is enabled. versions_cont = container_info.get( 'sysmeta', {}).get('versions-location') versioning_mode = container_info.get( 'sysmeta', {}).get('versions-mode', 'stack') if not versions_cont: versions_cont = container_info.get('versions') # if allow_versioned_writes is not set in the configuration files # but 'versions' is configured, enable feature to maintain # backwards compatibility if not allow_versioned_writes and versions_cont: is_enabled = True if is_enabled and versions_cont: versions_cont = wsgi_unquote(str_to_wsgi( versions_cont)).split('/')[0] vw_ctx = VersionedWritesContext(self.app, self.logger) if req.method == 'PUT': resp = vw_ctx.handle_obj_versions_put( req, versions_cont, api_version, account, obj) # handle DELETE elif versioning_mode == 'history': resp = vw_ctx.handle_obj_versions_delete_push( req, versions_cont, api_version, account, container, obj) else: resp = vw_ctx.handle_obj_versions_delete_pop( req, versions_cont, api_version, account, container, obj) if resp: return resp else: return self.app
def object_request(self, req, api_version, account, container, obj, allow_versioned_writes): """ Handle request for object resource. Note that account, container, obj should be unquoted by caller if the url path is under url encoding (e.g. %FF) :param req: swift.common.swob.Request instance :param api_version: should be v1 unless swift bumps api version :param account: account name string :param container: container name string :param object: object name string """ resp = None is_enabled = config_true_value(allow_versioned_writes) container_info = get_container_info( req.environ, self.app, swift_source='VW') # To maintain backwards compatibility, container version # location could be stored as sysmeta or not, need to check both. # If stored as sysmeta, check if middleware is enabled. If sysmeta # is not set, but versions property is set in container_info, then # for backwards compatibility feature is enabled. versions_cont = container_info.get( 'sysmeta', {}).get('versions-location') versioning_mode = container_info.get( 'sysmeta', {}).get('versions-mode', 'stack') if not versions_cont: versions_cont = container_info.get('versions') # if allow_versioned_writes is not set in the configuration files # but 'versions' is configured, enable feature to maintain # backwards compatibility if not allow_versioned_writes and versions_cont: is_enabled = True if is_enabled and versions_cont: versions_cont = wsgi_unquote(str_to_wsgi( versions_cont)).split('/')[0] vw_ctx = VersionedWritesContext(self.app, self.logger) if req.method == 'PUT': resp = vw_ctx.handle_obj_versions_put( req, versions_cont, api_version, account, obj) # handle DELETE elif versioning_mode == 'history': resp = vw_ctx.handle_obj_versions_delete_push( req, versions_cont, api_version, account, container, obj) else: resp = vw_ctx.handle_obj_versions_delete_pop( req, versions_cont, api_version, account, container, obj) if resp: return resp else: return self.app
def PUT(self, req): try: container_info = get_container_info(req.environ, self._app) req.acl = container_info['write_acl'] return self.try_deny(req) or self.clean_acls(req) or \ self.forward_request(req) except Exception as ex: print ex
def test_get_container_info_cache(self): swift.proxy.controllers.base.make_pre_authed_request = FakeRequest cached = {'status': 404, 'bytes': 3333, 'object_count': 10} req = Request.blank("/v1/account/cont", environ={'swift.cache': FakeCache(cached)}) resp = get_container_info(req.environ, 'xxx') self.assertEquals(resp['bytes'], 3333) self.assertEquals(resp['object_count'], 10) self.assertEquals(resp['status'], 404)
def get_controller(self, req): """ Get the controller to handle a request. :param req: the request :returns: tuple of (controller class, path dictionary) :raises ValueError: (thrown by split_path) if given invalid path """ if req.path == '/info': d = dict(version=None, expose_info=self.expose_info, disallowed_sections=self.disallowed_sections, admin_key=self.admin_key) return InfoController, d #分割请求路径,eg. http://127.0.0.1:8080/auth/v1.0/account/container version, account, container, obj = split_path(req.path, 1, 4, True) d = dict(version=version, account_name=account, container_name=container, object_name=obj) # 如果account为空或者版本号不对,则抛出异常 if account and not valid_api_version(version): raise APIVersionError('Invalid path') # 如果account,container,object都存在,表明对object操作,则返回 object controller if obj and container and account: # info={"status": ..., "sync_key": null, "write_acl": null, "object_count": 1, # "storage_policy": 0, "versions": null, "bytes": ..., "meta": {}, "sharding_state": ..., # "cors": {"allow_origin": null, "expose_headers": null, "max_age": null}, # "sysmeta": {}, "read_acl": null} info = get_container_info(req.environ, self) policy_index = req.headers.get('X-Backend-Storage-Policy-Index', info['storage_policy']) policy = POLICIES.get_by_index(policy_index) if not policy: # This indicates that a new policy has been created, # with rings, deployed, released (i.e. deprecated = # False), used by a client to create a container via # another proxy that was restarted after the policy # was released, and is now cached - all before this # worker was HUPed to stop accepting new # connections. There should never be an "unknown" # index - but when there is - it's probably operator # error and hopefully temporary. raise HTTPServiceUnavailable('Unknown Storage Policy') # obj_controller_router[policy] 等价于调用obj_controller_router类中的 __getitem__(policy) # 根据策略选择返回 ECObjectController 对象 还是 ReplicatedObjectController 对象 return self.obj_controller_router[policy], d # 如果account,container都存在,表明对container操作,则返回 container controller elif container and account: return ContainerController, d # 如果只存在account,表明对account操作,则返回 account controller elif account and not container and not obj: return AccountController, d return None, d
def get_controller(self, req): """ Get the controller to handle a request. :param req: the request :returns: tuple of (controller class, path dictionary) :raises: ValueError (thrown by split_path) if given invalid path """ print 'req.path',req.path if req.path == '/info': d = dict(version=None, expose_info=self.expose_info, disallowed_sections=self.disallowed_sections, admin_key=self.admin_key) print 'd',d return InfoController, d version, account, container, obj = split_path(req.path, 1, 4, True) d = dict(version=version, account_name=account, container_name=container, object_name=obj) print 'd',d #print 'valid_api_version(version)',valid_api_version(version) if account and not valid_api_version(version): raise APIVersionError('Invalid path') if obj and container and account: info = get_container_info(req.environ, self) print 'info of obj,Acc,Con',info policy_index = req.headers.get('X-Backend-Storage-Policy-Index', info['storage_policy']) print 'policy_index',policy_index policy = POLICIES.get_by_index(policy_index) print 'policy',policy if not policy: # This indicates that a new policy has been created, # with rings, deployed, released (i.e. deprecated = # False), used by a client to create a container via # another proxy that was restarted after the policy # was released, and is now cached - all before this # worker was HUPed to stop accepting new # connections. There should never be an "unknown" # index - but when there is - it's probably operator # error and hopefully temporary. raise HTTPServiceUnavailable('Unknown Storage Policy') return self.obj_controller_router[policy], d elif container and account: print 'container & account, returning containercontroller',container,account return ContainerController, d elif account and not container and not obj: print 'account, returning accountcontroller',account return AccountController, d return None, d
def test_get_container_info_cache(self): swift.proxy.controllers.base.make_pre_authed_request = FakeRequest cached = {'status': 404, 'bytes': 3333, 'object_count': 10} req = Request.blank("/v1/account/cont", environ={'swift.cache': FakeCache(cached)}) resp = get_container_info(req.environ, 'xxx') self.assertEquals(resp['bytes'], 3333) self.assertEquals(resp['object_count'], 10) self.assertEquals(resp['status'], 404)
def handle_delete(self, req, start_response): """ Handle request to delete a user's container. As part of deleting a container, this middleware will also delete the hidden container holding object versions. Before a user's container can be deleted, swift must check if there are still old object versions from that container. Only after disabling versioning and deleting *all* object versions can a container be deleted. """ container_info = get_container_info(req.environ, self.app, swift_source='OV') versions_cont = unquote( container_info.get('sysmeta', {}).get('versions-container', '')) if versions_cont: account = req.split_path(3, 3, True)[1] # using a HEAD request here as opposed to get_container_info # to make sure we get an up-to-date value versions_req = make_pre_authed_request( req.environ, method='HEAD', swift_source='OV', path=wsgi_quote('/v1/%s/%s' % (account, str_to_wsgi(versions_cont))), headers={'X-Backend-Allow-Reserved-Names': 'true'}) vresp = versions_req.get_response(self.app) drain_and_close(vresp) if vresp.is_success and int( vresp.headers.get('X-Container-Object-Count', 0)) > 0: raise HTTPConflict( 'Delete all versions before deleting container.', request=req) elif not vresp.is_success and vresp.status_int != 404: raise HTTPInternalServerError( 'Error deleting versioned container') else: versions_req.method = 'DELETE' resp = versions_req.get_response(self.app) drain_and_close(resp) if not is_success(resp.status_int) and resp.status_int != 404: raise HTTPInternalServerError( 'Error deleting versioned container') app_resp = self._app_call(req.environ) start_response(self._response_status, self._response_headers, self._response_exc_info) return app_resp
def list_containers_iter(self, *args, **kwargs): """ Returns a list of containers the user has access to """ try: version, account = self.request.split_path(2, 2) except ValueError: pass path = "/%s/%s?format=json" % (version, account) for key in ('limit', 'marker', 'end_marker', 'prefix', 'delimiter'): value = self.request.params.get(key) if value: path+= '&%s=%s' % (key, value) if self.memcache_client is None: self.memcache_client = cache_from_env(self.request.environ) memcache_key = 'containerlist%s%s' % (path, str(self.groups)) containers = self.memcache_client.get(memcache_key) if containers is not None: return containers req = make_pre_authed_request(self.request.environ, 'GET', path) resp = req.get_response(self.app) tmp_containers = json.loads(resp.body) # No cached result? -> ratelimit request to prevent abuse memcache_key_sleep = 'containerlist_sleep/%s' % self.account last_request_time = self.memcache_client.get(memcache_key_sleep) if last_request_time and len(tmp_containers) > 0: last_request = time.time() - last_request_time if last_request < self.min_sleep: eventlet.sleep(self.min_sleep - last_request) containers = [] for container in tmp_containers: tmp_env = copy.copy(self.request.environ) container_name = container['name'].encode("utf8") path_info = "/%s/%s/%s" % (version, account, container_name) tmp_env['PATH_INFO'] = path_info container_info = get_container_info(tmp_env, self.app) acl = (container_info.get('read_acl') or '').split(',') if (list(set(self.groups) & set(acl))): containers.append((container['name'], container['count'], container['bytes'], 0)) self.memcache_client.set(memcache_key, containers, time=60) self.memcache_client.set(memcache_key_sleep, time.time()) return containers
def test_get_container_info_cache(self): cache_stub = { 'status': 404, 'bytes': 3333, 'object_count': 10, 'versions': u"\u1F4A9"} req = Request.blank("/v1/account/cont", environ={'swift.cache': FakeCache(cache_stub)}) resp = get_container_info(req.environ, FakeApp()) self.assertEqual(resp['storage_policy'], 0) self.assertEqual(resp['bytes'], 3333) self.assertEqual(resp['object_count'], 10) self.assertEqual(resp['status'], 404) self.assertEqual(resp['versions'], "\xe1\xbd\x8a\x39")
def test_get_container_info_cache(self): cache_stub = { 'status': 404, 'bytes': 3333, 'object_count': 10, # simplejson sometimes hands back strings, sometimes unicodes 'versions': u"\u1F4A9"} req = Request.blank("/v1/account/cont", environ={'swift.cache': FakeCache(cache_stub)}) resp = get_container_info(req.environ, FakeApp()) self.assertEquals(resp['bytes'], 3333) self.assertEquals(resp['object_count'], 10) self.assertEquals(resp['status'], 404) self.assertEquals(resp['versions'], "\xe1\xbd\x8a\x39")
def get_controller(self, req): """ Get the controller to handle a request. :param req: the request :returns: tuple of (controller class, path dictionary) :raises: ValueError (thrown by split_path) if given invalid path """ if req.path == '/info': d = dict(version=None, expose_info=self.expose_info, disallowed_sections=self.disallowed_sections, admin_key=self.admin_key) return InfoController, d #分割路径信息 version, account, container, obj = split_path(req.path, 1, 4, True) #生成包含version、account、container、object的路径字典,用于返回 d = dict(version=version, account_name=account, container_name=container, object_name=obj) if account and not valid_api_version(version): raise APIVersionError('Invalid path') #如果是对象操作 if obj and container and account: #获取container信息 info = get_container_info(req.environ, self) policy_index = req.headers.get('X-Backend-Storage-Policy-Index', info['storage_policy']) #通过index获取存储策略对象 policy = POLICIES.get_by_index(policy_index) if not policy: # This indicates that a new policy has been created, # with rings, deployed, released (i.e. deprecated = # False), used by a client to create a container via # another proxy that was restarted after the policy # was released, and is now cached - all before this # worker was HUPed to stop accepting new # connections. There should never be an "unknown" # index - but when there is - it's probably operator # error and hopefully temporary. raise HTTPServiceUnavailable('Unknown Storage Policy') #返回对象操作的控制器对象,以及路径字典 return self.obj_controller_router[policy], d #如果是container操作,返回container控制器,以及路径字典 elif container and account: return ContainerController, d #如果是account操作,返回account控制器,以及路径字典 elif account and not container and not obj: return AccountController, d return None, d
def test_get_container_info_cache(self): cache_stub = { 'status': 404, 'bytes': 3333, 'object_count': 10, 'versions': u"\u1F4A9"} req = Request.blank("/v1/account/cont", environ={'swift.cache': FakeCache(cache_stub)}) resp = get_container_info(req.environ, FakeApp()) self.assertEqual(resp['storage_policy'], 0) self.assertEqual(resp['bytes'], 3333) self.assertEqual(resp['object_count'], 10) self.assertEqual(resp['status'], 404) self.assertEqual(resp['versions'], "\xe1\xbd\x8a\x39")
def get_obj_storage_nodes(self, account, container, obj): container_info = get_container_info( {'PATH_INFO': '/v1/%s/%s' % (account, container)}, self.app, swift_source='LE') storage_policy_index = container_info['storage_policy'] obj_ring = self.get_object_ring(storage_policy_index) partition, nodes = obj_ring.get_nodes(account, container, obj) self.logger.debug('Storage nodes: %s' % str(nodes)) ips = [] for node in nodes: ips.append(node['ip']) return ips
def __call__(self, req): try: (version, account, container, obj) = req.split_path(3, 4, True) except ValueError: return self.app # verify new quota headers are properly formatted if not obj and req.method in ("PUT", "POST"): val = req.headers.get("X-Container-Meta-Quota-Bytes") if val and not val.isdigit(): return HTTPBadRequest(body="Invalid bytes quota.") val = req.headers.get("X-Container-Meta-Quota-Count") if val and not val.isdigit(): return HTTPBadRequest(body="Invalid count quota.") # check user uploads against quotas elif obj and req.method == "PUT": container_info = get_container_info(req.environ, self.app, swift_source="CQ") if not container_info or not is_success(container_info["status"]): # this will hopefully 404 later return self.app if ( "quota-bytes" in container_info.get("meta", {}) and "bytes" in container_info and container_info["meta"]["quota-bytes"].isdigit() ): content_length = req.content_length or 0 copy_from = req.headers.get("X-Copy-From") if copy_from: path = "/%s/%s/%s" % (version, account, copy_from.lstrip("/")) object_info = get_object_info(req.environ, self.app, path) if not object_info or not object_info["length"]: content_length = 0 else: content_length = int(object_info["length"]) new_size = int(container_info["bytes"]) + content_length if int(container_info["meta"]["quota-bytes"]) < new_size: return self.bad_response(req, container_info) if ( "quota-count" in container_info.get("meta", {}) and "object_count" in container_info and container_info["meta"]["quota-count"].isdigit() ): new_count = int(container_info["object_count"]) + 1 if int(container_info["meta"]["quota-count"]) < new_count: return self.bad_response(req, container_info) return self.app
def is_enabled_for(self, env): """ Whether an account or container has meta-data to opt out of undelete protection """ sysmeta_c = get_container_info(env, self.app)['sysmeta'] # Container info gets & caches account info, so this is basically free sysmeta_a = get_account_info(env, self.app)['sysmeta'] enabled = sysmeta_c.get(SYSMETA_UNDELETE_ENABLED) if enabled is None: enabled = sysmeta_a.get(SYSMETA_UNDELETE_ENABLED, self.enable_by_default) return utils.config_true_value(enabled)
def GETorHEAD(self, req): resp = self.forward_request(req) # Enchance the request with ACL-related stuff before trying to deny. container_info = get_container_info(req.environ, self._app) try: # The key name might be a litte misleading, so be informed it's # just an alias to well-known X-Container-Read HTTP header. # ACL-related HTTP headers (X-Container-{Read, Write}) are # converted into {read, write}_acl by headers_to_container_info(). req.acl = container_info['read_acl'] except (KeyError): pass self._app.logger.debug(str(self) + ' got cont acls = ' + str(req.acl)) return self.try_deny(req) or resp
def _get_container_info(self): """ Retrieves all x-container-meta-web-* headers, and return them as a dict. """ if not self.container: # No configurable items in account return {} if self._container_info: return self._container_info self._container_info = get_container_info(self.env, self.app, swift_source='BSW') or {} return self._container_info
def is_enabled_for(self, env): """ Whether an account or container has meta-data to opt out of undelete protection """ sysmeta_c = get_container_info(env, self.app)['sysmeta'] # Container info gets & caches account info, so this is basically free sysmeta_a = get_account_info(env, self.app)['sysmeta'] enabled = sysmeta_c.get(SYSMETA_UNDELETE_ENABLED) if enabled is None: enabled = sysmeta_a.get(SYSMETA_UNDELETE_ENABLED, self.enable_by_default) return utils.config_true_value(enabled)
def __call__(self, req): try: (version, account, container, obj) = req.split_path(3, 4, True) except ValueError: return self.app # verify new quota headers are properly formatted if not obj and req.method in ('PUT', 'POST'): val = req.headers.get('X-Container-Meta-Quota-Bytes') if val and not val.isdigit(): return HTTPBadRequest(body='Invalid bytes quota.') val = req.headers.get('X-Container-Meta-Quota-Count') if val and not val.isdigit(): return HTTPBadRequest(body='Invalid count quota.') # check user uploads against quotas elif obj and req.method == 'PUT': container_info = get_container_info(req.environ, self.app, swift_source='CQ') if not container_info or not is_success(container_info['status']): # this will hopefully 404 later return self.app if 'quota-bytes' in container_info.get('meta', {}) and \ 'bytes' in container_info and \ container_info['meta']['quota-bytes'].isdigit(): content_length = (req.content_length or 0) copy_from = req.headers.get('X-Copy-From') if copy_from: path = '/%s/%s/%s' % (version, account, copy_from.lstrip('/')) object_info = get_object_info(req.environ, self.app, path) if not object_info or not object_info['length']: content_length = 0 else: content_length = int(object_info['length']) new_size = int(container_info['bytes']) + content_length if int(container_info['meta']['quota-bytes']) < new_size: return self.bad_response(req, container_info) if 'quota-count' in container_info.get('meta', {}) and \ 'object_count' in container_info and \ container_info['meta']['quota-count'].isdigit(): new_count = int(container_info['object_count']) + 1 if int(container_info['meta']['quota-count']) < new_count: return self.bad_response(req, container_info) return self.app
def test_get_container_info_cache(self): cached = {'status': 404, 'bytes': 3333, 'object_count': 10, # simplejson sometimes hands back strings, sometimes unicodes 'versions': u"\u1F4A9"} req = Request.blank("/v1/account/cont", environ={'swift.cache': FakeCache(cached)}) with patch('swift.proxy.controllers.base.' '_prepare_pre_auth_info_request', FakeRequest): resp = get_container_info(req.environ, 'xxx') self.assertEquals(resp['bytes'], 3333) self.assertEquals(resp['object_count'], 10) self.assertEquals(resp['status'], 404) self.assertEquals(resp['versions'], "\xe1\xbd\x8a\x39")
def PUT(self, env): """Handle PUT requests related to metadata""" req = Request(env) conn = HTTPConnection('%s:%s' % (self.mds_ip, self.mds_port)) headers = req.params try: info = get_container_info(env, self.app) if info: stor_policy = info['storage_policy'] headers['storage_policy'] = stor_policy except: pass conn.request('PUT', req.path, headers=headers) resp = conn.getresponse() return self.app
def test_get_container_info_cache(self): cache_stub = { "status": 404, "bytes": 3333, "object_count": 10, # simplejson sometimes hands back strings, sometimes unicodes "versions": u"\u1F4A9", } req = Request.blank("/v1/account/cont", environ={"swift.cache": FakeCache(cache_stub)}) resp = get_container_info(req.environ, FakeApp()) self.assertEquals(resp["storage_policy"], "0") self.assertEquals(resp["bytes"], 3333) self.assertEquals(resp["object_count"], 10) self.assertEquals(resp["status"], 404) self.assertEquals(resp["versions"], "\xe1\xbd\x8a\x39")
def object_request(self, req, api_version, account, container, obj, allow_versioned_writes): account_name = unquote(account) container_name = unquote(container) object_name = unquote(obj) resp = None is_enabled = config_true_value(allow_versioned_writes) container_info = get_container_info(req.environ, self.app) # To maintain backwards compatibility, container version # location could be stored as sysmeta or not, need to check both. # If stored as sysmeta, check if middleware is enabled. If sysmeta # is not set, but versions property is set in container_info, then # for backwards compatibility feature is enabled. versions_cont = container_info.get('sysmeta', {}).get('versions-location') versioning_mode = container_info.get('sysmeta', {}).get('versions-mode', 'stack') if not versions_cont: versions_cont = container_info.get('versions') # if allow_versioned_writes is not set in the configuration files # but 'versions' is configured, enable feature to maintain # backwards compatibility if not allow_versioned_writes and versions_cont: is_enabled = True if is_enabled and versions_cont: versions_cont = unquote(versions_cont).split('/')[0] vw_ctx = VersionedWritesContext(self.app, self.logger) if req.method == 'PUT': resp = vw_ctx.handle_obj_versions_put(req, versions_cont, api_version, account_name, object_name) # handle DELETE elif versioning_mode == 'history': resp = vw_ctx.handle_obj_versions_delete_push( req, versions_cont, api_version, account_name, container_name, object_name) else: resp = vw_ctx.handle_obj_versions_delete_pop( req, versions_cont, api_version, account_name, container_name, object_name) if resp: return resp else: return self.app