def POST(self, req): """HTTP POST request handler.""" if len(self.account_name) > MAX_ACCOUNT_NAME_LENGTH: resp = HTTPBadRequest(request=req) resp.body = 'Account name length of %d longer than %d' % \ (len(self.account_name), MAX_ACCOUNT_NAME_LENGTH) return resp error_response = check_metadata(req, 'account') if error_response: return error_response account_partition, accounts = \ self.app.account_ring.get_nodes(self.account_name) headers = self.generate_request_headers(req, transfer=True) if self.app.memcache: self.app.memcache.delete( get_account_memcache_key(self.account_name)) resp = self.make_requests(req, self.app.account_ring, account_partition, 'POST', req.path_info, [headers] * len(accounts)) if resp.status_int == HTTP_NOT_FOUND and self.app.account_autocreate: self.autocreate_account(self.account_name) resp = self.make_requests(req, self.app.account_ring, account_partition, 'POST', req.path_info, [headers] * len(accounts)) return resp
def PUT(self, req): """HTTP PUT request handler.""" if not self.app.allow_account_management: return HTTPMethodNotAllowed( request=req, headers={'Allow': ', '.join(self.allowed_methods)}) error_response = check_metadata(req, 'account') if error_response: return error_response if len(self.account_name) > MAX_ACCOUNT_NAME_LENGTH: resp = HTTPBadRequest(request=req) resp.body = 'Account name length of %d longer than %d' % \ (len(self.account_name), MAX_ACCOUNT_NAME_LENGTH) return resp account_partition, accounts = \ self.app.account_ring.get_nodes(self.account_name) headers = { 'X-Timestamp': normalize_timestamp(time.time()), 'x-trans-id': self.trans_id, 'Connection': 'close' } self.transfer_headers(req.headers, headers) if self.app.memcache: self.app.memcache.delete( get_account_memcache_key(self.account_name)) resp = self.make_requests(req, self.app.account_ring, account_partition, 'PUT', req.path_info, [headers] * len(accounts)) return resp
def POST(self, req): """HTTP POST request handler.""" if len(self.account_name) > MAX_ACCOUNT_NAME_LENGTH: resp = HTTPBadRequest(request=req) resp.body = 'Account name length of %d longer than %d' % \ (len(self.account_name), MAX_ACCOUNT_NAME_LENGTH) return resp error_response = check_metadata(req, 'account') if error_response: return error_response account_partition, accounts = \ self.app.account_ring.get_nodes(self.account_name) headers = self.generate_request_headers(req, transfer=True) if self.app.memcache: self.app.memcache.delete( get_account_memcache_key(self.account_name)) resp = self.make_requests( req, self.app.account_ring, account_partition, 'POST', req.path_info, [headers] * len(accounts)) if resp.status_int == HTTP_NOT_FOUND and self.app.account_autocreate: self.autocreate_account(self.account_name) resp = self.make_requests( req, self.app.account_ring, account_partition, 'POST', req.path_info, [headers] * len(accounts)) return resp
def DELETE(self, req): """HTTP DELETE request handler.""" # Extra safety in case someone typos a query string for an # account-level DELETE request that was really meant to be caught by # some middleware. if req.query_string: return HTTPBadRequest(request=req) if not self.app.allow_account_management: return HTTPMethodNotAllowed( request=req, headers={'Allow': ', '.join(self.allowed_methods)}) account_partition, accounts = \ self.app.account_ring.get_nodes(self.account_name) headers = { 'X-Timestamp': normalize_timestamp(time.time()), 'X-Trans-Id': self.trans_id, 'Connection': 'close' } if self.app.memcache: self.app.memcache.delete( get_account_memcache_key(self.account_name)) resp = self.make_requests(req, self.app.account_ring, account_partition, 'DELETE', req.path_info, [headers] * len(accounts)) return resp
def handle_delete(self, env, start_response, version, account, container, obj): """ Handle delete request. """ memcache_client = cache_from_env(env) if not memcache_client: return self.app(env, start_response) res = [None, None, None] result_code = None def _start_response(response_status, response_headers, exc_info=None): res[0] = response_status res[1] = response_headers res[2] = exc_info resp = self.app(env, _start_response) result_code = self._get_status_int(res[0]) try: if is_success(result_code): if obj: memcache_client.delete( get_container_memcache_key(account, container)) else: memcache_client.delete(get_account_memcache_key(account)) except Exception, err: self.logger.error( 'Error in [Quota] delete cache: %s' % (err.message))
def POST(self, req): """HTTP POST request handler.""" error_response = check_metadata(req, 'account') if error_response: return error_response account_partition, accounts = \ self.app.account_ring.get_nodes(self.account_name) headers = {'X-Timestamp': normalize_timestamp(time.time()), 'X-Trans-Id': self.trans_id, 'Connection': 'close'} self.transfer_headers(req.headers, headers) if self.app.memcache: self.app.memcache.delete( get_account_memcache_key(self.account_name)) resp = self.make_requests( req, self.app.account_ring, account_partition, 'POST', req.path_info, [headers] * len(accounts)) if resp.status_int == HTTP_NOT_FOUND and self.app.account_autocreate: if len(self.account_name) > MAX_ACCOUNT_NAME_LENGTH: resp = HTTPBadRequest(request=req) resp.body = 'Account name length of %d longer than %d' % \ (len(self.account_name), MAX_ACCOUNT_NAME_LENGTH) return resp resp = self.make_requests( Request.blank('/v1/' + self.account_name), self.app.account_ring, account_partition, 'PUT', '/' + self.account_name, [headers] * len(accounts)) if not is_success(resp.status_int): self.app.logger.warning('Could not autocreate account %r' % self.account_name) return resp return resp
def PUT(self, req): """HTTP PUT request handler.""" if not self.app.allow_account_management: return HTTPMethodNotAllowed( request=req, headers={'Allow': ', '.join(self.allowed_methods)}) error_response = check_metadata(req, 'account') if error_response: return error_response if len(self.account_name) > MAX_ACCOUNT_NAME_LENGTH: resp = HTTPBadRequest(request=req) resp.body = 'Account name length of %d longer than %d' % \ (len(self.account_name), MAX_ACCOUNT_NAME_LENGTH) return resp account_partition, accounts = \ self.app.account_ring.get_nodes(self.account_name) headers = {'X-Timestamp': normalize_timestamp(time.time()), 'x-trans-id': self.trans_id, 'Connection': 'close'} self.transfer_headers(req.headers, headers) if self.app.memcache: self.app.memcache.delete( get_account_memcache_key(self.account_name)) resp = self.make_requests( req, self.app.account_ring, account_partition, 'PUT', req.path_info, [headers] * len(accounts)) return resp
def POST(self, req): """HTTP POST request handler.""" error_response = check_metadata(req, 'account') if error_response: return error_response account_partition, accounts = \ self.app.account_ring.get_nodes(self.account_name) headers = { 'X-Timestamp': normalize_timestamp(time.time()), 'X-Trans-Id': self.trans_id, 'Connection': 'close' } self.transfer_headers(req.headers, headers) if self.app.memcache: self.app.memcache.delete( get_account_memcache_key(self.account_name)) resp = self.make_requests(req, self.app.account_ring, account_partition, 'POST', req.path_info, [headers] * len(accounts)) if resp.status_int == HTTP_NOT_FOUND and self.app.account_autocreate: if len(self.account_name) > MAX_ACCOUNT_NAME_LENGTH: resp = HTTPBadRequest(request=req) resp.body = 'Account name length of %d longer than %d' % \ (len(self.account_name), MAX_ACCOUNT_NAME_LENGTH) return resp resp = self.make_requests( Request.blank('/v1/' + self.account_name), self.app.account_ring, account_partition, 'PUT', '/' + self.account_name, [headers] * len(accounts)) if not is_success(resp.status_int): self.app.logger.warning('Could not autocreate account %r' % self.account_name) return resp return resp
def test_get_account_info_env(self): cache_key = get_account_memcache_key("account") env_key = 'swift.%s' % cache_key req = Request.blank("/v1/account", environ={env_key: {'bytes': 3867}, 'swift.cache': FakeCache({})}) resp = get_account_info(req.environ, 'xxx') self.assertEquals(resp['bytes'], 3867)
def test_allowed_without_cache(self): accname = 'acc_allowed' swproxy.make_pre_authed_request = FakeRequest req = self._make_request( environ={ 'REQUEST_METHOD': 'POST', 'PATH_INFO': '/v1/%s' % (accname), 'swift.cache': FakeCache({}), }) resp = req.get_response(self.test_default) key = swproxy.get_account_memcache_key(accname) memcache_key = resp.environ.get("swift.%s" % key) self.assertTrue('meta' in memcache_key) self.assertEquals(memcache_key['meta']['locked'], 'false') self.assertTrue('swift.authorize' not in resp.environ)
def DELETE(self, req): """HTTP DELETE request handler.""" if not self.app.allow_account_management: return HTTPMethodNotAllowed(request=req) account_partition, accounts = \ self.app.account_ring.get_nodes(self.account_name) headers = {'X-Timestamp': normalize_timestamp(time.time()), 'X-Trans-Id': self.trans_id, 'Connection': 'close'} if self.app.memcache: self.app.memcache.delete( get_account_memcache_key(self.account_name)) resp = self.make_requests(req, self.app.account_ring, account_partition, 'DELETE', req.path_info, [headers] * len(accounts)) return resp
def DELETE(self, req): """HTTP DELETE request handler.""" if not self.app.allow_account_management: return HTTPMethodNotAllowed( request=req, headers={'Allow': ', '.join(self.allowed_methods)}) account_partition, accounts = \ self.app.account_ring.get_nodes(self.account_name) headers = {'X-Timestamp': normalize_timestamp(time.time()), 'X-Trans-Id': self.trans_id, 'Connection': 'close'} if self.app.memcache: self.app.memcache.delete( get_account_memcache_key(self.account_name)) resp = self.make_requests( req, self.app.account_ring, account_partition, 'DELETE', req.path_info, [headers] * len(accounts)) return resp
def test_deny_other_header_without_cache(self): accname = 'acc_denied_other' other_header = 'other' conf = {'locked_header': other_header} self.test_default = middleware.filter_factory(conf)(FakeApp()) swproxy.make_pre_authed_request = FakeRequest req = self._make_request( environ={ 'REQUEST_METHOD': 'POST', 'PATH_INFO': '/v1/%s' % (accname), 'swift.cache': FakeCache({}), }) resp = req.get_response(self.test_default) key = swproxy.get_account_memcache_key(accname) memcache_key = resp.environ.get("swift.%s" % key) self.assertTrue('meta' in memcache_key) self.assertTrue(other_header in memcache_key['meta']) self.assertEquals(memcache_key['meta'][other_header], 'true') self.assertTrue('swift.authorize' in resp.environ) self.assertEquals(resp.environ['swift.authorize'], self.test_default.deny)
def DELETE(self, req): """HTTP DELETE request handler.""" # Extra safety in case someone typos a query string for an # account-level DELETE request that was really meant to be caught by # some middleware. if req.query_string: return HTTPBadRequest(request=req) if not self.app.allow_account_management: return HTTPMethodNotAllowed( request=req, headers={'Allow': ', '.join(self.allowed_methods)}) account_partition, accounts = \ self.app.account_ring.get_nodes(self.account_name) headers = self.generate_request_headers(req) if self.app.memcache: self.app.memcache.delete( get_account_memcache_key(self.account_name)) resp = self.make_requests( req, self.app.account_ring, account_partition, 'DELETE', req.path_info, [headers] * len(accounts)) return resp
def get_account_info(env, app, logger): """ Get the info structure for an account, based on env and app. This is useful to middlewares. """ # TODO: memcachet container_info = env.get("swift.container_info") if not container_info: cache = cache_from_env(env) if not cache: return None (version, account, _, _) = split_path(env["PATH_INFO"], 2, 4, True) cache_key = get_account_memcache_key(account) account_info = cache.get(cache_key) if account_info: print "Using memcache." if not account_info: new_env = dict(env, REQUEST_METHOD="HEAD") resp = Request.blank("/%s/%s" % (version, account), environ=new_env).get_response(app) account_info = headers_to_account_info(resp.headers) # TODO: use recheck_account_existence for timeout. cache.set(cache_key, account_info, timeout=60) return account_info
def handle_quota_container(self, env, start_response, version, account, container): """ Container Count Quota """ memcache_client = cache_from_env(env) container_count, quota_level = self._get_account_meta( env, version, account, memcache_client) try: quota = self.container_count[quota_level] except Exception: self.logger.warn('Invalid quota_leve %s/%s quota_level[%s].' % ( account, container, quota_level)) quota = None if quota and container_count and container_count + 1 > quota: self.logger.notice("Container count over quota, request[PUT %s/%s]," " container_count[%s] quota[%s]" % ( account, container, container_count + 1, quota)) return HTTPForbidden(body="The number of container is over quota")( env, start_response) elif self.precise_mode and memcache_client: res = [None, None, None] result_code = None def _start_response(response_status, response_headers, exc_info=None): res[0] = response_status res[1] = response_headers res[2] = exc_info resp = self.app(env, _start_response) result_code = self._get_status_int(res[0]) if is_success(result_code): memcache_client.delete(get_account_memcache_key(account)) start_response(res[0], res[1], res[2]) return resp else: return self.app(env, start_response)
def PUT(self, req): """HTTP PUT request handler.""" if not self.app.allow_account_management: return HTTPMethodNotAllowed( request=req, headers={'Allow': ', '.join(self.allowed_methods)}) error_response = check_metadata(req, 'account') if error_response: return error_response if len(self.account_name) > MAX_ACCOUNT_NAME_LENGTH: resp = HTTPBadRequest(request=req) resp.body = 'Account name length of %d longer than %d' % \ (len(self.account_name), MAX_ACCOUNT_NAME_LENGTH) return resp account_partition, accounts = \ self.app.account_ring.get_nodes(self.account_name) headers = self.generate_request_headers(req, transfer=True) if self.app.memcache: self.app.memcache.delete( get_account_memcache_key(self.account_name)) resp = self.make_requests( req, self.app.account_ring, account_partition, 'PUT', req.path_info, [headers] * len(accounts)) return resp
def PUT(self, req): """HTTP PUT request handler.""" if not self.app.allow_account_management: return HTTPMethodNotAllowed( request=req, headers={'Allow': ', '.join(self.allowed_methods)}) error_response = check_metadata(req, 'account') if error_response: return error_response if len(self.account_name) > MAX_ACCOUNT_NAME_LENGTH: resp = HTTPBadRequest(request=req) resp.body = 'Account name length of %d longer than %d' % \ (len(self.account_name), MAX_ACCOUNT_NAME_LENGTH) return resp account_partition, accounts = \ self.app.account_ring.get_nodes(self.account_name) headers = self.generate_request_headers(req, transfer=True) if self.app.memcache: self.app.memcache.delete( get_account_memcache_key(self.account_name)) resp = self.make_requests(req, self.app.account_ring, account_partition, 'PUT', req.path_info, [headers] * len(accounts)) return resp
def _get_account_meta(self, env, version, account, memcache_client): """ Get metadata of account. :param env: The WSGI environment :param version: The api version in PATH_INFO :param account: The name of account :return: tuple of (container_count, quota_level) or (None, None) """ value = None quota_level = None container_count = None res = [None, None, None] result_code = None def _start_response(response_status, response_headers, exc_info=None): res[0] = response_status res[1] = response_headers res[2] = exc_info # get quota_level and container_count account_key = get_account_memcache_key(account) if memcache_client: value = memcache_client.get(account_key) if value: self.logger.debug('value from mc: %s' % (value)) if not isinstance(value, dict): result_code = value else: result_code = value.get('status') if is_success(result_code): # get from memcached container_count = int(value.get('container_count') or 0) quota_level = value.get('quota_level') or 'default' return container_count, quota_level else: # get from account-server temp_env = self._get_escalated_env(env) temp_env['REQUEST_METHOD'] = 'HEAD' temp_env['PATH_INFO'] = '/%s/%s' % (version, account) resp = self.app(temp_env, _start_response) self.logger.debug( 'value form account-server status[%s] header[%s]' % (res[0], res[1])) result_code = self._get_status_int(res[0]) if is_success(result_code): headers = dict(res[1]) container_count = int( headers.get('x-account-container-count') or 0) quota_level = headers.get('x-account-meta-quota') or 'default' if memcache_client: memcache_client.set( account_key, { 'status': result_code, 'container_count': container_count, 'quota_level': quota_level }, timeout=self.cache_timeout ) return container_count, quota_level else: return None, None
def test_get_account_info_env(self): cache_key = get_account_memcache_key("account") env_key = "swift.%s" % cache_key req = Request.blank("/v1/account", environ={env_key: {"bytes": 3867}, "swift.cache": FakeCache({})}) resp = get_account_info(req.environ, "xxx") self.assertEquals(resp["bytes"], 3867)