def test_remove_node(self):
replicas = 128
hash_ring_object = HashRing(
nodes=["redis01", "redis02"],
replicas=replicas,
)
hash_ring_object.remove_node("redis01")
eq_(hash_ring_object.nodes, ["redis02"])
eq_(list(hash_ring_object.ring.values()), ["redis02"] * replicas)
def test_remove_node(self):
replicas = 128
hash_ring_object = HashRing(
nodes=["redis01", "redis02"],
replicas=replicas,
)
hash_ring_object.remove_node("redis01")
eq_(hash_ring_object.nodes, ["redis02"])
eq_(list(hash_ring_object.ring.values()), ["redis02"] * replicas)
def __init__(self, servers):
VERSION = tuple(map(int, redis.__version__.split('.')))
self.nodes = []
self.connections = {}
if VERSION < (2, 4, 0):
self.pool = redis.ConnectionPool()
else:
self.pool = None
if isinstance(servers, list):
for server in servers:
conn = redis.Redis(
host=server['host'], port=server[
'port'], db=server['db'], connection_pool=self.pool,
password=server.get('password'), socket_timeout=server.get('socket_timeout'))
name = server['name']
if name in self.connections:
raise ValueError("server's name config must be unique")
self.connections[name] = conn
self.nodes.append(name)
elif isinstance(servers, dict):
for server_name, server in servers.items():
conn = redis.Redis(
host=server['host'], port=server[
'port'], db=server['db'], connection_pool=self.pool,
password=server.get('password'), socket_timeout=server.get('socket_timeout'))
name = server_name
if name in self.connections:
raise ValueError("server's name config must be unique")
self.connections[name] = conn
self.nodes.append(name)
else:
raise ValueError("server's config must be list or dict")
self.ring = HashRing(self.nodes)
def __init__(self, settings=None):
self.nodes = []
self.connections = {}
settings = format_config(settings)
for server_config in settings:
name = server_config.pop('name')
conn = redis.StrictRedis(**server_config)
if name in self.connections:
raise ValueError("server's name config must be unique")
server_config['name'] = name
self.connections[name] = conn
self.nodes.append(name)
self.ring = HashRing(self.nodes)
class RedisShardAPI(object):
def __init__(self, settings=None):
self.nodes = []
self.connections = {}
settings = format_config(settings)
for server_config in settings:
name = server_config.pop('name')
conn = redis.StrictRedis(**server_config)
if name in self.connections:
raise ValueError("server's name config must be unique")
server_config['name'] = name
self.connections[name] = conn
self.nodes.append(name)
self.ring = HashRing(self.nodes)
def get_server_name(self, key):
g = _findhash.match(key)
if g is not None and len(g.groups()) > 0:
key = g.groups()[0]
name = self.ring.get_node(key)
#print key, '->', name
return name
def get_server(self, key):
name = self.get_server_name(key)
return self.connections[name]
def __wrap(self, method, *args, **kwargs):
try:
key = args[0]
assert isinstance(key, basestring)
except:
raise ValueError("method '%s' requires a key param as the first argument" % method)
server = self.get_server(key)
f = getattr(server, method)
return f(*args, **kwargs)
def __wrap_tag(self, method, *args, **kwargs):
key = args[0]
if isinstance(key, basestring) and '{' in key:
server = self.get_server(key)
elif isinstance(key, list) and '{' in key[0]:
server = self.get_server(key[0])
else:
raise ValueError("method '%s' requires tag key params as its arguments" % method)
method = method.lstrip("tag_")
f = getattr(server, method)
return f(*args, **kwargs)
def __getattr__(self, method):
if method in SHARD_METHODS:
return functools.partial(self.__wrap, method)
elif method.startswith("tag_"):
return functools.partial(self.__wrap_tag, method)
else:
raise NotImplementedError("method '%s' cannot be sharded" % method)
#########################################
### some methods implement as needed ###
########################################
def brpop(self, key, timeout=0):
if not isinstance(key, basestring):
raise NotImplementedError("The key must be single string;mutiple keys cannot be sharded")
server = self.get_server(key)
return server.brpop(key, timeout)
def blpop(self, key, timeout=0):
if not isinstance(key, basestring):
raise NotImplementedError("The key must be single string;mutiple keys cannot be sharded")
server = self.get_server(key)
return server.blpop(key, timeout)
def keys(self, key):
_keys = []
for node in self.nodes:
server = self.connections[node]
_keys.extend(server.keys(key))
return _keys
def mget(self, keys, *args):
"""
Returns a list of values ordered identically to ``keys``
"""
args = list_or_args(keys, args)
server_keys = {}
ret_dict = {}
for key in args:
server_name = self.get_server_name(key)
server_keys[server_name] = server_keys.get(server_name, [])
server_keys[server_name].append(key)
for server_name, sub_keys in server_keys.iteritems():
values = self.connections[server_name].mget(sub_keys)
ret_dict.update(dict(zip(sub_keys, values)))
result = []
for key in args:
result.append(ret_dict.get(key, None))
return result
def flushdb(self):
for node in self.nodes:
server = self.connections[node]
server.flushdb()
def lock(self, name, timeout=None, sleep=0.1):
"""
Return a new Lock object using key ``name`` that mimics
the behavior of threading.Lock.
If specified, ``timeout`` indicates a maximum life for the lock.
By default, it will remain locked until release() is called.
``sleep`` indicates the amount of time to sleep per loop iteration
when the lock is in blocking mode and another client is currently
holding the lock.
"""
return Lock(self, name, timeout=timeout, sleep=sleep)
def pipeline(self):
return Pipeline(self)
class RedisShardAPI(object):
def __init__(self, servers):
VERSION = tuple(map(int, redis.__version__.split('.')))
self.nodes = []
self.connections = {}
if VERSION < (2, 4, 0):
self.pool = redis.ConnectionPool()
else:
self.pool = None
if isinstance(servers, list):
for server in servers:
conn = redis.Redis(
host=server['host'], port=server[
'port'], db=server['db'], connection_pool=self.pool,
password=server.get('password'), socket_timeout=server.get('socket_timeout'))
name = server['name']
if name in self.connections:
raise ValueError("server's name config must be unique")
self.connections[name] = conn
self.nodes.append(name)
elif isinstance(servers, dict):
for server_name, server in servers.items():
conn = redis.Redis(
host=server['host'], port=server[
'port'], db=server['db'], connection_pool=self.pool,
password=server.get('password'), socket_timeout=server.get('socket_timeout'))
name = server_name
if name in self.connections:
raise ValueError("server's name config must be unique")
self.connections[name] = conn
self.nodes.append(name)
else:
raise ValueError("server's config must be list or dict")
self.ring = HashRing(self.nodes)
def get_server_name(self, key):
g = _findhash.match(key)
if g is not None and len(g.groups()) > 0:
key = g.groups()[0]
name = self.ring.get_node(key)
return name
def get_server(self, key):
name = self.get_server_name(key)
return self.connections[name]
def __wrap(self, method, *args, **kwargs):
try:
key = args[0]
assert isinstance(key, str)
except:
raise ValueError("method '%s' requires a key param as the first argument" % method)
server = self.get_server(key)
f = getattr(server, method)
return f(*args, **kwargs)
def __wrap_tag(self, method, *args, **kwargs):
key = args[0]
if isinstance(key, str) and '{' in key:
server = self.get_server(key)
elif isinstance(key, list) and '{' in key[0]:
server = self.get_server(key[0])
else:
raise ValueError("method '%s' requires tag key params as its arguments" % method)
method = method.lstrip("tag_")
f = getattr(server, method)
return f(*args, **kwargs)
def __hop_in(self, method, *args, **kwargs):
'''
使用field作为查询hashring的key
'''
if not isinstance(args[1], str):
key = str(args[1])
else:
key = args[1]
try:
assert isinstance(key, str)
except:
raise ValueError("method '%s' requires a key param as the second argument" % method)
server = self.get_server(key)
if method == "hget_in":
method = "hget"
elif method == "hset_in":
method = "hset"
elif method == "hdel_in":
method = "hdel"
else:
print("you can't be here")
f = getattr(server, method)
return f(*args, **kwargs)
def __qop_in(self, method, *args, **kwargs):
'''
指定key值所对应的hashring上的一个节点作为队列服务器
'''
key = "queue"
server = self.get_server(key)
if method == "rpush_in":
method = "rpush"
elif method == "blpop_in":
method = "blpop"
else:
print("you can't be here")
f = getattr(server, method)
return f(*args, **kwargs)
def __getattr__(self, method):
if method in [
"get", "set", "getset",
"setnx", "setex",
"incr", "decr", "exists",
"delete", "get_type", "type", "rename",
"expire", "ttl", "push", "persist",
"llen", "lrange", "ltrim", "lpush", "lpop",
"lindex", "pop", "lset",
"lrem", "sadd", "srem", "scard",
"sismember", "smembers",
"zadd", "zrem", "zincrby", "zincr", "zrank",
"zrange", "zrevrange", "zrangebyscore", "zremrangebyrank", "zrevrangebyscore",
"zremrangebyscore", "zcard", "zscore", "zcount",
"hget", "hset", "hdel", "hincrby", "hlen",
"hkeys", "hvals", "hgetall", "hexists", "hmget", "hmset",
"publish", "rpush", "rpop"
]:
return functools.partial(self.__wrap, method)
elif method.startswith("tag_"):
return functools.partial(self.__wrap_tag, method)
elif method in ["hget_in", "hset_in", "hdel_in"]:
return functools.partial(self.__hop_in, method)
elif method in ["blpop_in", "rpush_in"]:
return functools.partial(self.__qop_in, method)
else:
raise NotImplementedError("method '%s' cannot be sharded" % method)
#########################################
### some methods implement as needed ###
########################################
def brpop(self, key, timeout=0):
if not isinstance(key, str):
raise NotImplementedError("The key must be single string;mutiple keys cannot be sharded")
server = self.get_server(key)
return server.brpop(key, timeout)
def blpop(self, key, timeout=0):
if not isinstance(key, str):
raise NotImplementedError("The key must be single string;mutiple keys cannot be sharded")
server = self.get_server(key)
return server.blpop(key, timeout)
def keys(self, key):
_keys = []
for node in self.nodes:
server = self.connections[node]
_keys.extend(server.keys(key))
return _keys
def flushdb(self):
for node in self.nodes:
server = self.connections[node]
server.flushdb()
def hgetall_in(self, key):
result = {}
for node in self.nodes:
server = self.connections[node]
result.update(server.hgetall(key))
return result
def hmget_in(self, key, fields):
result = {}
node_field = {}
for field in fields:
node = self.get_server_name(field)
node_field.setdefault(node, [])
node_field[node].append(field)
for node, field_list in node_field.items():
server = self.connections[node]
value = server.hmget(key, field_list)
for i in range(len(field_list)):
result[field_list[i]] = value[i]
return result
def lock(self, name, timeout=None, sleep=0.1):
"""
Return a new Lock object using key ``name`` that mimics
the behavior of threading.Lock.
If specified, ``timeout`` indicates a maximum life for the lock.
By default, it will remain locked until release() is called.
``sleep`` indicates the amount of time to sleep per loop iteration
when the lock is in blocking mode and another client is currently
holding the lock.
"""
return Lock(self, name, timeout=timeout, sleep=sleep)
def pipeline(self):
return Pipeline(self)
