class RedisShardAPI(object): def __init__(self, settings=None): self.nodes = [] self.connections = {} settings = format_config(settings) for server_config in settings: name = server_config.pop('name') conn = redis.StrictRedis(**server_config) if name in self.connections: raise ValueError("server's name config must be unique") server_config['name'] = name self.connections[name] = conn self.nodes.append(name) self.ring = HashRing(self.nodes) def get_server_name(self, key): g = _findhash.match(key) if g is not None and len(g.groups()) > 0: key = g.groups()[0] name = self.ring.get_node(key) #print key, '->', name return name def get_server(self, key): name = self.get_server_name(key) return self.connections[name] def __wrap(self, method, *args, **kwargs): try: key = args[0] assert isinstance(key, basestring) except: raise ValueError("method '%s' requires a key param as the first argument" % method) server = self.get_server(key) f = getattr(server, method) return f(*args, **kwargs) def __wrap_tag(self, method, *args, **kwargs): key = args[0] if isinstance(key, basestring) and '{' in key: server = self.get_server(key) elif isinstance(key, list) and '{' in key[0]: server = self.get_server(key[0]) else: raise ValueError("method '%s' requires tag key params as its arguments" % method) method = method.lstrip("tag_") f = getattr(server, method) return f(*args, **kwargs) def __getattr__(self, method): if method in SHARD_METHODS: return functools.partial(self.__wrap, method) elif method.startswith("tag_"): return functools.partial(self.__wrap_tag, method) else: raise NotImplementedError("method '%s' cannot be sharded" % method) ######################################### ### some methods implement as needed ### ######################################## def brpop(self, key, timeout=0): if not isinstance(key, basestring): raise NotImplementedError("The key must be single string;mutiple keys cannot be sharded") server = self.get_server(key) return server.brpop(key, timeout) def blpop(self, key, timeout=0): if not isinstance(key, basestring): raise NotImplementedError("The key must be single string;mutiple keys cannot be sharded") server = self.get_server(key) return server.blpop(key, timeout) def keys(self, key): _keys = [] for node in self.nodes: server = self.connections[node] _keys.extend(server.keys(key)) return _keys def mget(self, keys, *args): """ Returns a list of values ordered identically to ``keys`` """ args = list_or_args(keys, args) server_keys = {} ret_dict = {} for key in args: server_name = self.get_server_name(key) server_keys[server_name] = server_keys.get(server_name, []) server_keys[server_name].append(key) for server_name, sub_keys in server_keys.iteritems(): values = self.connections[server_name].mget(sub_keys) ret_dict.update(dict(zip(sub_keys, values))) result = [] for key in args: result.append(ret_dict.get(key, None)) return result def flushdb(self): for node in self.nodes: server = self.connections[node] server.flushdb() def lock(self, name, timeout=None, sleep=0.1): """ Return a new Lock object using key ``name`` that mimics the behavior of threading.Lock. If specified, ``timeout`` indicates a maximum life for the lock. By default, it will remain locked until release() is called. ``sleep`` indicates the amount of time to sleep per loop iteration when the lock is in blocking mode and another client is currently holding the lock. """ return Lock(self, name, timeout=timeout, sleep=sleep) def pipeline(self): return Pipeline(self)
class RedisShardAPI(object): def __init__(self, servers): VERSION = tuple(map(int, redis.__version__.split('.'))) self.nodes = [] self.connections = {} if VERSION < (2, 4, 0): self.pool = redis.ConnectionPool() else: self.pool = None if isinstance(servers, list): for server in servers: conn = redis.Redis( host=server['host'], port=server[ 'port'], db=server['db'], connection_pool=self.pool, password=server.get('password'), socket_timeout=server.get('socket_timeout')) name = server['name'] if name in self.connections: raise ValueError("server's name config must be unique") self.connections[name] = conn self.nodes.append(name) elif isinstance(servers, dict): for server_name, server in servers.items(): conn = redis.Redis( host=server['host'], port=server[ 'port'], db=server['db'], connection_pool=self.pool, password=server.get('password'), socket_timeout=server.get('socket_timeout')) name = server_name if name in self.connections: raise ValueError("server's name config must be unique") self.connections[name] = conn self.nodes.append(name) else: raise ValueError("server's config must be list or dict") self.ring = HashRing(self.nodes) def get_server_name(self, key): g = _findhash.match(key) if g is not None and len(g.groups()) > 0: key = g.groups()[0] name = self.ring.get_node(key) return name def get_server(self, key): name = self.get_server_name(key) return self.connections[name] def __wrap(self, method, *args, **kwargs): try: key = args[0] assert isinstance(key, str) except: raise ValueError("method '%s' requires a key param as the first argument" % method) server = self.get_server(key) f = getattr(server, method) return f(*args, **kwargs) def __wrap_tag(self, method, *args, **kwargs): key = args[0] if isinstance(key, str) and '{' in key: server = self.get_server(key) elif isinstance(key, list) and '{' in key[0]: server = self.get_server(key[0]) else: raise ValueError("method '%s' requires tag key params as its arguments" % method) method = method.lstrip("tag_") f = getattr(server, method) return f(*args, **kwargs) def __hop_in(self, method, *args, **kwargs): ''' 使用field作为查询hashring的key ''' if not isinstance(args[1], str): key = str(args[1]) else: key = args[1] try: assert isinstance(key, str) except: raise ValueError("method '%s' requires a key param as the second argument" % method) server = self.get_server(key) if method == "hget_in": method = "hget" elif method == "hset_in": method = "hset" elif method == "hdel_in": method = "hdel" else: print("you can't be here") f = getattr(server, method) return f(*args, **kwargs) def __qop_in(self, method, *args, **kwargs): ''' 指定key值所对应的hashring上的一个节点作为队列服务器 ''' key = "queue" server = self.get_server(key) if method == "rpush_in": method = "rpush" elif method == "blpop_in": method = "blpop" else: print("you can't be here") f = getattr(server, method) return f(*args, **kwargs) def __getattr__(self, method): if method in [ "get", "set", "getset", "setnx", "setex", "incr", "decr", "exists", "delete", "get_type", "type", "rename", "expire", "ttl", "push", "persist", "llen", "lrange", "ltrim", "lpush", "lpop", "lindex", "pop", "lset", "lrem", "sadd", "srem", "scard", "sismember", "smembers", "zadd", "zrem", "zincrby", "zincr", "zrank", "zrange", "zrevrange", "zrangebyscore", "zremrangebyrank", "zrevrangebyscore", "zremrangebyscore", "zcard", "zscore", "zcount", "hget", "hset", "hdel", "hincrby", "hlen", "hkeys", "hvals", "hgetall", "hexists", "hmget", "hmset", "publish", "rpush", "rpop" ]: return functools.partial(self.__wrap, method) elif method.startswith("tag_"): return functools.partial(self.__wrap_tag, method) elif method in ["hget_in", "hset_in", "hdel_in"]: return functools.partial(self.__hop_in, method) elif method in ["blpop_in", "rpush_in"]: return functools.partial(self.__qop_in, method) else: raise NotImplementedError("method '%s' cannot be sharded" % method) ######################################### ### some methods implement as needed ### ######################################## def brpop(self, key, timeout=0): if not isinstance(key, str): raise NotImplementedError("The key must be single string;mutiple keys cannot be sharded") server = self.get_server(key) return server.brpop(key, timeout) def blpop(self, key, timeout=0): if not isinstance(key, str): raise NotImplementedError("The key must be single string;mutiple keys cannot be sharded") server = self.get_server(key) return server.blpop(key, timeout) def keys(self, key): _keys = [] for node in self.nodes: server = self.connections[node] _keys.extend(server.keys(key)) return _keys def flushdb(self): for node in self.nodes: server = self.connections[node] server.flushdb() def hgetall_in(self, key): result = {} for node in self.nodes: server = self.connections[node] result.update(server.hgetall(key)) return result def hmget_in(self, key, fields): result = {} node_field = {} for field in fields: node = self.get_server_name(field) node_field.setdefault(node, []) node_field[node].append(field) for node, field_list in node_field.items(): server = self.connections[node] value = server.hmget(key, field_list) for i in range(len(field_list)): result[field_list[i]] = value[i] return result def lock(self, name, timeout=None, sleep=0.1): """ Return a new Lock object using key ``name`` that mimics the behavior of threading.Lock. If specified, ``timeout`` indicates a maximum life for the lock. By default, it will remain locked until release() is called. ``sleep`` indicates the amount of time to sleep per loop iteration when the lock is in blocking mode and another client is currently holding the lock. """ return Lock(self, name, timeout=timeout, sleep=sleep) def pipeline(self): return Pipeline(self)