class selectclass:
def __init__(self):
self.rc = StrictRedisCluster(startup_nodes=config.startup_nodes)
def _select(self, key_type, kwargs):
partition = str(random.randint(1, kwargs['partition']))
_key_name = key_type + '_r_test_' + partition + ':00' + str(
random.randint(1, kwargs['key_num']))
if key_type == 'string':
self.rc.get(_key_name)
elif key_type == 'hash':
_filed = str(random.randint(1, int(
kwargs['filed_num']))) + ':value_test'
self.rc.hget(_key_name, _filed)
elif key_type == 'list':
_num_1 = random.randint(1, int(kwargs['filed_num']))
_num_2 = random.randint(1, int(kwargs['filed_num']))
if _num_1 < _num_2:
self.rc.lrange(_key_name, _num_1, _num_2)
else:
self.rc.lrange(_key_name, _num_2, _num_1)
elif key_type == 'set':
self.rc.srandmember(_key_name)
else:
exit()
def _update(self, key_type, kwargs):
_value = ''.join(
map(lambda xx: (hex(ord(xx))[2:]),
os.urandom(random.randint(1, 200) / 2)))
partition = str(random.randint(1, kwargs['partition']))
_key_name = key_type + '_r_test_' + partition + ':00' + str(
random.randint(1, kwargs['key_num']))
if key_type == 'string':
self.rc.set(_key_name, _value)
elif key_type == 'hash':
_filed_name = str(random.randint(
1, kwargs['filed_num'])) + ':_value_test'
self.rc.hset(_key_name, _filed_name, _value)
elif key_type == 'list':
_filed_int = random.randint(0, kwargs['filed_num'] - 1)
self.rc.lset(_key_name, _filed_int, _value)
elif key_type == 'set':
self.rc.spop(_key_name)
self.rc.sadd(_key_name, _value)
else:
exit()
def redis_cluster():
'''集群操作'''
redis_nodes = [{
'host': '10.12.28.222',
'port': 6380
}, {
'host': '10.12.28.222',
'port': 6381
}, {
'host': '10.12.28.224',
'port': 6380
}, {
'host': '10.12.28.224',
'port': 6381
}, {
'host': '10.12.28.227',
'port': 6380
}, {
'host': '10.12.28.227',
'port': 6381
}]
try:
r = StrictRedisCluster(startup_nodes=redis_nodes)
except Exception as e:
print("connect error %s" % e)
# string 操作
r.set('thoth:thoth-ai:robot:1', 'kk')
# r.delete('thoth:thoth-ai:robot:1')
print("name is", r.get('thoth:thoth-ai:robot:1'))
# list 操作
r.lpush('thoth:thoth-ai:robot:2', [[1, 2, 3], [2, 3, 4]])
print('list len:', r.llen("thoth:thoth-ai:robot:2")) # list size
print("list ", r.lindex('thoth:thoth-ai:robot:2', 0))
# hash 操作
r.hset('thoth:thoth-ai:robot:3', 'private_vector', [[1, 2, 3], [2, 3, 4]])
r.hset('thoth:thoth-ai:robot:3', 'public_vector', [['4', 3, 2], [0, 1, 1]])
pv = r.hget(
'thoth:thoth-ai:robot:3',
'public_vector',
)
print('hash.robot3.public_vector:', pv)
aaa = pv.decode('utf-8')
print(type(aaa), aaa)
b = eval(aaa) # eval 函数妙用：将string‘[1,2,3]’--->list [1,2,3]
print(type(b), b)
def test_redis_cluster_connect():
"""
测试redis集群链接。
:return:
"""
try:
rds_cluster = StrictRedisCluster(startup_nodes=redis_nodes,
password=redis_pass,
decode_responses=True)
except RedisClusterException:
rds_cluster = StrictRedisCluster(startup_nodes=redis_nodes,
decode_responses=True)
value = rds_cluster.hget('my_hash', "my_key")
common_logger.info(value)
def exec_redis(redis_nodes,cmd_list):
'''
redis_nodes = [{'host':'10.101.104.132','port':1321},
{'host':'10.101.104.132','port':1322},
{'host':'10.101.104.132','port':1323}
{'host':'10.101.104.132','port':1323,'password':None,'db':0}
]
cmd_list = [ #暂时只支持read，而且只有这几种
"get key1",
"llen lis1",
"smembers set1",
"scard set1",
"hgetall hashtable",
"lindex list1 2",
"hget hashtable key1",
"lrange list1 1 3",
]
'''
redisconn = StrictRedisCluster(startup_nodes=redis_nodes,decode_responses=True)
res = {}
for cmd in cmd_list:
cmd = cmd.strip()
arr = re.split(" +",cmd)
type = arr[0].lower()
if type == 'get':
tmp = redisconn.get(arr[1])
elif type == 'hgetall':
tmp = redisconn.hgetall(arr[1])
elif type == 'llen':
tmp = redisconn.llen(arr[1])
elif type == 'smembers':
tmp = redisconn.smembers(arr[1])
elif type == 'scard':
tmp = redisconn.scard(arr[1])
elif type == 'hget':
tmp = redisconn.hget(arr[1],arr[2])
elif type == 'lrange':
tmp = redisconn.lrange(arr[1],arr[2],arr[3])
elif type == 'lindex':
tmp = redisconn.lrange(arr[1],arr[2])
else:
tmp = 'this cmd is not support yeild'
res[cmd] = [json_decode(tmp)]
return res
def redis_cluster():
redis_nodes = [{
'host': '10.1.120.112',
'port': 8001,
'password': 123456
}, {
'host': '10.1.120.111',
'port': 8002,
'password': 123456
}, {
'host': '10.1.120.85',
'port': 8003,
'password': 123456
}, {
'host': '10.1.120.112',
'port': 9001,
'password': 123456
}, {
'host': '10.1.120.111',
'port': 9002,
'password': 123456
}, {
'host': '10.1.120.85',
'port': 9003,
'password': 123456
}, {
'host': '10.1.120.112',
'port': 9101,
'password': 123456
}, {
'host': '10.1.120.111',
'port': 9102,
'password': 123456
}, {
'host': '10.1.120.85',
'port': 9103,
'password': 123456
}]
redisconn = StrictRedisCluster(startup_nodes=redis_nodes, password=123456)
redisconn.hset("test1", "hello", '你好')
print(redisconn.hget('test1', 'hello').decode('utf-8'))
class RedisClusterNode(object):
def __init__(self, node_list):
self.node_list = node_list
self.__connect()
#析构函数，断开连接
def __del__(self):
if self.connect:
pass
def __connect(self):
try:
self.connect = StrictRedisCluster(startup_nodes=self.node_list)
except Exception as e:
self.connect = None
print("Connect redisCluster node error!", e)
return
#set方法
def set(self, key, value):
try:
self.connect.set(key, value)
return True
except Exception as e:
print("RedisUtil excute set method failed!", e)
return False
#get方法
def get(self, key):
try:
return self.connect.get(key)
except Exception as e:
print("RedisUtil excute get method failed!", e)
return None
#del方法
def delete(self, key):
try:
self.connect.delete(key)
return True
except Exception as e:
print("RedisUtil excute delete method failed!", e)
return False
#mset方法
def mset(self, map):
try:
self.connect.mset(map)
return True
except Exception as e:
print("RedisUtil excute mset method failed!", e)
return False
#mget方法
def mget(self, array_keys):
try:
return self.connect.mget(array_keys)
except Exception as e:
print("RedisUtil excute mget method failed!", e)
return None
#hset方法
def hset(self, key, field, value):
try:
self.connect.hset(key, field, value)
return True
except Exception as e:
print("RedisUtil excute hset method failed!", e)
return False
#hget方法
def hget(self, key, field):
try:
return self.connect.hget(key, field)
except Exception as e:
print("RedisUtil excute hget method failed!", e)
return None
class TcRedis():
def __init__(self, *nodelist):
self.redisinst = None
if len(nodelist) == 0:
self.redisinst = redis.StrictRedis(host='localhost',
port=6379,
db=0)
elif len(nodelist) == 1:
host, port = nodelist[0].split(':')
self.redisinst = redis.StrictRedis(host=host, port=int(port))
elif len(nodelist) > 1:
redis_nodes = []
for node in nodelist:
host, port = node.split(':')
redis_nodes.append({"host": host, "port": int(port)})
self.redisinst = StrictRedisCluster(startup_nodes=redis_nodes,
decode_responses=True)
def set(self, key, value):
return self.redisinst.set(key, value)
def get(self, key):
return self.redisinst.get(key)
def delete(self, key):
return self.redisinst.delete(key)
def ttl(self, key):
return self.redisinst.ttl(key)
def scan(self, cursor=0, match=None, count=2000):
depth = 1000
cursor = 0
result = []
for i in range(depth):
cursor, result0 = self.redisinst.scan(cursor, match=match)
if len(result0) != 0:
result = result + result0
if cursor == 0:
break
return result
def hset(self, key, name, value):
return self.redisinst.hset(key, name, value)
def hget(self, key, name):
return self.redisinst.hget(key, name)
def lpush(self, key, *values):
return self.redisinst.lpush(key, *values)
def lpop(self, key):
return self.redisinst.lpop(key)
def sadd(self, key, *values):
return self.redisinst.sadd(key, *values)
def smembers(self, key):
return self.redisinst.smembers(key)
def zadd(self, key, *args, **kwargs):
return self.redisinst.zadd(key, *args, **kwargs)
def zrange(self, key, start, end):
return self.redisinst.zrange(key, start, end)
class RedisDB:
def __init__(self,
ip_ports=None,
db=None,
user_pass=None,
url=None,
decode_responses=True,
service_name=None,
max_connections=32,
**kwargs):
"""
redis的封装
Args:
ip_ports: ip:port 多个可写为列表或者逗号隔开 如 ip1:port1,ip2:port2 或 ["ip1:port1", "ip2:port2"]
db:
user_pass:
url:
decode_responses:
service_name: 适用于redis哨兵模式
"""
# 可能会改setting中的值，所以此处不能直接赋值为默认值，需要后加载赋值
if ip_ports is None:
ip_ports = setting.REDISDB_IP_PORTS
if db is None:
db = setting.REDISDB_DB
if user_pass is None:
user_pass = setting.REDISDB_USER_PASS
if service_name is None:
service_name = setting.REDISDB_SERVICE_NAME
self._is_redis_cluster = False
try:
if not url:
ip_ports = (ip_ports if isinstance(ip_ports, list) else
ip_ports.split(","))
if len(ip_ports) > 1:
startup_nodes = []
for ip_port in ip_ports:
ip, port = ip_port.split(":")
startup_nodes.append({"host": ip, "port": port})
if service_name:
log.debug("使用redis哨兵模式")
hosts = [(node["host"], node["port"])
for node in startup_nodes]
sentinel = Sentinel(hosts, socket_timeout=3, **kwargs)
self._redis = sentinel.master_for(
service_name,
password=user_pass,
db=db,
redis_class=redis.StrictRedis,
decode_responses=decode_responses,
max_connections=max_connections,
**kwargs)
else:
log.debug("使用redis集群模式")
self._redis = StrictRedisCluster(
startup_nodes=startup_nodes,
decode_responses=decode_responses,
password=user_pass,
max_connections=max_connections,
**kwargs)
self._is_redis_cluster = True
else:
ip, port = ip_ports[0].split(":")
self._redis = redis.StrictRedis(
host=ip,
port=port,
db=db,
password=user_pass,
decode_responses=decode_responses,
max_connections=max_connections,
**kwargs)
else:
self._redis = redis.StrictRedis.from_url(
url, decode_responses=decode_responses)
except Exception as e:
raise
else:
if not url:
log.debug("连接到redis数据库 %s db%s" % (ip_ports, db))
else:
log.debug("连接到redis数据库 %s" % (url))
self._ip_ports = ip_ports
self._db = db
self._user_pass = user_pass
self._url = url
def __repr__(self):
if self._url:
return "<Redisdb url:{}>".format(self._url)
return "<Redisdb ip_ports: {} db:{} user_pass:{}>".format(
self._ip_ports, self._db, self._user_pass)
@classmethod
def from_url(cls, url):
return cls(url=url)
def sadd(self, table, values):
"""
@summary: 使用无序set集合存储数据， 去重
---------
@param table:
@param values: 值； 支持list 或 单个值
---------
@result: 若库中存在 返回0，否则入库，返回1。 批量添加返回None
"""
if isinstance(values, list):
pipe = self._redis.pipeline(
transaction=True
) # redis-py默认在执行每次请求都会创建（连接池申请连接）和断开（归还连接池）一次连接操作，如果想要在一次请求中指定多个命令，则可以使用pipline实现一次请求指定多个命令，并且默认情况下一次pipline 是原子性操作。
if not self._is_redis_cluster:
pipe.multi()
for value in values:
pipe.sadd(table, value)
pipe.execute()
else:
return self._redis.sadd(table, values)
def sget(self, table, count=1, is_pop=True):
"""
返回 list 如 ['1'] 或 []
@param table:
@param count:
@param is_pop:
@return:
"""
datas = []
if is_pop:
count = count if count <= self.sget_count(
table) else self.sget_count(table)
if count:
if count > 1:
pipe = self._redis.pipeline(
transaction=True
) # redis-py默认在执行每次请求都会创建（连接池申请连接）和断开（归还连接池）一次连接操作，如果想要在一次请求中指定多个命令，则可以使用pipline实现一次请求指定多个命令，并且默认情况下一次pipline 是原子性操作。
if not self._is_redis_cluster:
pipe.multi()
while count:
pipe.spop(table)
count -= 1
datas = pipe.execute()
else:
datas.append(self._redis.spop(table))
else:
datas = self._redis.srandmember(table, count)
return datas
def srem(self, table, values):
"""
@summary: 移除集合中的指定元素
---------
@param table:
@param values: 一个或者列表
---------
@result:
"""
if isinstance(values, list):
pipe = self._redis.pipeline(
transaction=True
) # redis-py默认在执行每次请求都会创建（连接池申请连接）和断开（归还连接池）一次连接操作，如果想要在一次请求中指定多个命令，则可以使用pipline实现一次请求指定多个命令，并且默认情况下一次pipline 是原子性操作。
if not self._is_redis_cluster:
pipe.multi()
for value in values:
pipe.srem(table, value)
pipe.execute()
else:
self._redis.srem(table, values)
def sget_count(self, table):
return self._redis.scard(table)
def sdelete(self, table):
"""
@summary: 删除set集合的大键（数据量大的表）
删除大set键，使用sscan命令，每次扫描集合中500个元素，再用srem命令每次删除一个键
若直接用delete命令，会导致Redis阻塞，出现故障切换和应用程序崩溃的故障。
---------
@param table:
---------
@result:
"""
# 当 SCAN 命令的游标参数被设置为 0 时， 服务器将开始一次新的迭代， 而当服务器向用户返回值为 0 的游标时， 表示迭代已结束
cursor = "0"
while cursor != 0:
cursor, data = self._redis.sscan(table, cursor=cursor, count=500)
for item in data:
# pipe.srem(table, item)
self._redis.srem(table, item)
# pipe.execute()
def sismember(self, table, key):
"Return a boolean indicating if ``value`` is a member of set ``name``"
return self._redis.sismember(table, key)
def zadd(self, table, values, prioritys=0):
"""
@summary: 使用有序set集合存储数据， 去重(值存在更新)
---------
@param table:
@param values: 值； 支持list 或 单个值
@param prioritys: 优先级； double类型，支持list 或 单个值。 根据此字段的值来排序, 值越小越优先。 可不传值，默认value的优先级为0
---------
@result:若库中存在 返回0，否则入库，返回1。 批量添加返回 [0, 1 ...]
"""
if isinstance(values, list):
if not isinstance(prioritys, list):
prioritys = [prioritys] * len(values)
else:
assert len(values) == len(prioritys), "values值要与prioritys值一一对应"
pipe = self._redis.pipeline(transaction=True)
if not self._is_redis_cluster:
pipe.multi()
for value, priority in zip(values, prioritys):
pipe.zadd(table, priority, value)
return pipe.execute()
else:
return self._redis.zadd(table, prioritys, values)
def zget(self, table, count=1, is_pop=True):
"""
@summary: 从有序set集合中获取数据 优先返回分数小的（优先级高的）
---------
@param table:
@param count: 数量 -1 返回全部数据
@param is_pop:获取数据后，是否在原set集合中删除，默认是
---------
@result: 列表
"""
start_pos = 0 # 包含
end_pos = count - 1 if count > 0 else count
pipe = self._redis.pipeline(
transaction=True
) # redis-py默认在执行每次请求都会创建（连接池申请连接）和断开（归还连接池）一次连接操作，如果想要在一次请求中指定多个命令，则可以使用pipline实现一次请求指定多个命令，并且默认情况下一次pipline 是原子性操作。
if not self._is_redis_cluster:
pipe.multi(
) # 标记事务的开始 参考 http://www.runoob.com/redis/redis-transactions.html
pipe.zrange(table, start_pos, end_pos) # 取值
if is_pop:
pipe.zremrangebyrank(table, start_pos, end_pos) # 删除
results, *count = pipe.execute()
return results
def zremrangebyscore(self, table, priority_min, priority_max):
"""
根据分数移除成员 闭区间
@param table:
@param priority_min:
@param priority_max:
@return: 被移除的成员个数
"""
return self._redis.zremrangebyscore(table, priority_min, priority_max)
def zrangebyscore(self,
table,
priority_min,
priority_max,
count=None,
is_pop=True):
"""
@summary: 返回指定分数区间的数据 闭区间
---------
@param table:
@param priority_min: 优先级越小越优先
@param priority_max:
@param count: 获取的数量，为空则表示分数区间内的全部数据
@param is_pop: 是否删除
---------
@result:
"""
# 使用lua脚本， 保证操作的原子性
lua = """
local key = KEYS[1]
local min_score = ARGV[2]
local max_score = ARGV[3]
local is_pop = ARGV[4]
local count = ARGV[5]
-- 取值
local datas = nil
if count then
datas = redis.call('zrangebyscore', key, min_score, max_score, 'limit', 0, count)
else
datas = redis.call('zrangebyscore', key, min_score, max_score)
end
-- 删除redis中刚取到的值
if (is_pop) then
for i=1, #datas do
redis.call('zrem', key, datas[i])
end
end
return datas
"""
cmd = self._redis.register_script(lua)
if count:
res = cmd(keys=[table],
args=[table, priority_min, priority_max, is_pop, count])
else:
res = cmd(keys=[table],
args=[table, priority_min, priority_max, is_pop])
return res
def zrangebyscore_increase_score(self,
table,
priority_min,
priority_max,
increase_score,
count=None):
"""
@summary: 返回指定分数区间的数据 闭区间， 同时修改分数
---------
@param table:
@param priority_min: 最小分数
@param priority_max: 最大分数
@param increase_score: 分数值增量 正数则在原有的分数上叠加，负数则相减
@param count: 获取的数量，为空则表示分数区间内的全部数据
---------
@result:
"""
# 使用lua脚本， 保证操作的原子性
lua = """
local key = KEYS[1]
local min_score = ARGV[1]
local max_score = ARGV[2]
local increase_score = ARGV[3]
local count = ARGV[4]
-- 取值
local datas = nil
if count then
datas = redis.call('zrangebyscore', key, min_score, max_score, 'limit', 0, count)
else
datas = redis.call('zrangebyscore', key, min_score, max_score)
end
--修改优先级
for i=1, #datas do
redis.call('zincrby', key, increase_score, datas[i])
end
return datas
"""
cmd = self._redis.register_script(lua)
if count:
res = cmd(keys=[table],
args=[priority_min, priority_max, increase_score, count])
else:
res = cmd(keys=[table],
args=[priority_min, priority_max, increase_score])
return res
def zrangebyscore_set_score(self,
table,
priority_min,
priority_max,
score,
count=None):
"""
@summary: 返回指定分数区间的数据 闭区间， 同时修改分数
---------
@param table:
@param priority_min: 最小分数
@param priority_max: 最大分数
@param score: 分数值
@param count: 获取的数量，为空则表示分数区间内的全部数据
---------
@result:
"""
# 使用lua脚本， 保证操作的原子性
lua = """
local key = KEYS[1]
local min_score = ARGV[1]
local max_score = ARGV[2]
local set_score = ARGV[3]
local count = ARGV[4]
-- 取值
local datas = nil
if count then
datas = redis.call('zrangebyscore', key, min_score, max_score, 'withscores','limit', 0, count)
else
datas = redis.call('zrangebyscore', key, min_score, max_score, 'withscores')
end
local real_datas = {} -- 数据
--修改优先级
for i=1, #datas, 2 do
local data = datas[i]
local score = datas[i+1]
table.insert(real_datas, data) -- 添加数据
redis.call('zincrby', key, set_score - score, datas[i])
end
return real_datas
"""
cmd = self._redis.register_script(lua)
if count:
res = cmd(keys=[table],
args=[priority_min, priority_max, score, count])
else:
res = cmd(keys=[table], args=[priority_min, priority_max, score])
return res
def zget_count(self, table, priority_min=None, priority_max=None):
"""
@summary: 获取表数据的数量
---------
@param table:
@param priority_min:优先级范围 最小值（包含）
@param priority_max:优先级范围 最大值（包含）
---------
@result:
"""
if priority_min != None and priority_max != None:
return self._redis.zcount(table, priority_min, priority_max)
else:
return self._redis.zcard(table)
def zrem(self, table, values):
"""
@summary: 移除集合中的指定元素
---------
@param table:
@param values: 一个或者列表
---------
@result:
"""
if isinstance(values, list):
pipe = self._redis.pipeline(
transaction=True
) # redis-py默认在执行每次请求都会创建（连接池申请连接）和断开（归还连接池）一次连接操作，如果想要在一次请求中指定多个命令，则可以使用pipline实现一次请求指定多个命令，并且默认情况下一次pipline 是原子性操作。
if not self._is_redis_cluster:
pipe.multi()
for value in values:
pipe.zrem(table, value)
pipe.execute()
else:
self._redis.zrem(table, values)
def zexists(self, table, values):
"""
利用zscore判断某元素是否存在
@param values:
@return:
"""
is_exists = []
if isinstance(values, list):
pipe = self._redis.pipeline(
transaction=True
) # redis-py默认在执行每次请求都会创建（连接池申请连接）和断开（归还连接池）一次连接操作，如果想要在一次请求中指定多个命令，则可以使用pipline实现一次请求指定多个命令，并且默认情况下一次pipline 是原子性操作。
pipe.multi()
for value in values:
pipe.zscore(table, value)
is_exists_temp = pipe.execute()
for is_exist in is_exists_temp:
if is_exist != None:
is_exists.append(1)
else:
is_exists.append(0)
else:
is_exists = self._redis.zscore(table, values)
is_exists = 1 if is_exists != None else 0
return is_exists
def lpush(self, table, values):
if isinstance(values, list):
pipe = self._redis.pipeline(
transaction=True
) # redis-py默认在执行每次请求都会创建（连接池申请连接）和断开（归还连接池）一次连接操作，如果想要在一次请求中指定多个命令，则可以使用pipline实现一次请求指定多个命令，并且默认情况下一次pipline 是原子性操作。
if not self._is_redis_cluster:
pipe.multi()
for value in values:
pipe.rpush(table, value)
pipe.execute()
else:
return self._redis.rpush(table, values)
def lpop(self, table, count=1):
"""
@summary:
---------
@param table:
@param count:
---------
@result: count>1时返回列表
"""
datas = None
count = count if count <= self.lget_count(table) else self.lget_count(
table)
if count:
if count > 1:
pipe = self._redis.pipeline(
transaction=True
) # redis-py默认在执行每次请求都会创建（连接池申请连接）和断开（归还连接池）一次连接操作，如果想要在一次请求中指定多个命令，则可以使用pipline实现一次请求指定多个命令，并且默认情况下一次pipline 是原子性操作。
if not self._is_redis_cluster:
pipe.multi()
while count:
pipe.lpop(table)
count -= 1
datas = pipe.execute()
else:
datas = self._redis.lpop(table)
return datas
def rpoplpush(self, from_table, to_table=None):
"""
将列表 from_table 中的最后一个元素(尾元素)弹出，并返回给客户端。
将 from_table 弹出的元素插入到列表 to_table ，作为 to_table 列表的的头元素。
如果 from_table 和 to_table 相同，则列表中的表尾元素被移动到表头，并返回该元素，可以把这种特殊情况视作列表的旋转(rotation)操作
@param from_table:
@param to_table:
@return:
"""
if not to_table:
to_table = from_table
return self._redis.rpoplpush(from_table, to_table)
def lget_count(self, table):
return self._redis.llen(table)
def lrem(self, table, value, num=0):
return self._redis.lrem(table, value, num)
def hset(self, table, key, value):
"""
@summary:
如果 key 不存在，一个新的哈希表被创建并进行 HSET 操作。
如果域 field 已经存在于哈希表中，旧值将被覆盖
---------
@param table:
@param key:
@param value:
---------
@result: 1 新插入； 0 覆盖
"""
return self._redis.hset(table, key, value)
def hset_batch(self, table, datas):
"""
批量插入
Args:
datas:
[[key, value]]
Returns:
"""
pipe = self._redis.pipeline(transaction=True)
if not self._is_redis_cluster:
pipe.multi()
for key, value in datas:
pipe.hset(table, key, value)
return pipe.execute()
def hincrby(self, table, key, increment):
return self._redis.hincrby(table, key, increment)
def hget(self, table, key, is_pop=False):
if not is_pop:
return self._redis.hget(table, key)
else:
lua = """
local key = KEYS[1]
local field = ARGV[1]
-- 取值
local datas = redis.call('hget', key, field)
-- 删除值
redis.call('hdel', key, field)
return datas
"""
cmd = self._redis.register_script(lua)
res = cmd(keys=[table], args=[key])
return res
def hgetall(self, table):
return self._redis.hgetall(table)
def hexists(self, table, key):
return self._redis.hexists(table, key)
def hdel(self, table, *keys):
"""
@summary: 删除对应的key 可传多个
---------
@param table:
@param *keys:
---------
@result:
"""
self._redis.hdel(table, *keys)
def hget_count(self, table):
return self._redis.hlen(table)
def setbit(self, table, offsets, values):
"""
设置字符串数组某一位的值， 返回之前的值
@param table:
@param offsets: 支持列表或单个值
@param values: 支持列表或单个值
@return: list / 单个值
"""
if isinstance(offsets, list):
if not isinstance(values, list):
values = [values] * len(offsets)
else:
assert len(offsets) == len(values), "offsets值要与values值一一对应"
pipe = self._redis.pipeline(
transaction=True
) # redis-py默认在执行每次请求都会创建（连接池申请连接）和断开（归还连接池）一次连接操作，如果想要在一次请求中指定多个命令，则可以使用pipline实现一次请求指定多个命令，并且默认情况下一次pipline 是原子性操作。
pipe.multi()
for offset, value in zip(offsets, values):
pipe.setbit(table, offset, value)
return pipe.execute()
else:
return self._redis.setbit(table, offsets, values)
def getbit(self, table, offsets):
"""
取字符串数组某一位的值
@param table:
@param offsets: 支持列表
@return: list / 单个值
"""
if isinstance(offsets, list):
pipe = self._redis.pipeline(
transaction=True
) # redis-py默认在执行每次请求都会创建（连接池申请连接）和断开（归还连接池）一次连接操作，如果想要在一次请求中指定多个命令，则可以使用pipline实现一次请求指定多个命令，并且默认情况下一次pipline 是原子性操作。
pipe.multi()
for offset in offsets:
pipe.getbit(table, offset)
return pipe.execute()
else:
return self._redis.getbit(table, offsets)
def bitcount(self, table):
return self._redis.bitcount(table)
def strset(self, table, value, **kwargs):
return self._redis.set(table, value, **kwargs)
def str_incrby(self, table, value):
return self._redis.incrby(table, value)
def strget(self, table):
return self._redis.get(table)
def strlen(self, table):
return self._redis.strlen(table)
def getkeys(self, regex):
return self._redis.keys(regex)
def exists_key(self, key):
return self._redis.exists(key)
def set_expire(self, key, seconds):
"""
@summary: 设置过期时间
---------
@param key:
@param seconds: 秒
---------
@result:
"""
self._redis.expire(key, seconds)
def clear(self, table):
try:
self._redis.delete(table)
except Exception as e:
log.error(e)
def get_redis_obj(self):
return self._redis
class RedisClient(object):
def __init__(self, name='common', db=0, host=None, port=None):
if host is None and port is None:
self.config = DBConfigParser().get_config(
server='redis_common_colony', key='colony')
# self.config = DBConfigParser().get_config(server='redis_common_colony', key='105-62-93colony')
self.host = self.config.get('host')
self.port = self.config.get('port')
self.db = self.config.get('db')
if '|' in self.host:
host_list = self.host.split('|')
redis_nodes = []
for ho in host_list:
redis_nodes.append({
'host': str(ho),
'port': self.port,
'db': self.db
})
self.conn = StrictRedisCluster(startup_nodes=redis_nodes)
else:
self.conn = redis.Redis(host=self.host,
port=self.port,
db=self.db)
else:
self.host = host
self.port = port
self.db = db
self.conn = redis.Redis(host=self.host, port=self.port, db=self.db)
self.name = name
def get(self):
"""
随机从redis里获取一个ip
:return:
"""
key = self.conn.hgetall(name=self.name)
rkey = random.choice(list(key.keys())) if key else None
if isinstance(rkey, bytes):
return rkey.decode('utf-8')
else:
return rkey
def save(self, key):
"""
保存ip
:param key:
:return:
"""
key = json.dumps(key) if isinstance(key, (dict, list)) else key
return self.conn.hincrby(self.name, key, str(int(time.time())))
def get_value(self, key):
"""
获取ip的值
:param key:
:return:
"""
value = self.conn.hget(self.name, key)
return value if value else None
def pop(self):
"""
获取一个ip并从池中删除
:return:
"""
key = self.get()
if key:
self.conn.hdel(self.name, key)
return key
def del_ip(self, key):
"""
删除指定ip
:param key:
:return:
"""
self.conn.hdel(self.name, key)
def del_all(self):
"""
删除该表所有ip
:return:
"""
self.conn.flushdb()
def get_all(self):
"""
获取所有的ip
:return:
"""
if sys.version_info.major == 3:
return [
key.decode('utf-8')
for key in self.conn.hgetall(self.name).keys()
]
else:
return self.conn.hgetall(self.name).keys()
def get_count(self):
"""
获取池里ip数量
:return:
"""
return self.conn.hlen(self.name)
class RedisClient(object):
def __init__(self, key, startup_nodes):
"""
init cluster
"""
self.key = key
self.conn = StrictRedisCluster(startup_nodes=startup_nodes,
decode_responses=True)
def hdel(self, field):
"""
delete an item
:param field:
:return:
"""
self.conn.hdel(self.key, field)
def hexists(self, field):
"""
判断 key 中是否含有 field
:param field:
:return:
"""
return self.conn.hexists(self.key, field)
def hget(self, field):
"""
返回key中指定 field 中的 value
:param field:
:return:
"""
value = self.conn.hget(self.key, field)
if isinstance(value, bytes):
return value.decode('utf-8')
else:
return value if value else None
def hgetall(self):
"""
获取 {filed: value, field1: value1....}
:return:
"""
all_dict = self.conn.hgetall(self.key)
if not all_dict:
return
elif sys.version_info.major == 3:
return {
field.decode('utf-8'): value.decode('utf-8')
for field, value in all_dict.items()
}
else:
return all_dict
def hkeys(self):
"""
获取key中所有field
:return:
"""
field = self.conn.hkeys(self.key)
if isinstance(field, bytes):
return field.decode('utf-8')
else:
return field if field else None
def hlen(self):
"""
获取所有 filed 数量
:return:
"""
return self.conn.hlen(self.key)
def hset(self, field, value):
"""
设置 field: value
:param field:
:param value:
:return:
"""
self.conn.hset(self.key, field, value)
def hvals(self):
"""
获取所有values
:return:
"""
values = self.conn.hvals(self.key)
if not values:
return
elif sys.version_info.major == 3:
return [value.decode('utf-8') for value in values]
else:
return values
def change_key(self, key):
"""
替换 key
:param key:
:return:
"""
self.key = key
# ===============================================
def blpop(self, timeout):
self.conn.blpop(self.key, timeout=timeout)
def brpop(self, timeout):
self.conn.brpop(self.key, timeout=timeout)
def brpoplpush(self, dst, timeout):
self.conn.brpoplpush(self.key, dst=dst, timeout=timeout)
def lindex(self, i):
self.conn.lindex(self.key, index=i)
def llen(self):
self.conn.llen(self.key)
def lpop(self):
self.conn.lpop(self.key)
def lpush(self):
self.conn.lpush(self.key)
def lrange(self, start, stop):
self.conn.lrange(self.key, start, stop)
def lset(self, i, value):
self.conn.lset(self.key, index=i, value=value)
def rpop(self):
self.conn.rpop(self.key)
def rpoplpush(self, dst):
self.conn.rpoplpush(self.key, dst=dst)
def rpush(self, value):
self.conn.rpush(self.key, value)
class RedisCluster:
def __init__(self):
try:
self.rc = StrictRedisCluster(startup_nodes=StartupNodesServer,
decode_responses=True)
except:
traceback.print_exc()
def count_keys(self): # 查询当前库里有多少key
return self.rc.dbsize()
def exists_key(self, key):
return self.rc.exists(key)
def delete_key(self, key):
self.rc.delete(key)
def rename_key(self, key1, key2):
self.rc.rename(key1, key2)
# String操作
def set_key_value(self, key, value):
self.rc.set(key, value)
def get_key_value(self, key): # 没有对应key返回None
return self.rc.get(key)
# Hash操作
def set_hash(self, key, mapping): # mapping为字典, 已存在key会覆盖mapping
self.rc.hmset(key, mapping)
def delete_hash_field(self, key, field): # 删除hash表中某个字段，无论字段是否存在
self.rc.hdel(key, field)
def exists_hash_field(self, key, field): # 检查hash表中某个字段存在
return self.rc.hexists(key, field)
def get_hash_field(self, key, field): # 获取hash表中指定字段的值, 没有返回None
return self.rc.hget(key, field)
def get_hash_all_field(self, key): # 获取hash表中指定key所有字段和值,以字典形式，没有key返回空字典
return self.rc.hgetall(key)
def increase_hash_field(self, key, field,
increment): # 为hash表key某个字段的整数型值增加increment
self.rc.hincrby(key, field, increment)
# List操作
def rpush_into_lst(self, key, value): # url从头至尾入列
self.rc.rpush(key, value)
def lpush_into_lst(self, key, value): # url从尾至头入列
self.rc.lpush(key, value)
def lpop_lst_item(self, key): # 从头取出列表第一个元素，没有返回None
return self.rc.lpop(key)
def blpop_lst_item(
self, key): # 从头取出列表第一个元素(元组形式,值为元祖[1], 元祖[0]为key名)，并设置超时，超时返回None
return self.rc.blpop(key, timeout=1)
def rpop_lst_item(self, key): # 从尾取出列表最后一个元素，没有返回None
return self.rc.rpop(key)
def brpop_lst_item(
self,
key): # 从尾取出列表最后一个元素(元组形式,值为元祖[1], 元祖[0]为key名)，并设置超时，超时返回None
return self.rc.brpop(key, timeout=1)
# Set操作
def add_set(self, key, value):
self.rc.sadd(key, value)
def is_member(self, key, value):
return self.rc.sismember(key, value)
def pop_member(self, key): # 随机移除一个值并返回该值,没有返回None
return self.rc.spop(key)
def pop_members(self, key, num): # 随机取出num个值（非移除），列表形式返回这些值，没有返回空列表
return self.rc.srandmember(key, num)
def remove_member(self, key, value): # 移除集合中指定元素
self.rc.srem(key, value)
def get_all_members(self, key): # 返回集合中全部元素,不删除
return self.rc.smembers(key)
def remove_into(self, key1, key2, value): # 把集合key1中value元素移入集合key2中
self.rc.smove(key1, key2, value)
def count_members(self, key): # 计算集合中成员数量
return self.rc.scard(key)
class Leaderboard(object):
VERSION = '3.5.0'
DEFAULT_PAGE_SIZE = 25
DEFAULT_REDIS_HOST = 'localhost'
DEFAULT_REDIS_PORT = 6379
DEFAULT_REDIS_DB = 0
DEFAULT_MEMBER_DATA_NAMESPACE = 'member_data'
DEFAULT_GLOBAL_MEMBER_DATA = False
DEFAULT_POOLS = {}
ASC = 'asc'
DESC = 'desc'
MEMBER_KEY = 'member'
MEMBER_DATA_KEY = 'member_data'
SCORE_KEY = 'score'
RANK_KEY = 'rank'
DEFAULT_CLUSTER_MODE = False
@classmethod
def pool(self, host, port, db, pools={}, **options):
'''
Fetch a redis conenction pool for the unique combination of host
and port. Will create a new one if there isn't one already.
'''
key = (host, port, db)
rval = pools.get(key)
if not isinstance(rval, ConnectionPool):
rval = ConnectionPool(host=host, port=port, db=db, **options)
pools[key] = rval
return rval
def __init__(self, leaderboard_name, **options):
'''
Initialize a connection to a specific leaderboard. By default, will use a
redis connection pool for any unique host:port:db pairing.
The options and their default values (if any) are:
host : the host to connect to if creating a new handle ('localhost')
port : the port to connect to if creating a new handle (6379)
db : the redis database to connect to if creating a new handle (0)
page_size : the default number of items to return in each page (25)
connection : an existing redis handle if re-using for this leaderboard
connection_pool : redis connection pool to use if creating a new handle
'''
self.leaderboard_name = leaderboard_name
self.options = options
self.page_size = self.options.pop('page_size', self.DEFAULT_PAGE_SIZE)
if self.page_size < 1:
self.page_size = self.DEFAULT_PAGE_SIZE
self.member_data_namespace = self.options.pop(
'member_data_namespace', self.DEFAULT_MEMBER_DATA_NAMESPACE)
self.global_member_data = self.options.pop(
'global_member_data', self.DEFAULT_GLOBAL_MEMBER_DATA)
self.order = self.options.pop('order', self.DESC).lower()
if not self.order in [self.ASC, self.DESC]:
raise ValueError("%s is not one of [%s]" %
(self.order, ",".join([self.ASC, self.DESC])))
redis_connection = self.options.pop('redis_connection', None)
if redis_connection:
# allow the developer to pass a raw redis connection and
# we will use it directly instead of creating a new one
self.redis_connection = redis_connection
else:
connection = self.options.pop('connection', None)
if isinstance(connection, (StrictRedis, Redis)):
self.options['connection_pool'] = connection.connection_pool
host = self.options.pop('host', self.DEFAULT_REDIS_HOST)
port = self.options.pop('port', self.DEFAULT_REDIS_PORT)
if 'connection_pool' not in self.options:
self.options['connection_pool'] = self.pool(
host, port, self.options.pop('db', self.DEFAULT_REDIS_DB),
self.options.pop('pools', self.DEFAULT_POOLS),
**self.options)
if not self.options.get("cluster_mode", self.DEFAULT_CLUSTER_MODE):
self.redis_connection = Redis(**self.options)
else:
startup_nodes = [{"host": host, "port": port}]
self.redis_connection = StrictRedisCluster(
startup_nodes=startup_nodes,
decode_responses=True,
skip_full_coverage_check=True)
def delete_leaderboard(self):
'''
Delete the current leaderboard.
'''
self.delete_leaderboard_named(self.leaderboard_name)
def delete_leaderboard_named(self, leaderboard_name):
'''
Delete the named leaderboard.
@param leaderboard_name [String] Name of the leaderboard.
'''
pipeline = self.redis_connection.pipeline()
pipeline.delete(leaderboard_name)
pipeline.delete(self._member_data_key(leaderboard_name))
pipeline.execute()
def rank_member(self, member, score, member_data=None):
'''
Rank a member in the leaderboard.
@param member [String] Member name.
@param score [float] Member score.
@param member_data [String] Optional member data.
'''
self.rank_member_in(self.leaderboard_name, member, score, member_data)
def rank_member_in(self,
leaderboard_name,
member,
score,
member_data=None):
'''
Rank a member in the named leaderboard.
@param leaderboard_name [String] Name of the leaderboard.
@param member [String] Member name.
@param score [float] Member score.
@param member_data [String] Optional member data.
'''
pipeline = self.redis_connection.pipeline()
if isinstance(self.redis_connection, Redis):
pipeline.zadd(leaderboard_name, member, score)
else:
pipeline.zadd(leaderboard_name, score, member)
if member_data:
pipeline.hset(self._member_data_key(leaderboard_name), member,
member_data)
pipeline.execute()
def rank_member_across(self,
leaderboards,
member,
score,
member_data=None):
'''
Rank a member across multiple leaderboards.
@param leaderboards [Array] Leaderboard names.
@param member [String] Member name.
@param score [float] Member score.
@param member_data [String] Optional member data.
'''
pipeline = self.redis_connection.pipeline()
for leaderboard_name in leaderboards:
if isinstance(self.redis_connection, Redis):
pipeline.zadd(leaderboard_name, member, score)
else:
pipeline.zadd(leaderboard_name, score, member)
if member_data:
pipeline.hset(self._member_data_key(leaderboard_name), member,
member_data)
pipeline.execute()
def rank_member_if(self,
rank_conditional,
member,
score,
member_data=None):
'''
Rank a member in the leaderboard based on execution of the +rank_conditional+.
The +rank_conditional+ is passed the following parameters:
member: Member name.
current_score: Current score for the member in the leaderboard.
score: Member score.
member_data: Optional member data.
leaderboard_options: Leaderboard options, e.g. 'reverse': Value of reverse option
@param rank_conditional [function] Function which must return +True+ or +False+ that controls whether or not the member is ranked in the leaderboard.
@param member [String] Member name.
@param score [float] Member score.
@param member_data [String] Optional member_data.
'''
self.rank_member_if_in(self.leaderboard_name, rank_conditional, member,
score, member_data)
def rank_member_if_in(self,
leaderboard_name,
rank_conditional,
member,
score,
member_data=None):
'''
Rank a member in the named leaderboard based on execution of the +rank_conditional+.
The +rank_conditional+ is passed the following parameters:
member: Member name.
current_score: Current score for the member in the leaderboard.
score: Member score.
member_data: Optional member data.
leaderboard_options: Leaderboard options, e.g. 'reverse': Value of reverse option
@param leaderboard_name [String] Name of the leaderboard.
@param rank_conditional [function] Function which must return +True+ or +False+ that controls whether or not the member is ranked in the leaderboard.
@param member [String] Member name.
@param score [float] Member score.
@param member_data [String] Optional member_data.
'''
current_score = self.redis_connection.zscore(leaderboard_name, member)
if current_score is not None:
current_score = float(current_score)
if rank_conditional(self, member, current_score, score, member_data,
{'reverse': self.order}):
self.rank_member_in(leaderboard_name, member, score, member_data)
def rank_members(self, members_and_scores):
'''
Rank an array of members in the leaderboard.
@param members_and_scores [Array] Variable list of members and scores.
'''
self.rank_members_in(self.leaderboard_name, members_and_scores)
def rank_members_in(self, leaderboard_name, members_and_scores):
'''
Rank an array of members in the named leaderboard.
@param leaderboard_name [String] Name of the leaderboard.
@param members_and_scores [Array] Variable list of members and scores.
'''
pipeline = self.redis_connection.pipeline()
for member, score in grouper(2, members_and_scores):
if isinstance(self.redis_connection, Redis):
pipeline.zadd(leaderboard_name, member, score)
else:
pipeline.zadd(leaderboard_name, score, member)
pipeline.execute()
def member_data_for(self, member):
'''
Retrieve the optional member data for a given member in the leaderboard.
@param member [String] Member name.
@return String of optional member data.
'''
return self.member_data_for_in(self.leaderboard_name, member)
def member_data_for_in(self, leaderboard_name, member):
'''
Retrieve the optional member data for a given member in the named leaderboard.
@param leaderboard_name [String] Name of the leaderboard.
@param member [String] Member name.
@return String of optional member data.
'''
return self.redis_connection.hget(
self._member_data_key(leaderboard_name), member)
def members_data_for(self, members):
'''
Retrieve the optional member data for a given list of members in the leaderboard.
@param members [Array] Member names.
@return Array of strings of optional member data.
'''
return self.members_data_for_in(self.leaderboard_name, members)
def members_data_for_in(self, leaderboard_name, members):
'''
Retrieve the optional member data for a given list of members in the named leaderboard.
@param leaderboard_name [String] Name of the leaderboard.
@param members [Array] Member names.
@return Array of strings of optional member data.
'''
return self.redis_connection.hmget(
self._member_data_key(leaderboard_name), members)
def update_member_data(self, member, member_data):
'''
Update the optional member data for a given member in the leaderboard.
@param member [String] Member name.
@param member_data [String] Optional member data.
'''
self.update_member_data_in(self.leaderboard_name, member, member_data)
def update_member_data_in(self, leaderboard_name, member, member_data):
'''
Update the optional member data for a given member in the named leaderboard.
@param leaderboard_name [String] Name of the leaderboard.
@param member [String] Member name.
@param member_data [String] Optional member data.
'''
self.redis_connection.hset(self._member_data_key(leaderboard_name),
member, member_data)
def remove_member_data(self, member):
'''
Remove the optional member data for a given member in the leaderboard.
@param member [String] Member name.
'''
self.remove_member_data_in(self.leaderboard_name, member)
def remove_member_data_in(self, leaderboard_name, member):
'''
Remove the optional member data for a given member in the named leaderboard.
@param leaderboard_name [String] Name of the leaderboard.
@param member [String] Member name.
'''
self.redis_connection.hdel(self._member_data_key(leaderboard_name),
member)
def total_members(self):
'''
Retrieve the total number of members in the leaderboard.
@return total number of members in the leaderboard.
'''
return self.total_members_in(self.leaderboard_name)
def total_members_in(self, leaderboard_name):
'''
Retrieve the total number of members in the named leaderboard.
@param leaderboard_name [String] Name of the leaderboard.
@return the total number of members in the named leaderboard.
'''
return self.redis_connection.zcard(leaderboard_name)
def remove_member(self, member):
'''
Remove a member from the leaderboard.
@param member [String] Member name.
'''
self.remove_member_from(self.leaderboard_name, member)
def remove_member_from(self, leaderboard_name, member):
'''
Remove the optional member data for a given member in the named leaderboard.
@param leaderboard_name [String] Name of the leaderboard.
@param member [String] Member name.
'''
pipeline = self.redis_connection.pipeline()
pipeline.zrem(leaderboard_name, member)
pipeline.hdel(self._member_data_key(leaderboard_name), member)
pipeline.execute()
def total_pages(self, page_size=None):
'''
Retrieve the total number of pages in the leaderboard.
@param page_size [int, nil] Page size to be used when calculating the total number of pages.
@return the total number of pages in the leaderboard.
'''
return self.total_pages_in(self.leaderboard_name, page_size)
def total_pages_in(self, leaderboard_name, page_size=None):
'''
Retrieve the total number of pages in the named leaderboard.
@param leaderboard_name [String] Name of the leaderboard.
@param page_size [int, nil] Page size to be used when calculating the total number of pages.
@return the total number of pages in the named leaderboard.
'''
if page_size is None:
page_size = self.page_size
return int(
math.ceil(
self.total_members_in(leaderboard_name) / float(page_size)))
def total_members_in_score_range(self, min_score, max_score):
'''
Retrieve the total members in a given score range from the leaderboard.
@param min_score [float] Minimum score.
@param max_score [float] Maximum score.
@return the total members in a given score range from the leaderboard.
'''
return self.total_members_in_score_range_in(self.leaderboard_name,
min_score, max_score)
def total_members_in_score_range_in(self, leaderboard_name, min_score,
max_score):
'''
Retrieve the total members in a given score range from the named leaderboard.
@param leaderboard_name Name of the leaderboard.
@param min_score [float] Minimum score.
@param max_score [float] Maximum score.
@return the total members in a given score range from the named leaderboard.
'''
return self.redis_connection.zcount(leaderboard_name, min_score,
max_score)
def total_scores(self):
'''
Sum of scores for all members in the leaderboard.
@return Sum of scores for all members in the leaderboard.
'''
return self.total_scores_in(self.leaderboard_name)
def total_scores_in(self, leaderboard_name):
'''
Sum of scores for all members in the named leaderboard.
@param leaderboard_name Name of the leaderboard.
@return Sum of scores for all members in the named leaderboard.
'''
return sum([
leader[self.SCORE_KEY]
for leader in self.all_leaders_from(self.leaderboard_name)
])
def check_member(self, member):
'''
Check to see if a member exists in the leaderboard.
@param member [String] Member name.
@return +true+ if the member exists in the leaderboard, +false+ otherwise.
'''
return self.check_member_in(self.leaderboard_name, member)
def check_member_in(self, leaderboard_name, member):
'''
Check to see if a member exists in the named leaderboard.
@param leaderboard_name [String] Name of the leaderboard.
@param member [String] Member name.
@return +true+ if the member exists in the named leaderboard, +false+ otherwise.
'''
return self.redis_connection.zscore(leaderboard_name,
member) is not None
def rank_for(self, member):
'''
Retrieve the rank for a member in the leaderboard.
@param member [String] Member name.
@return the rank for a member in the leaderboard.
'''
return self.rank_for_in(self.leaderboard_name, member)
def rank_for_in(self, leaderboard_name, member):
'''
Retrieve the rank for a member in the named leaderboard.
@param leaderboard_name [String] Name of the leaderboard.
@param member [String] Member name.
@return the rank for a member in the leaderboard.
'''
if self.order == self.ASC:
try:
return self.redis_connection.zrank(leaderboard_name,
member) + 1
except:
return None
else:
try:
return self.redis_connection.zrevrank(leaderboard_name,
member) + 1
except:
return None
def score_for(self, member):
'''
Retrieve the score for a member in the leaderboard.
@param member Member name.
@return the score for a member in the leaderboard or +None+ if the member is not in the leaderboard.
'''
return self.score_for_in(self.leaderboard_name, member)
def score_for_in(self, leaderboard_name, member):
'''
Retrieve the score for a member in the named leaderboard.
@param leaderboard_name Name of the leaderboard.
@param member [String] Member name.
@return the score for a member in the leaderboard or +None+ if the member is not in the leaderboard.
'''
score = self.redis_connection.zscore(leaderboard_name, member)
if score is not None:
score = float(score)
return score
def score_and_rank_for(self, member):
'''
Retrieve the score and rank for a member in the leaderboard.
@param member [String] Member name.
@return the score and rank for a member in the leaderboard as a Hash.
'''
return self.score_and_rank_for_in(self.leaderboard_name, member)
def score_and_rank_for_in(self, leaderboard_name, member):
'''
Retrieve the score and rank for a member in the named leaderboard.
@param leaderboard_name [String]Name of the leaderboard.
@param member [String] Member name.
@return the score and rank for a member in the named leaderboard as a Hash.
'''
return {
self.MEMBER_KEY: member,
self.SCORE_KEY: self.score_for_in(leaderboard_name, member),
self.RANK_KEY: self.rank_for_in(leaderboard_name, member)
}
def change_score_for(self, member, delta, member_data=None):
'''
Change the score for a member in the leaderboard by a score delta which can be positive or negative.
@param member [String] Member name.
@param delta [float] Score change.
@param member_data [String] Optional member data.
'''
self.change_score_for_member_in(self.leaderboard_name, member, delta,
member_data)
def change_score_for_member_in(self,
leaderboard_name,
member,
delta,
member_data=None):
'''
Change the score for a member in the named leaderboard by a delta which can be positive or negative.
@param leaderboard_name [String] Name of the leaderboard.
@param member [String] Member name.
@param delta [float] Score change.
@param member_data [String] Optional member data.
'''
pipeline = self.redis_connection.pipeline()
pipeline.zincrby(leaderboard_name, member, delta)
if member_data:
pipeline.hset(self._member_data_key(leaderboard_name), member,
member_data)
pipeline.execute()
def remove_members_in_score_range(self, min_score, max_score):
'''
Remove members from the leaderboard in a given score range.
@param min_score [float] Minimum score.
@param max_score [float] Maximum score.
'''
self.remove_members_in_score_range_in(self.leaderboard_name, min_score,
max_score)
def remove_members_in_score_range_in(self, leaderboard_name, min_score,
max_score):
'''
Remove members from the named leaderboard in a given score range.
@param leaderboard_name [String] Name of the leaderboard.
@param min_score [float] Minimum score.
@param max_score [float] Maximum score.
'''
self.redis_connection.zremrangebyscore(leaderboard_name, min_score,
max_score)
def remove_members_outside_rank(self, rank):
'''
Remove members from the leaderboard in a given rank range.
@param rank [int] the rank (inclusive) which we should keep.
@return the total member count which was removed.
'''
return self.remove_members_outside_rank_in(self.leaderboard_name, rank)
def remove_members_outside_rank_in(self, leaderboard_name, rank):
'''
Remove members from the named leaderboard in a given rank range.
@param leaderboard_name [String] Name of the leaderboard.
@param rank [int] the rank (inclusive) which we should keep.
@return the total member count which was removed.
'''
if self.order == self.DESC:
rank = -(rank) - 1
return self.redis_connection.zremrangebyrank(
leaderboard_name, 0, rank)
else:
return self.redis_connection.zremrangebyrank(
leaderboard_name, rank, -1)
def page_for(self, member, page_size=DEFAULT_PAGE_SIZE):
'''
Determine the page where a member falls in the leaderboard.
@param member [String] Member name.
@param page_size [int] Page size to be used in determining page location.
@return the page where a member falls in the leaderboard.
'''
return self.page_for_in(self.leaderboard_name, member, page_size)
def page_for_in(self,
leaderboard_name,
member,
page_size=DEFAULT_PAGE_SIZE):
'''
Determine the page where a member falls in the named leaderboard.
@param leaderboard [String] Name of the leaderboard.
@param member [String] Member name.
@param page_size [int] Page size to be used in determining page location.
@return the page where a member falls in the leaderboard.
'''
rank_for_member = None
if self.order == self.ASC:
rank_for_member = self.redis_connection.zrank(
leaderboard_name, member)
else:
rank_for_member = self.redis_connection.zrevrank(
leaderboard_name, member)
if rank_for_member is None:
rank_for_member = 0
else:
rank_for_member += 1
return int(math.ceil(float(rank_for_member) / float(page_size)))
def percentile_for(self, member):
'''
Retrieve the percentile for a member in the leaderboard.
@param member [String] Member name.
@return the percentile for a member in the leaderboard. Return +nil+ for a non-existent member.
'''
return self.percentile_for_in(self.leaderboard_name, member)
def percentile_for_in(self, leaderboard_name, member):
'''
Retrieve the percentile for a member in the named leaderboard.
@param leaderboard_name [String] Name of the leaderboard.
@param member [String] Member name.
@return the percentile for a member in the named leaderboard.
'''
if not self.check_member_in(leaderboard_name, member):
return None
responses = self.redis_connection.pipeline().zcard(
leaderboard_name).zrevrank(leaderboard_name, member).execute()
percentile = math.ceil((float(
(responses[0] - responses[1] - 1)) / float(responses[0]) * 100))
if self.order == self.ASC:
return 100 - percentile
else:
return percentile
def score_for_percentile(self, percentile):
'''
Calculate the score for a given percentile value in the leaderboard.
@param percentile [float] Percentile value (0.0 to 100.0 inclusive).
@return the score corresponding to the percentile argument. Return +None+ for arguments outside 0-100 inclusive and for leaderboards with no members.
'''
return self.score_for_percentile_in(self.leaderboard_name, percentile)
def score_for_percentile_in(self, leaderboard_name, percentile):
'''
Calculate the score for a given percentile value in the leaderboard.
@param leaderboard_name [String] Name of the leaderboard.
@param percentile [float] Percentile value (0.0 to 100.0 inclusive).
@return the score corresponding to the percentile argument. Return +None+ for arguments outside 0-100 inclusive and for leaderboards with no members.
'''
if not 0 <= percentile <= 100:
return None
total_members = self.total_members_in(leaderboard_name)
if total_members < 1:
return None
if self.order == self.ASC:
percentile = 100 - percentile
index = (total_members - 1) * (percentile / 100.0)
scores = [
pair[1]
for pair in self.redis_connection.zrange(leaderboard_name,
int(math.floor(index)),
int(math.ceil(index)),
withscores=True)
]
if index == math.floor(index):
return scores[0]
else:
interpolate_fraction = index - math.floor(index)
return scores[0] + interpolate_fraction * (scores[1] - scores[0])
def expire_leaderboard(self, seconds):
'''
Expire the current leaderboard in a set number of seconds. Do not use this with
leaderboards that utilize member data as there is no facility to cascade the
expiration out to the keys for the member data.
@param seconds [int] Number of seconds after which the leaderboard will be expired.
'''
self.expire_leaderboard_for(self.leaderboard_name, seconds)
def expire_leaderboard_for(self, leaderboard_name, seconds):
'''
Expire the given leaderboard in a set number of seconds. Do not use this with
leaderboards that utilize member data as there is no facility to cascade the
expiration out to the keys for the member data.
@param leaderboard_name [String] Name of the leaderboard.
@param seconds [int] Number of seconds after which the leaderboard will be expired.
'''
pipeline = self.redis_connection.pipeline()
pipeline.expire(leaderboard_name, seconds)
pipeline.expire(self._member_data_key(leaderboard_name), seconds)
pipeline.execute()
def expire_leaderboard_at(self, timestamp):
'''
Expire the current leaderboard at a specific UNIX timestamp. Do not use this with
leaderboards that utilize member data as there is no facility to cascade the
expiration out to the keys for the member data.
@param timestamp [int] UNIX timestamp at which the leaderboard will be expired.
'''
self.expire_leaderboard_at_for(self.leaderboard_name, timestamp)
def expire_leaderboard_at_for(self, leaderboard_name, timestamp):
'''
Expire the given leaderboard at a specific UNIX timestamp. Do not use this with
leaderboards that utilize member data as there is no facility to cascade the
expiration out to the keys for the member data.
@param leaderboard_name [String] Name of the leaderboard.
@param timestamp [int] UNIX timestamp at which the leaderboard will be expired.
'''
pipeline = self.redis_connection.pipeline()
pipeline.expireat(leaderboard_name, timestamp)
pipeline.expireat(self._member_data_key(leaderboard_name), timestamp)
pipeline.execute()
def leaders(self, current_page, **options):
'''
Retrieve a page of leaders from the leaderboard.
@param current_page [int] Page to retrieve from the leaderboard.
@param options [Hash] Options to be used when retrieving the page from the leaderboard.
@return a page of leaders from the leaderboard.
'''
return self.leaders_in(self.leaderboard_name, current_page, **options)
def leaders_in(self, leaderboard_name, current_page, **options):
'''
Retrieve a page of leaders from the named leaderboard.
@param leaderboard_name [String] Name of the leaderboard.
@param current_page [int] Page to retrieve from the named leaderboard.
@param options [Hash] Options to be used when retrieving the page from the named leaderboard.
@return a page of leaders from the named leaderboard.
'''
if current_page < 1:
current_page = 1
page_size = options.get('page_size', self.page_size)
index_for_redis = current_page - 1
starting_offset = (index_for_redis * page_size)
if starting_offset < 0:
starting_offset = 0
ending_offset = (starting_offset + page_size) - 1
raw_leader_data = self._range_method(self.redis_connection,
leaderboard_name,
int(starting_offset),
int(ending_offset),
withscores=False)
return self._parse_raw_members(leaderboard_name, raw_leader_data,
**options)
def all_leaders(self, **options):
'''
Retrieve all leaders from the leaderboard.
@param options [Hash] Options to be used when retrieving the leaders from the leaderboard.
@return the leaders from the leaderboard.
'''
return self.all_leaders_from(self.leaderboard_name, **options)
def all_leaders_from(self, leaderboard_name, **options):
'''
Retrieves all leaders from the named leaderboard.
@param leaderboard_name [String] Name of the leaderboard.
@param options [Hash] Options to be used when retrieving the leaders from the named leaderboard.
@return the named leaderboard.
'''
raw_leader_data = self._range_method(self.redis_connection,
leaderboard_name,
0,
-1,
withscores=False)
return self._parse_raw_members(leaderboard_name, raw_leader_data,
**options)
def members_from_score_range(self, minimum_score, maximum_score,
**options):
'''
Retrieve members from the leaderboard within a given score range.
@param minimum_score [float] Minimum score (inclusive).
@param maximum_score [float] Maximum score (inclusive).
@param options [Hash] Options to be used when retrieving the data from the leaderboard.
@return members from the leaderboard that fall within the given score range.
'''
return self.members_from_score_range_in(self.leaderboard_name,
minimum_score, maximum_score,
**options)
def members_from_score_range_in(self, leaderboard_name, minimum_score,
maximum_score, **options):
'''
Retrieve members from the named leaderboard within a given score range.
@param leaderboard_name [String] Name of the leaderboard.
@param minimum_score [float] Minimum score (inclusive).
@param maximum_score [float] Maximum score (inclusive).
@param options [Hash] Options to be used when retrieving the data from the leaderboard.
@return members from the leaderboard that fall within the given score range.
'''
raw_leader_data = []
if self.order == self.DESC:
raw_leader_data = self.redis_connection.zrevrangebyscore(
leaderboard_name, maximum_score, minimum_score)
else:
raw_leader_data = self.redis_connection.zrangebyscore(
leaderboard_name, minimum_score, maximum_score)
return self._parse_raw_members(leaderboard_name, raw_leader_data,
**options)
def members_from_rank_range(self, starting_rank, ending_rank, **options):
'''
Retrieve members from the leaderboard within a given rank range.
@param starting_rank [int] Starting rank (inclusive).
@param ending_rank [int] Ending rank (inclusive).
@param options [Hash] Options to be used when retrieving the data from the leaderboard.
@return members from the leaderboard that fall within the given rank range.
'''
return self.members_from_rank_range_in(self.leaderboard_name,
starting_rank, ending_rank,
**options)
def members_from_rank_range_in(self, leaderboard_name, starting_rank,
ending_rank, **options):
'''
Retrieve members from the named leaderboard within a given rank range.
@param leaderboard_name [String] Name of the leaderboard.
@param starting_rank [int] Starting rank (inclusive).
@param ending_rank [int] Ending rank (inclusive).
@param options [Hash] Options to be used when retrieving the data from the leaderboard.
@return members from the leaderboard that fall within the given rank range.
'''
starting_rank -= 1
if starting_rank < 0:
starting_rank = 0
ending_rank -= 1
if ending_rank > self.total_members_in(leaderboard_name):
ending_rank = self.total_members_in(leaderboard_name) - 1
raw_leader_data = []
if self.order == self.DESC:
raw_leader_data = self.redis_connection.zrevrange(leaderboard_name,
starting_rank,
ending_rank,
withscores=False)
else:
raw_leader_data = self.redis_connection.zrange(leaderboard_name,
starting_rank,
ending_rank,
withscores=False)
return self._parse_raw_members(leaderboard_name, raw_leader_data,
**options)
def top(self, number, **options):
'''
Retrieve members from the leaderboard within a range from 1 to the number given.
@param ending_rank [int] Ending rank (inclusive).
@param options [Hash] Options to be used when retrieving the data from the leaderboard.
@return number from the leaderboard that fall within the given rank range.
'''
return self.top_in(self.leaderboard_name, number, **options)
def top_in(self, leaderboard_name, number, **options):
'''
Retrieve members from the named leaderboard within a range from 1 to the number given.
@param leaderboard_name [String] Name of the leaderboard.
@param starting_rank [int] Starting rank (inclusive).
@param ending_rank [int] Ending rank (inclusive).
@param options [Hash] Options to be used when retrieving the data from the leaderboard.
@return members from the leaderboard that fall within the given rank range.
'''
return self.members_from_rank_range_in(leaderboard_name, 1, number,
**options)
def member_at(self, position, **options):
'''
Retrieve a member at the specified index from the leaderboard.
@param position [int] Position in leaderboard.
@param options [Hash] Options to be used when retrieving the member from the leaderboard.
@return a member from the leaderboard.
'''
return self.member_at_in(self.leaderboard_name, position, **options)
def member_at_in(self, leaderboard_name, position, **options):
'''
Retrieve a member at the specified index from the leaderboard.
@param leaderboard_name [String] Name of the leaderboard.
@param position [int] Position in named leaderboard.
@param options [Hash] Options to be used when retrieving the member from the named leaderboard.
@return a page of leaders from the named leaderboard.
'''
if position > 0 and position <= self.total_members_in(
leaderboard_name):
page_size = options.get('page_size', self.page_size)
current_page = math.ceil(float(position) / float(page_size))
offset = (position - 1) % page_size
leaders = self.leaders_in(leaderboard_name, current_page,
**options)
if leaders:
return leaders[offset]
def around_me(self, member, **options):
'''
Retrieve a page of leaders from the leaderboard around a given member.
@param member [String] Member name.
@param options [Hash] Options to be used when retrieving the page from the leaderboard.
@return a page of leaders from the leaderboard around a given member.
'''
return self.around_me_in(self.leaderboard_name, member, **options)
def around_me_in(self, leaderboard_name, member, **options):
'''
Retrieve a page of leaders from the named leaderboard around a given member.
@param leaderboard_name [String] Name of the leaderboard.
@param member [String] Member name.
@param options [Hash] Options to be used when retrieving the page from the named leaderboard.
@return a page of leaders from the named leaderboard around a given member. Returns an empty array for a non-existent member.
'''
reverse_rank_for_member = None
if self.order == self.DESC:
reverse_rank_for_member = self.redis_connection.zrevrank(
leaderboard_name, member)
else:
reverse_rank_for_member = self.redis_connection.zrank(
leaderboard_name, member)
if reverse_rank_for_member is None:
return []
page_size = options.get('page_size', self.page_size)
starting_offset = reverse_rank_for_member - (page_size // 2)
if starting_offset < 0:
starting_offset = 0
ending_offset = (starting_offset + page_size) - 1
raw_leader_data = self._range_method(self.redis_connection,
leaderboard_name,
int(starting_offset),
int(ending_offset),
withscores=False)
return self._parse_raw_members(leaderboard_name, raw_leader_data,
**options)
def ranked_in_list(self, members, **options):
'''
Retrieve a page of leaders from the leaderboard for a given list of members.
@param members [Array] Member names.
@param options [Hash] Options to be used when retrieving the page from the leaderboard.
@return a page of leaders from the leaderboard for a given list of members.
'''
return self.ranked_in_list_in(self.leaderboard_name, members,
**options)
def ranked_in_list_in(self, leaderboard_name, members, **options):
'''
Retrieve a page of leaders from the named leaderboard for a given list of members.
@param leaderboard_name [String] Name of the leaderboard.
@param members [Array] Member names.
@param options [Hash] Options to be used when retrieving the page from the named leaderboard.
@return a page of leaders from the named leaderboard for a given list of members.
'''
ranks_for_members = []
pipeline = self.redis_connection.pipeline()
for member in members:
if self.order == self.ASC:
pipeline.zrank(leaderboard_name, member)
else:
pipeline.zrevrank(leaderboard_name, member)
pipeline.zscore(leaderboard_name, member)
responses = pipeline.execute()
for index, member in enumerate(members):
data = {}
data[self.MEMBER_KEY] = member
rank = responses[index * 2]
if rank is not None:
rank += 1
else:
if not options.get('include_missing', True):
continue
data[self.RANK_KEY] = rank
score = responses[index * 2 + 1]
if score is not None:
score = float(score)
data[self.SCORE_KEY] = score
ranks_for_members.append(data)
if ('with_member_data'
in options) and (True == options['with_member_data']):
for index, member_data in enumerate(
self.members_data_for_in(leaderboard_name, members)):
try:
ranks_for_members[index][
self.MEMBER_DATA_KEY] = member_data
except:
pass
if 'sort_by' in options:
if self.RANK_KEY == options['sort_by']:
ranks_for_members = sorted(
ranks_for_members,
key=lambda member: member[self.RANK_KEY])
elif self.SCORE_KEY == options['sort_by']:
ranks_for_members = sorted(
ranks_for_members,
key=lambda member: member[self.SCORE_KEY])
return ranks_for_members
def merge_leaderboards(self, destination, keys, aggregate='SUM'):
'''
Merge leaderboards given by keys with this leaderboard into a named destination leaderboard.
@param destination [String] Destination leaderboard name.
@param keys [Array] Leaderboards to be merged with the current leaderboard.
@param options [Hash] Options for merging the leaderboards.
'''
keys.insert(0, self.leaderboard_name)
self.redis_connection.zunionstore(destination, keys, aggregate)
def intersect_leaderboards(self, destination, keys, aggregate='SUM'):
'''
Intersect leaderboards given by keys with this leaderboard into a named destination leaderboard.
@param destination [String] Destination leaderboard name.
@param keys [Array] Leaderboards to be merged with the current leaderboard.
@param options [Hash] Options for intersecting the leaderboards.
'''
keys.insert(0, self.leaderboard_name)
self.redis_connection.zinterstore(destination, keys, aggregate)
def _range_method(self, connection, *args, **kwargs):
if self.order == self.DESC:
return connection.zrevrange(*args, **kwargs)
else:
return connection.zrange(*args, **kwargs)
def _member_data_key(self, leaderboard_name):
'''
Key for retrieving optional member data.
@param leaderboard_name [String] Name of the leaderboard.
@return a key in the form of +leaderboard_name:member_data+
'''
if self.global_member_data is False:
return '%s:%s' % (leaderboard_name, self.member_data_namespace)
else:
return self.member_data_namespace
def _parse_raw_members(self,
leaderboard_name,
members,
members_only=False,
**options):
'''
Parse the raw leaders data as returned from a given leader board query. Do associative
lookups with the member to rank, score and potentially sort the results.
@param leaderboard_name [String] Name of the leaderboard.
@param members [List] A list of members as returned from a sorted set range query
@param members_only [bool] Set True to return the members as is, Default is False.
@param options [Hash] Options to be used when retrieving the page from the named leaderboard.
@return a list of members.
'''
if members_only:
return [{self.MEMBER_KEY: m} for m in members]
if members:
return self.ranked_in_list_in(leaderboard_name, members, **options)
else:
return []
"""
存放key map
:param key:
:param value:
:param overWrite:
:return:
"""
if overWrite:
self.__redis.hset(self.__name, key, value)
return key, value
else:
if self.exists_key(key):
return key, self.get(key)
def exists_key(self, key):
"""
是否存在key,如果是返回true,否则返回false
:param key:
:return:
"""
if self.__redis.hexists(self.__name, key) == 1:
return True
else:
return False
if __name__ == '__main__':
redis_nodes = [{'host': '10.1.13.111', 'port': 7000}]
r = StrictRedisCluster(startup_nodes=redis_nodes)
print r.hget()
class Session(object):
_uui = None
_max_retry = 5
_retry_num = 0
def __init__(self, **options):
self.options = {
'key_prefix': 'session',
'expire': 7200,
}
self.options.update(options)
self.logger = Logger('session').get()
self.connect()
def connect(self):
def func():
self.redis = StrictRedisCluster(
startup_nodes=self.options['startup_nodes'])
self._safe_way(func)
def get(self, uui):
self.logger.debug('Try to get session')
def _get():
return self._get_session(uui)
return self._safe_way(_get)
def set(self, uui):
self.logger.debug('Try to set new session: '
'uuid {name}'.format(name=uui))
self._uui = uui
def _set():
self._set_session(uui,
str(datetime.datetime.now().time()))
self._safe_way(_set)
def _safe_way(self, func):
try:
return func()
except:
if self._max_retry == self._retry_num:
self.logger.debug('Max try to reconnect. Closed')
tornado.ioloop.IOLoop.instance().stop()
else:
self._retry_num += 1
self.logger.debug('Fail to connect.'
'Try to reconnect after 6 sec')
time.sleep(6)
self.connect()
return func()
def delete(self):
self.logger.debug('Try to delete session')
self.redis.delete(self._prefixed(self._uui))
def _prefixed(self, sid):
return '%s:%s' % (self.options['key_prefix'], sid)
def _get_session(self, uui):
try_to_get = self._prefixed(uui)
self.logger.debug('Get session data: {data}'.format(data=try_to_get))
data = self.redis.hget(try_to_get, 'user')
session = pickle.loads(data) if data else None
self.logger.debug('Got: %s' % session)
return session
def _set_session(self, uui, session_data):
expiry = self.options['expire']
try_to_set = self._prefixed(uui)
self.logger.debug('Set session data: {data}'.format(data=try_to_set))
self.redis.hset(try_to_set,
'user',
pickle.dumps(session_data))
if expiry:
self.redis.expire(self._prefixed(uui), expiry)
