def redis_cluster(): redis_nodes = [{ "host": "192.168.1.161", "port": 7000 }, { "host": "192.168.1.161", "port": 7001 }, { "host": "192.168.1.161", "port": 7002 }, { "host": "192.168.1.147", "port": 7000 }, { "host": "192.168.1.147", "port": 7001 }, { "host": "192.168.1.147", "port": 7002 }] try: redisconn = StrictRedisCluster(startup_nodes=redis_nodes) except Exception as e: print("connect error!!") sys.exit(1) redisconn.hset("info1", "name", "yiruiduan") redisconn.hset("info1", "age", 30) redisconn.hset("info1", "id", 2) # for key in redisconn.keys(): # redisconn.delete(key) print(redisconn.keys())
class redis_cluster(): def __init__(self): redis_nodes = [{ 'host': '172.16.0.33', 'port': "9001" }, { 'host': '172.16.0.33', 'port': "9002" }, { 'host': '172.16.0.33', 'port': "9003" }, { 'host': '172.16.0.33', 'port': "9004" }, { 'host': '172.16.0.33', 'port': "9005" }, { 'host': '172.16.0.33', 'port': "9006" }] try: self.redisconn = StrictRedisCluster(startup_nodes=redis_nodes, decode_responses=True, password='******') except Exception as e: print("Connect Error! reason is %s" % (e)) sys.exit(1) def flushdb(self, params): for i in range(0, len(params)): result = self.redisconn.delete(*self.redisconn.keys( pattern=params[i])) print(result) return
def redis_cluster(): redis_nodes = [{ 'host': '172.20.4.91', 'port': 7000 }, { 'host': '172.20.4.91', 'port': 7001 }, { 'host': '172.20.4.92', 'port': 7002 }, { 'host': '172.20.4.92', 'port': 7003 }, { 'host': '172.20.4.95', 'port': 7004 }, { 'host': '172.20.4.95', 'port': 7005 }] try: redisconn = StrictRedisCluster(startup_nodes=redis_nodes) except Exception: print("Connect Error!") sys.exit(1) keys = redisconn.keys("rc.item.ft.*") for key in keys: redisconn.delete(key) print(len(keys))
def getRedisKeyValues(host, port, keyFilter): try: startup_nodes = [{"host": host, "port": port}] rc = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True) info = rc.info() for key in info: print "%s: %s" % (key, info[key]) result = [] # keysCount = len(rc.keys()) count = 0 for key in rc.keys(): if count >= 1024: break else: if key.find(keyFilter) != -1: count += 1 # 查询类型 # (key, value) result.append([key, getValue(rc, key)]) # pprint.pprint("type:" + rc.type(key) + "$key:" + key + "$value:" + getValue(rc, key)) # print rc.get(key) return result except Exception, e: return [["<font color='red'>Error Info</font>", "<font color='red'>%s</font>" % (str(e) + "<br/>请检查ip和端口是否正确")]]
def get_online_pids_from_redis(peer_prefix="00010048"): rc = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True) match_keys = rc.keys("PNIC_{0}*".format(peer_prefix)) peer_id_list = list() for i in match_keys: peer_id = i.replace("PNIC_", "") peer_id_list.append(str(peer_id)) return peer_id_list
def get_peer_ids_from_redis(redis_cluster_hosts): """ get all peer_id PNIC_ form redis-cluster :param redis_cluster_hosts: :return: """ rc = StrictRedisCluster(startup_nodes=redis_cluster_hosts, decode_responses=True) match_keys = rc.keys("PNIC_*") peer_id_list = list() for pnic_key in match_keys: peer_id = str(pnic_key)[5:] peer_id_list.append(peer_id) return peer_id_list
def get_online_pids_from_redis(peer_prefix="00010048"): """ 获得redis cluster里面所有的已登录节点peer id :param peer_prefix: prefix视情况修改 :return: """ rc = StrictRedisCluster(startup_nodes=redis_startup_nodes, decode_responses=True) match_keys = rc.keys("PNIC_{0}*".format(peer_prefix)) peer_id_list = list() for i in match_keys: peer_id = i.replace("PNIC_", "") peer_id_list.append(str(peer_id)) return peer_id_list
class RedisHandle(object): def __init__(self, host, port, db=0): nodes = [{"host": host, "port": port}] self.redis_client = StrictRedisCluster(startup_nodes=nodes, decode_responses=True) # self.redis_client = redis.StrictRedis(host=host, port=port, db=db) def get(self, redis_key): logging.info("start to get value of redis_key: %s" % (redis_key)) return self.redis_client.get(redis_key) def delete(self, redis_key): logging.info("start to match redis_key: %s" % (redis_key)) keys = self.redis_client.keys(redis_key + "*") for key in keys: logging.info("start to delete redis_key: %s" % (key)) self.redis_client.delete(key)
class RedisProcess(object): cluster_nodes = [] redisconn = None def __init__(self, cluster_nodes): self.cluster_nodes = cluster_nodes self.connect() def connect(self): try: self.redisconn = StrictRedisCluster( startup_nodes=self.cluster_nodes) except Exception as e: print("Connect Error!") sys.exit(1) def clear(self, pattern): print('Clean keys with pattern: ' + pattern) print('started') if self.redisconn: for key in self.redisconn.keys(pattern): self.redisconn.delete(key) print(str(key) + ' removed')
import redis from rediscluster import StrictRedisCluster serverip='101.200.214.57' serverip='192.168.1.146' startup_nodes=[{"host": serverip,"port": i} for i in xrange(7003, 7004)] print startup_nodes r = StrictRedisCluster(startup_nodes=startup_nodes) print r.keys() #print r.hgetall("41_56a466e0ce65")
class AmazonRedis: def __init__(self): try: self.rds = StrictRedisCluster(startup_nodes=StartupNodesServer, password='******', decode_responses=True) except: traceback.print_exc() def count_keys(self): # 查询当前库里有多少key return self.rds.dbsize() # key操作 def get_random_key(self): # 从当前数据库随机返回一个key, 没有返回None, 不删除key return self.rds.randomkey() def exists_key(self, key): return self.rds.exists(key) def delete_key(self, key): self.rds.delete(key) def get_all_keys(self): return self.rds.keys() def move_keys(self, key, num): # 如果移入的库中已有此key,则无法移入 self.rds.move(key, num) # String操作 def set_key_value(self, key, value): self.rds.set(key, value) def get_key_value(self, key): # 没有对应key返回None return self.rds.get(key) # Hash操作 def set_hash(self, key, mapping): # mapping为字典, 已存在key会覆盖mapping self.rds.hmset(key, mapping) def delete_hash_field(self, key, field): # 删除hash表中某个字段,无论字段是否存在 self.rds.hdel(key, field) def exists_hash_field(self, key, field): # 检查hash表中某个字段存在 return self.rds.hexists(key, field) def get_hash_field(self, key, field): # 获取hash表中指定字段的值, 没有返回None return self.rds.hget(key, field) def get_hash_all_field(self, key): # 获取hash表中指定key所有字段和值,以字典形式,没有key返回空字典 return self.rds.hgetall(key) def increase_hash_field(self, key, field, increment): # 为hash表key某个字段的整数型值增加increment self.rds.hincrby(key, field, increment) # List操作 def rpush_into_lst(self, key, value): # url从头至尾入列 self.rds.rpush(key, value) def lpush_into_lst(self, key, value): # url从尾至头入列 self.rds.lpush(key, value) def lpop_lst_item(self, key): # 从头取出列表第一个元素(元组形式),没有返回None return self.rds.lpop(key) def blpop_lst_item(self, key): # 从头取出列表第一个元素(元组形式),并设置超时,超时返回None return self.rds.blpop(key, timeout=1) def rpop_lst_item(self, key): # 从尾取出列表最后一个元素(元组形式),没有返回None return self.rds.rpop(key) def brpop_lst_item(self, key): # 从尾取出列表最后一个元素(元组形式),并设置超时,超时返回None return self.rds.brpop(key, timeout=1) # Set操作 def add_set(self, key, value): self.rds.sadd(key, value) def is_member(self, key, value): return self.rds.sismember(key, value) def pop_member(self, key): # 随机移除一个值并返回该值,没有返回None return self.rds.spop(key) def pop_members(self, key, num): # 随机取出num个值(非移除),列表形式返回这些值,没有返回空列表 return self.rds.srandmember(key, num) def remove_member(self, key, value): # 移除集合中指定元素 self.rds.srem(key, value) def get_all_members(self, key): # 返回集合中全部元素,不删除 return self.rds.smembers(key) def remove_into(self, key1, key2, value): # 把集合key1中value元素移入集合key2中 self.rds.smove(key1, key2, value) def count_members(self, key): # 计算集合中成员数量 return self.rds.scard(key)
# for key in info: # print "%s: %s" % (key, info[key]) # pprint.pprint(dir(rc)) print "-------------------" # pprint.pprint( rc.keys()) # zset rc.zadd("zset_", 9000, "zset.cc") # set rc.sadd("set_", "set.cc") # hash rc.hset("hash", "filed", "123456789") # string rc.set("foo", "xiaorui.cc") # list rc.lpush("list", "1") rc.lpush("list", "2") keysCount = len(rc.keys()) if keysCount < 1024: for key in rc.keys(): # 查询类型 pprint.pprint("type:" + rc.type(key) + "$key:" + key + "$value:" + getValue(rc, key)) # print rc.get(key) else: print "keysCount:" + str(keysCount) pass
class DB(object): def __init__(self, config): self.config = config cache_backend = self.config_get('cache_backend') or 'redis' if cache_backend == 'redis': self.__backend = cache_backend self.__host = self.config_get('cache_host') or '127.0.0.1' self.__port = self.config_get('cache_port') or 6379 self.__db_name = self.config_get('cache_db_name') or 0 self.db = redis.StrictRedis(host=self.__host, port=self.__port, db=self.__db_name) elif cache_backend == 'redis-cluster': self.__backend = cache_backend node1_host = self.config_get('node1_host') node1_port = self.config_get('node1_port') node2_host = self.config_get('node2_host') node2_port = self.config_get('node2_port') node3_host = self.config_get('node3_host') node3_port = self.config_get('node3_port') node4_host = self.config_get('node4_host') node4_port = self.config_get('node4_port') node5_host = self.config_get('node5_host') node5_port = self.config_get('node5_port') node6_host = self.config_get('node6_host') node6_port = self.config_get('node6_port') self.__host = (node1_host, node2_host, node3_host, node4_host, node5_host, node6_host) self.__port = (node1_port, node2_port, node3_port, node4_port, node5_port, node6_port) self.__db_name = 'cluster' cluster_nodes = [{ 'host': node1_host, 'port': node1_port }, { 'host': node2_host, 'port': node2_port }, { 'host': node3_host, 'port': node3_port }, { 'host': node4_host, 'port': node4_port }, { 'host': node5_host, 'port': node5_port }, { 'host': node6_host, 'port': node6_port }] self.db = StrictRedisCluster(startup_nodes=cluster_nodes, decode_responses=True) self.set_value = self.db.set self.has_value = self.db.exists self.remove_value = self.db.delete def config_get(self, name): if isinstance(self.config, ConfigParser): return self.config.get('app:api', name) else: return self.config.get('app:api').get(name) def get(self, key): return self.db.get(key) def keys(self, prefix): keys = self.db.keys(prefix) return keys def put(self, key, value, ex=90000): self.set_value(key, value, ex) def remove(self, key): self.remove_value(key) def has(self, key): return self.has_value(key) def scan_iter(self, prefix=None): return [key for key in self.db.scan_iter(prefix)] def remove_pattern(self, prefix): for key in self.db.scan_iter(prefix): self.remove(key) def flushall(self): self.db.flushall() @property def backend(self): return self.__backend @property def host(self): return self.__host @property def port(self): return self.__port @property def db_name(self): return self.__db_name
class RedisQueue(object): # def __init__(self, redis_type=None,**args): # # if redis_type == "cluster": # try: # self.rc = StrictRedisCluster(**args) # except Exception as e: # print("Connect Error!") # else: # try: # self.pool = redis.ConnectionPool(**args) # self.rc = redis.Redis(connection_pool=self.pool) # except Exception as e: # print("Connect Error!") def __init__(self, res): redis_type = res[1] if redis_type == "cluster": try: con = res[0] self.rc = StrictRedisCluster(**con) except Exception as e: print(e) print("Connect Error!") else: try: con = res[0] self.pool = redis.ConnectionPool(**con) self.rc = redis.Redis(connection_pool=self.pool) except Exception as e: print(e) print("Connect Error!") def get_all(self, key, block=True, timeout=None): for i in self.rc.keys(): if key in str(i): type = self.rc.type(i) if type == 'string': vals = self.rc.get(i) elif type == 'list': vals = self.rc.lrange(i, 0, -1) elif type == 'set': vals = self.rc.smembers(i) elif type == 'zset': vals = self.rc.zrange(i, 0, -1) elif type == "hash": vals = self.rc.hgetall(i) else: print(type, i) return list(vals[0]) def keys(self): keys = self.rc.keys() return keys def iskey(self, key): if self.rc.exists(key): return 1 else: return 0 def get(self, key): res = self.rc.get(key) nres = json.loads(res) return nres def put(self, key, value): new_value = json.dumps(value) res = self.rc.set(key, new_value) def delall(self, key): self.rc.delete(key) # print(getredis('ak_sit')) #RedisQueue(getredis('ak_cluster'))
class RedisDB: def __init__(self, ip_ports=None, db=None, user_pass=None, url=None, decode_responses=True, service_name=None, max_connections=32, **kwargs): """ redis的封装 Args: ip_ports: ip:port 多个可写为列表或者逗号隔开 如 ip1:port1,ip2:port2 或 ["ip1:port1", "ip2:port2"] db: user_pass: url: decode_responses: service_name: 适用于redis哨兵模式 """ # 可能会改setting中的值,所以此处不能直接赋值为默认值,需要后加载赋值 if ip_ports is None: ip_ports = setting.REDISDB_IP_PORTS if db is None: db = setting.REDISDB_DB if user_pass is None: user_pass = setting.REDISDB_USER_PASS if service_name is None: service_name = setting.REDISDB_SERVICE_NAME self._is_redis_cluster = False try: if not url: ip_ports = (ip_ports if isinstance(ip_ports, list) else ip_ports.split(",")) if len(ip_ports) > 1: startup_nodes = [] for ip_port in ip_ports: ip, port = ip_port.split(":") startup_nodes.append({"host": ip, "port": port}) if service_name: log.debug("使用redis哨兵模式") hosts = [(node["host"], node["port"]) for node in startup_nodes] sentinel = Sentinel(hosts, socket_timeout=3, **kwargs) self._redis = sentinel.master_for( service_name, password=user_pass, db=db, redis_class=redis.StrictRedis, decode_responses=decode_responses, max_connections=max_connections, **kwargs) else: log.debug("使用redis集群模式") self._redis = StrictRedisCluster( startup_nodes=startup_nodes, decode_responses=decode_responses, password=user_pass, max_connections=max_connections, **kwargs) self._is_redis_cluster = True else: ip, port = ip_ports[0].split(":") self._redis = redis.StrictRedis( host=ip, port=port, db=db, password=user_pass, decode_responses=decode_responses, max_connections=max_connections, **kwargs) else: self._redis = redis.StrictRedis.from_url( url, decode_responses=decode_responses) except Exception as e: raise else: if not url: log.debug("连接到redis数据库 %s db%s" % (ip_ports, db)) else: log.debug("连接到redis数据库 %s" % (url)) self._ip_ports = ip_ports self._db = db self._user_pass = user_pass self._url = url def __repr__(self): if self._url: return "<Redisdb url:{}>".format(self._url) return "<Redisdb ip_ports: {} db:{} user_pass:{}>".format( self._ip_ports, self._db, self._user_pass) @classmethod def from_url(cls, url): return cls(url=url) def sadd(self, table, values): """ @summary: 使用无序set集合存储数据, 去重 --------- @param table: @param values: 值; 支持list 或 单个值 --------- @result: 若库中存在 返回0,否则入库,返回1。 批量添加返回None """ if isinstance(values, list): pipe = self._redis.pipeline( transaction=True ) # redis-py默认在执行每次请求都会创建(连接池申请连接)和断开(归还连接池)一次连接操作,如果想要在一次请求中指定多个命令,则可以使用pipline实现一次请求指定多个命令,并且默认情况下一次pipline 是原子性操作。 if not self._is_redis_cluster: pipe.multi() for value in values: pipe.sadd(table, value) pipe.execute() else: return self._redis.sadd(table, values) def sget(self, table, count=1, is_pop=True): """ 返回 list 如 ['1'] 或 [] @param table: @param count: @param is_pop: @return: """ datas = [] if is_pop: count = count if count <= self.sget_count( table) else self.sget_count(table) if count: if count > 1: pipe = self._redis.pipeline( transaction=True ) # redis-py默认在执行每次请求都会创建(连接池申请连接)和断开(归还连接池)一次连接操作,如果想要在一次请求中指定多个命令,则可以使用pipline实现一次请求指定多个命令,并且默认情况下一次pipline 是原子性操作。 if not self._is_redis_cluster: pipe.multi() while count: pipe.spop(table) count -= 1 datas = pipe.execute() else: datas.append(self._redis.spop(table)) else: datas = self._redis.srandmember(table, count) return datas def srem(self, table, values): """ @summary: 移除集合中的指定元素 --------- @param table: @param values: 一个或者列表 --------- @result: """ if isinstance(values, list): pipe = self._redis.pipeline( transaction=True ) # redis-py默认在执行每次请求都会创建(连接池申请连接)和断开(归还连接池)一次连接操作,如果想要在一次请求中指定多个命令,则可以使用pipline实现一次请求指定多个命令,并且默认情况下一次pipline 是原子性操作。 if not self._is_redis_cluster: pipe.multi() for value in values: pipe.srem(table, value) pipe.execute() else: self._redis.srem(table, values) def sget_count(self, table): return self._redis.scard(table) def sdelete(self, table): """ @summary: 删除set集合的大键(数据量大的表) 删除大set键,使用sscan命令,每次扫描集合中500个元素,再用srem命令每次删除一个键 若直接用delete命令,会导致Redis阻塞,出现故障切换和应用程序崩溃的故障。 --------- @param table: --------- @result: """ # 当 SCAN 命令的游标参数被设置为 0 时, 服务器将开始一次新的迭代, 而当服务器向用户返回值为 0 的游标时, 表示迭代已结束 cursor = "0" while cursor != 0: cursor, data = self._redis.sscan(table, cursor=cursor, count=500) for item in data: # pipe.srem(table, item) self._redis.srem(table, item) # pipe.execute() def sismember(self, table, key): "Return a boolean indicating if ``value`` is a member of set ``name``" return self._redis.sismember(table, key) def zadd(self, table, values, prioritys=0): """ @summary: 使用有序set集合存储数据, 去重(值存在更新) --------- @param table: @param values: 值; 支持list 或 单个值 @param prioritys: 优先级; double类型,支持list 或 单个值。 根据此字段的值来排序, 值越小越优先。 可不传值,默认value的优先级为0 --------- @result:若库中存在 返回0,否则入库,返回1。 批量添加返回 [0, 1 ...] """ if isinstance(values, list): if not isinstance(prioritys, list): prioritys = [prioritys] * len(values) else: assert len(values) == len(prioritys), "values值要与prioritys值一一对应" pipe = self._redis.pipeline(transaction=True) if not self._is_redis_cluster: pipe.multi() for value, priority in zip(values, prioritys): pipe.zadd(table, priority, value) return pipe.execute() else: return self._redis.zadd(table, prioritys, values) def zget(self, table, count=1, is_pop=True): """ @summary: 从有序set集合中获取数据 优先返回分数小的(优先级高的) --------- @param table: @param count: 数量 -1 返回全部数据 @param is_pop:获取数据后,是否在原set集合中删除,默认是 --------- @result: 列表 """ start_pos = 0 # 包含 end_pos = count - 1 if count > 0 else count pipe = self._redis.pipeline( transaction=True ) # redis-py默认在执行每次请求都会创建(连接池申请连接)和断开(归还连接池)一次连接操作,如果想要在一次请求中指定多个命令,则可以使用pipline实现一次请求指定多个命令,并且默认情况下一次pipline 是原子性操作。 if not self._is_redis_cluster: pipe.multi( ) # 标记事务的开始 参考 http://www.runoob.com/redis/redis-transactions.html pipe.zrange(table, start_pos, end_pos) # 取值 if is_pop: pipe.zremrangebyrank(table, start_pos, end_pos) # 删除 results, *count = pipe.execute() return results def zremrangebyscore(self, table, priority_min, priority_max): """ 根据分数移除成员 闭区间 @param table: @param priority_min: @param priority_max: @return: 被移除的成员个数 """ return self._redis.zremrangebyscore(table, priority_min, priority_max) def zrangebyscore(self, table, priority_min, priority_max, count=None, is_pop=True): """ @summary: 返回指定分数区间的数据 闭区间 --------- @param table: @param priority_min: 优先级越小越优先 @param priority_max: @param count: 获取的数量,为空则表示分数区间内的全部数据 @param is_pop: 是否删除 --------- @result: """ # 使用lua脚本, 保证操作的原子性 lua = """ local key = KEYS[1] local min_score = ARGV[2] local max_score = ARGV[3] local is_pop = ARGV[4] local count = ARGV[5] -- 取值 local datas = nil if count then datas = redis.call('zrangebyscore', key, min_score, max_score, 'limit', 0, count) else datas = redis.call('zrangebyscore', key, min_score, max_score) end -- 删除redis中刚取到的值 if (is_pop) then for i=1, #datas do redis.call('zrem', key, datas[i]) end end return datas """ cmd = self._redis.register_script(lua) if count: res = cmd(keys=[table], args=[table, priority_min, priority_max, is_pop, count]) else: res = cmd(keys=[table], args=[table, priority_min, priority_max, is_pop]) return res def zrangebyscore_increase_score(self, table, priority_min, priority_max, increase_score, count=None): """ @summary: 返回指定分数区间的数据 闭区间, 同时修改分数 --------- @param table: @param priority_min: 最小分数 @param priority_max: 最大分数 @param increase_score: 分数值增量 正数则在原有的分数上叠加,负数则相减 @param count: 获取的数量,为空则表示分数区间内的全部数据 --------- @result: """ # 使用lua脚本, 保证操作的原子性 lua = """ local key = KEYS[1] local min_score = ARGV[1] local max_score = ARGV[2] local increase_score = ARGV[3] local count = ARGV[4] -- 取值 local datas = nil if count then datas = redis.call('zrangebyscore', key, min_score, max_score, 'limit', 0, count) else datas = redis.call('zrangebyscore', key, min_score, max_score) end --修改优先级 for i=1, #datas do redis.call('zincrby', key, increase_score, datas[i]) end return datas """ cmd = self._redis.register_script(lua) if count: res = cmd(keys=[table], args=[priority_min, priority_max, increase_score, count]) else: res = cmd(keys=[table], args=[priority_min, priority_max, increase_score]) return res def zrangebyscore_set_score(self, table, priority_min, priority_max, score, count=None): """ @summary: 返回指定分数区间的数据 闭区间, 同时修改分数 --------- @param table: @param priority_min: 最小分数 @param priority_max: 最大分数 @param score: 分数值 @param count: 获取的数量,为空则表示分数区间内的全部数据 --------- @result: """ # 使用lua脚本, 保证操作的原子性 lua = """ local key = KEYS[1] local min_score = ARGV[1] local max_score = ARGV[2] local set_score = ARGV[3] local count = ARGV[4] -- 取值 local datas = nil if count then datas = redis.call('zrangebyscore', key, min_score, max_score, 'withscores','limit', 0, count) else datas = redis.call('zrangebyscore', key, min_score, max_score, 'withscores') end local real_datas = {} -- 数据 --修改优先级 for i=1, #datas, 2 do local data = datas[i] local score = datas[i+1] table.insert(real_datas, data) -- 添加数据 redis.call('zincrby', key, set_score - score, datas[i]) end return real_datas """ cmd = self._redis.register_script(lua) if count: res = cmd(keys=[table], args=[priority_min, priority_max, score, count]) else: res = cmd(keys=[table], args=[priority_min, priority_max, score]) return res def zget_count(self, table, priority_min=None, priority_max=None): """ @summary: 获取表数据的数量 --------- @param table: @param priority_min:优先级范围 最小值(包含) @param priority_max:优先级范围 最大值(包含) --------- @result: """ if priority_min != None and priority_max != None: return self._redis.zcount(table, priority_min, priority_max) else: return self._redis.zcard(table) def zrem(self, table, values): """ @summary: 移除集合中的指定元素 --------- @param table: @param values: 一个或者列表 --------- @result: """ if isinstance(values, list): pipe = self._redis.pipeline( transaction=True ) # redis-py默认在执行每次请求都会创建(连接池申请连接)和断开(归还连接池)一次连接操作,如果想要在一次请求中指定多个命令,则可以使用pipline实现一次请求指定多个命令,并且默认情况下一次pipline 是原子性操作。 if not self._is_redis_cluster: pipe.multi() for value in values: pipe.zrem(table, value) pipe.execute() else: self._redis.zrem(table, values) def zexists(self, table, values): """ 利用zscore判断某元素是否存在 @param values: @return: """ is_exists = [] if isinstance(values, list): pipe = self._redis.pipeline( transaction=True ) # redis-py默认在执行每次请求都会创建(连接池申请连接)和断开(归还连接池)一次连接操作,如果想要在一次请求中指定多个命令,则可以使用pipline实现一次请求指定多个命令,并且默认情况下一次pipline 是原子性操作。 pipe.multi() for value in values: pipe.zscore(table, value) is_exists_temp = pipe.execute() for is_exist in is_exists_temp: if is_exist != None: is_exists.append(1) else: is_exists.append(0) else: is_exists = self._redis.zscore(table, values) is_exists = 1 if is_exists != None else 0 return is_exists def lpush(self, table, values): if isinstance(values, list): pipe = self._redis.pipeline( transaction=True ) # redis-py默认在执行每次请求都会创建(连接池申请连接)和断开(归还连接池)一次连接操作,如果想要在一次请求中指定多个命令,则可以使用pipline实现一次请求指定多个命令,并且默认情况下一次pipline 是原子性操作。 if not self._is_redis_cluster: pipe.multi() for value in values: pipe.rpush(table, value) pipe.execute() else: return self._redis.rpush(table, values) def lpop(self, table, count=1): """ @summary: --------- @param table: @param count: --------- @result: count>1时返回列表 """ datas = None count = count if count <= self.lget_count(table) else self.lget_count( table) if count: if count > 1: pipe = self._redis.pipeline( transaction=True ) # redis-py默认在执行每次请求都会创建(连接池申请连接)和断开(归还连接池)一次连接操作,如果想要在一次请求中指定多个命令,则可以使用pipline实现一次请求指定多个命令,并且默认情况下一次pipline 是原子性操作。 if not self._is_redis_cluster: pipe.multi() while count: pipe.lpop(table) count -= 1 datas = pipe.execute() else: datas = self._redis.lpop(table) return datas def rpoplpush(self, from_table, to_table=None): """ 将列表 from_table 中的最后一个元素(尾元素)弹出,并返回给客户端。 将 from_table 弹出的元素插入到列表 to_table ,作为 to_table 列表的的头元素。 如果 from_table 和 to_table 相同,则列表中的表尾元素被移动到表头,并返回该元素,可以把这种特殊情况视作列表的旋转(rotation)操作 @param from_table: @param to_table: @return: """ if not to_table: to_table = from_table return self._redis.rpoplpush(from_table, to_table) def lget_count(self, table): return self._redis.llen(table) def lrem(self, table, value, num=0): return self._redis.lrem(table, value, num) def hset(self, table, key, value): """ @summary: 如果 key 不存在,一个新的哈希表被创建并进行 HSET 操作。 如果域 field 已经存在于哈希表中,旧值将被覆盖 --------- @param table: @param key: @param value: --------- @result: 1 新插入; 0 覆盖 """ return self._redis.hset(table, key, value) def hset_batch(self, table, datas): """ 批量插入 Args: datas: [[key, value]] Returns: """ pipe = self._redis.pipeline(transaction=True) if not self._is_redis_cluster: pipe.multi() for key, value in datas: pipe.hset(table, key, value) return pipe.execute() def hincrby(self, table, key, increment): return self._redis.hincrby(table, key, increment) def hget(self, table, key, is_pop=False): if not is_pop: return self._redis.hget(table, key) else: lua = """ local key = KEYS[1] local field = ARGV[1] -- 取值 local datas = redis.call('hget', key, field) -- 删除值 redis.call('hdel', key, field) return datas """ cmd = self._redis.register_script(lua) res = cmd(keys=[table], args=[key]) return res def hgetall(self, table): return self._redis.hgetall(table) def hexists(self, table, key): return self._redis.hexists(table, key) def hdel(self, table, *keys): """ @summary: 删除对应的key 可传多个 --------- @param table: @param *keys: --------- @result: """ self._redis.hdel(table, *keys) def hget_count(self, table): return self._redis.hlen(table) def setbit(self, table, offsets, values): """ 设置字符串数组某一位的值, 返回之前的值 @param table: @param offsets: 支持列表或单个值 @param values: 支持列表或单个值 @return: list / 单个值 """ if isinstance(offsets, list): if not isinstance(values, list): values = [values] * len(offsets) else: assert len(offsets) == len(values), "offsets值要与values值一一对应" pipe = self._redis.pipeline( transaction=True ) # redis-py默认在执行每次请求都会创建(连接池申请连接)和断开(归还连接池)一次连接操作,如果想要在一次请求中指定多个命令,则可以使用pipline实现一次请求指定多个命令,并且默认情况下一次pipline 是原子性操作。 pipe.multi() for offset, value in zip(offsets, values): pipe.setbit(table, offset, value) return pipe.execute() else: return self._redis.setbit(table, offsets, values) def getbit(self, table, offsets): """ 取字符串数组某一位的值 @param table: @param offsets: 支持列表 @return: list / 单个值 """ if isinstance(offsets, list): pipe = self._redis.pipeline( transaction=True ) # redis-py默认在执行每次请求都会创建(连接池申请连接)和断开(归还连接池)一次连接操作,如果想要在一次请求中指定多个命令,则可以使用pipline实现一次请求指定多个命令,并且默认情况下一次pipline 是原子性操作。 pipe.multi() for offset in offsets: pipe.getbit(table, offset) return pipe.execute() else: return self._redis.getbit(table, offsets) def bitcount(self, table): return self._redis.bitcount(table) def strset(self, table, value, **kwargs): return self._redis.set(table, value, **kwargs) def str_incrby(self, table, value): return self._redis.incrby(table, value) def strget(self, table): return self._redis.get(table) def strlen(self, table): return self._redis.strlen(table) def getkeys(self, regex): return self._redis.keys(regex) def exists_key(self, key): return self._redis.exists(key) def set_expire(self, key, seconds): """ @summary: 设置过期时间 --------- @param key: @param seconds: 秒 --------- @result: """ self._redis.expire(key, seconds) def clear(self, table): try: self._redis.delete(table) except Exception as e: log.error(e) def get_redis_obj(self): return self._redis
#!/usr/bin/env python # _*_coding:utf-8_*_ # Author: create by yang.hong # Time: 2018-08-17 11:06 """ 集群模式下,匹配方式删除多个key """ from rediscluster import StrictRedisCluster redis_cluster = [{'host': '172.28.246.12', 'port': '28000'}, {'host': '172.28.246.12', 'port': '28001'}, {'host': '172.28.246.12', 'port': '28002'}, {'host': '172.28.246.12', 'port': '28003'}, {'host': '172.28.246.12', 'port': '28004'}, {'host': '172.28.246.12', 'port': '28005'}, ] r = StrictRedisCluster(startup_nodes=redis_cluster, decode_responses=True) print r.keys(pattern="*mc_payChannel_router*") r.delete(*r.keys(pattern="*mc_payChannel_router*"))
def read_seed_pool_redis(): peer_ids_in_redis = [] seeds_add_to_mysql = [] file_ids = [] redis_nodes = [ {'host': SEED_POOL_REDIS_CLUSTER_HOST_1, 'port': SEED_POOL_REDIS_CLUSTER_PORT_1}, {'host': SEED_POOL_REDIS_CLUSTER_HOST_2, 'port': SEED_POOL_REDIS_CLUSTER_PORT_2}, {'host': SEED_POOL_REDIS_CLUSTER_HOST_3, 'port': SEED_POOL_REDIS_CLUSTER_PORT_3} ] peer_ids_in_mysql = obtain_peer_id_from_table(Seed_pool) redis_connect = StrictRedisCluster(startup_nodes=redis_nodes) # get keys contain TSGO for key in redis_connect.keys(): # key: {TSGO if key.find('TSGO') == 1: file_ids.append(key) if file_ids: # get file id, currently we only have one channel, so we only handle # file_ids[0] for file_id in file_ids: file_id_write_to_mysql = file_id.split("_")[2] # get value by key selected_seeds = redis_connect.smembers(file_id) log.logger.info('redis seeds num:{0}'.format(len(selected_seeds))) for seed in selected_seeds: # eval it from string to dictionary seed_info = eval(seed) peer_id = seed_info['peer_id'] peer_ids_in_redis.append(peer_id) if peer_id not in peer_ids_in_mysql: sdk_version = seed_info['version'] private_ip = seed_info['privateIP'] private_port = seed_info['privatePort'] public_ip = seed_info['publicIP'] public_port = seed_info['publicPort'] nat_type = seed_info['natType'] stun_ip = seed_info['stunIP'] isp_id = seed_info['isp_id'] province_id = seed_info['province_id'] ppc = seed_info['ppc'] cppc = seed_info['cppc'] seed_redis = (Seed_pool(peer_id=peer_id, sdk_version=sdk_version, public_ip=public_ip, public_port=public_port, private_ip=private_ip, private_port=private_port, nat_type=nat_type, stun_ip=stun_ip, isp_id=isp_id, province_id=province_id, ppc=ppc, cppc=cppc, file_id=file_id_write_to_mysql)) orm.session.add(seed_redis) seeds_add_to_mysql.append(seed_redis) orm.session.commit() orm.session.close() log.logger.info('add {0} seeds to mysql'.format(len(seeds_add_to_mysql))) # del seeds expired(in mysql but not in redis) peer_ids_in_mysql_after_add = obtain_peer_id_from_table(Seed_pool) for peer in peer_ids_in_mysql_after_add: if peer not in peer_ids_in_redis: orm.session.query(Seed_pool).filter_by( peer_id=peer).delete() orm.session.commit() orm.session.close() else: # if seeds list in redis is empty, clean MySQL log.logger.info('no seeds in redis so we clean table seed_pool ') orm.session.query(Seed_pool).filter().delete() orm.session.commit() orm.session.close()