Esempio n. 1
0
def redis_cluster():
    redis_nodes = [{
        "host": "192.168.1.161",
        "port": 7000
    }, {
        "host": "192.168.1.161",
        "port": 7001
    }, {
        "host": "192.168.1.161",
        "port": 7002
    }, {
        "host": "192.168.1.147",
        "port": 7000
    }, {
        "host": "192.168.1.147",
        "port": 7001
    }, {
        "host": "192.168.1.147",
        "port": 7002
    }]
    try:
        redisconn = StrictRedisCluster(startup_nodes=redis_nodes)
    except Exception as e:
        print("connect error!!")
        sys.exit(1)
    redisconn.hset("info1", "name", "yiruiduan")
    redisconn.hset("info1", "age", 30)
    redisconn.hset("info1", "id", 2)

    # for key in redisconn.keys():
    #     redisconn.delete(key)
    print(redisconn.keys())
Esempio n. 2
0
class redis_cluster():
    def __init__(self):
        redis_nodes = [{
            'host': '172.16.0.33',
            'port': "9001"
        }, {
            'host': '172.16.0.33',
            'port': "9002"
        }, {
            'host': '172.16.0.33',
            'port': "9003"
        }, {
            'host': '172.16.0.33',
            'port': "9004"
        }, {
            'host': '172.16.0.33',
            'port': "9005"
        }, {
            'host': '172.16.0.33',
            'port': "9006"
        }]
        try:
            self.redisconn = StrictRedisCluster(startup_nodes=redis_nodes,
                                                decode_responses=True,
                                                password='******')
        except Exception as e:
            print("Connect Error!  reason is %s" % (e))
            sys.exit(1)

    def flushdb(self, params):
        for i in range(0, len(params)):
            result = self.redisconn.delete(*self.redisconn.keys(
                pattern=params[i]))
            print(result)
        return
Esempio n. 3
0
def redis_cluster():
    redis_nodes = [{
        'host': '172.20.4.91',
        'port': 7000
    }, {
        'host': '172.20.4.91',
        'port': 7001
    }, {
        'host': '172.20.4.92',
        'port': 7002
    }, {
        'host': '172.20.4.92',
        'port': 7003
    }, {
        'host': '172.20.4.95',
        'port': 7004
    }, {
        'host': '172.20.4.95',
        'port': 7005
    }]
    try:
        redisconn = StrictRedisCluster(startup_nodes=redis_nodes)
    except Exception:
        print("Connect Error!")
        sys.exit(1)

    keys = redisconn.keys("rc.item.ft.*")
    for key in keys:
        redisconn.delete(key)
    print(len(keys))
def getRedisKeyValues(host, port, keyFilter):

    try:
        startup_nodes = [{"host": host, "port": port}]
        rc = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True)
        info = rc.info()
        for key in info:
            print "%s: %s" % (key, info[key])

        result = []
        # keysCount = len(rc.keys())

        count = 0
        for key in rc.keys():
            if count >= 1024:
                break
            else:
                if key.find(keyFilter) != -1:
                    count += 1
                    # 查询类型
                    # (key, value)
                    result.append([key, getValue(rc, key)])
            #        pprint.pprint("type:" + rc.type(key) + "$key:" + key + "$value:" + getValue(rc, key))
            #        print rc.get(key)
        return result
    except Exception, e:
        return [["<font color='red'>Error Info</font>", "<font color='red'>%s</font>" % (str(e) + "<br/>请检查ip和端口是否正确")]]
Esempio n. 5
0
def get_online_pids_from_redis(peer_prefix="00010048"):
    rc = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True)
    match_keys = rc.keys("PNIC_{0}*".format(peer_prefix))
    peer_id_list = list()
    for i in match_keys:
        peer_id = i.replace("PNIC_", "")
        peer_id_list.append(str(peer_id))
    return peer_id_list
Esempio n. 6
0
def get_peer_ids_from_redis(redis_cluster_hosts):
    """
    get all peer_id PNIC_ form redis-cluster
    :param redis_cluster_hosts:
    :return:
    """

    rc = StrictRedisCluster(startup_nodes=redis_cluster_hosts, decode_responses=True)
    match_keys = rc.keys("PNIC_*")
    peer_id_list = list()
    for pnic_key in match_keys:
        peer_id = str(pnic_key)[5:]
        peer_id_list.append(peer_id)
    return peer_id_list
Esempio n. 7
0
def get_online_pids_from_redis(peer_prefix="00010048"):
    """
    获得redis cluster里面所有的已登录节点peer id
    :param peer_prefix: prefix视情况修改
    :return:
    """
    rc = StrictRedisCluster(startup_nodes=redis_startup_nodes,
                            decode_responses=True)
    match_keys = rc.keys("PNIC_{0}*".format(peer_prefix))
    peer_id_list = list()
    for i in match_keys:
        peer_id = i.replace("PNIC_", "")
        peer_id_list.append(str(peer_id))
    return peer_id_list
Esempio n. 8
0
class RedisHandle(object):
    def __init__(self, host, port, db=0):
        nodes = [{"host": host, "port": port}]
        self.redis_client = StrictRedisCluster(startup_nodes=nodes,
                                               decode_responses=True)
        # self.redis_client = redis.StrictRedis(host=host, port=port, db=db)

    def get(self, redis_key):
        logging.info("start to get value of redis_key: %s" % (redis_key))
        return self.redis_client.get(redis_key)

    def delete(self, redis_key):
        logging.info("start to match redis_key: %s" % (redis_key))
        keys = self.redis_client.keys(redis_key + "*")
        for key in keys:
            logging.info("start to delete redis_key: %s" % (key))
            self.redis_client.delete(key)
Esempio n. 9
0
class RedisProcess(object):
    cluster_nodes = []
    redisconn = None

    def __init__(self, cluster_nodes):
        self.cluster_nodes = cluster_nodes
        self.connect()

    def connect(self):
        try:
            self.redisconn = StrictRedisCluster(
                startup_nodes=self.cluster_nodes)
        except Exception as e:
            print("Connect Error!")
            sys.exit(1)

    def clear(self, pattern):
        print('Clean keys with pattern: ' + pattern)
        print('started')
        if self.redisconn:
            for key in self.redisconn.keys(pattern):
                self.redisconn.delete(key)
                print(str(key) + ' removed')
Esempio n. 10
0
File: test.py Progetto: 9wfox/robot
import redis
from rediscluster import StrictRedisCluster
serverip='101.200.214.57'
serverip='192.168.1.146'

startup_nodes=[{"host": serverip,"port": i} for i in xrange(7003, 7004)]
print startup_nodes

r = StrictRedisCluster(startup_nodes=startup_nodes)

print r.keys()
#print r.hgetall("41_56a466e0ce65")

Esempio n. 11
0
class AmazonRedis:
    def __init__(self):
        try:
            self.rds = StrictRedisCluster(startup_nodes=StartupNodesServer,
                                          password='******',
                                          decode_responses=True)
        except:
            traceback.print_exc()

    def count_keys(self):  # 查询当前库里有多少key
        return self.rds.dbsize()

    # key操作
    def get_random_key(self):  # 从当前数据库随机返回一个key, 没有返回None, 不删除key
        return self.rds.randomkey()

    def exists_key(self, key):
        return self.rds.exists(key)

    def delete_key(self, key):
        self.rds.delete(key)

    def get_all_keys(self):
        return self.rds.keys()

    def move_keys(self, key, num):  # 如果移入的库中已有此key,则无法移入
        self.rds.move(key, num)

    # String操作
    def set_key_value(self, key, value):
        self.rds.set(key, value)

    def get_key_value(self, key):  # 没有对应key返回None
        return self.rds.get(key)

    # Hash操作
    def set_hash(self, key, mapping):  # mapping为字典, 已存在key会覆盖mapping
        self.rds.hmset(key, mapping)

    def delete_hash_field(self, key, field):  # 删除hash表中某个字段,无论字段是否存在
        self.rds.hdel(key, field)

    def exists_hash_field(self, key, field):  # 检查hash表中某个字段存在
        return self.rds.hexists(key, field)

    def get_hash_field(self, key, field):  # 获取hash表中指定字段的值, 没有返回None
        return self.rds.hget(key, field)

    def get_hash_all_field(self, key):  # 获取hash表中指定key所有字段和值,以字典形式,没有key返回空字典
        return self.rds.hgetall(key)

    def increase_hash_field(self, key, field,
                            increment):  # 为hash表key某个字段的整数型值增加increment
        self.rds.hincrby(key, field, increment)

    # List操作
    def rpush_into_lst(self, key, value):  # url从头至尾入列
        self.rds.rpush(key, value)

    def lpush_into_lst(self, key, value):  # url从尾至头入列
        self.rds.lpush(key, value)

    def lpop_lst_item(self, key):  # 从头取出列表第一个元素(元组形式),没有返回None
        return self.rds.lpop(key)

    def blpop_lst_item(self, key):  # 从头取出列表第一个元素(元组形式),并设置超时,超时返回None
        return self.rds.blpop(key, timeout=1)

    def rpop_lst_item(self, key):  # 从尾取出列表最后一个元素(元组形式),没有返回None
        return self.rds.rpop(key)

    def brpop_lst_item(self, key):  # 从尾取出列表最后一个元素(元组形式),并设置超时,超时返回None
        return self.rds.brpop(key, timeout=1)

    # Set操作
    def add_set(self, key, value):
        self.rds.sadd(key, value)

    def is_member(self, key, value):
        return self.rds.sismember(key, value)

    def pop_member(self, key):  # 随机移除一个值并返回该值,没有返回None
        return self.rds.spop(key)

    def pop_members(self, key, num):  # 随机取出num个值(非移除),列表形式返回这些值,没有返回空列表
        return self.rds.srandmember(key, num)

    def remove_member(self, key, value):  # 移除集合中指定元素
        self.rds.srem(key, value)

    def get_all_members(self, key):  # 返回集合中全部元素,不删除
        return self.rds.smembers(key)

    def remove_into(self, key1, key2, value):  # 把集合key1中value元素移入集合key2中
        self.rds.smove(key1, key2, value)

    def count_members(self, key):  # 计算集合中成员数量
        return self.rds.scard(key)
Esempio n. 12
0
    #    for key in info:
    #      print "%s: %s" % (key, info[key])

    #    pprint.pprint(dir(rc))

    print "-------------------"

    #    pprint.pprint( rc.keys())
    # zset
    rc.zadd("zset_", 9000, "zset.cc")
    # set
    rc.sadd("set_", "set.cc")
    # hash
    rc.hset("hash", "filed", "123456789")
    # string
    rc.set("foo", "xiaorui.cc")
    # list
    rc.lpush("list", "1")
    rc.lpush("list", "2")

    keysCount = len(rc.keys())
    if keysCount < 1024:
        for key in rc.keys():
            # 查询类型
            pprint.pprint("type:" + rc.type(key) + "$key:" + key + "$value:" + getValue(rc, key))
    #        print rc.get(key)
    else:
        print "keysCount:" + str(keysCount)
    pass
Esempio n. 13
0
class DB(object):
    def __init__(self, config):
        self.config = config
        cache_backend = self.config_get('cache_backend') or 'redis'

        if cache_backend == 'redis':
            self.__backend = cache_backend
            self.__host = self.config_get('cache_host') or '127.0.0.1'
            self.__port = self.config_get('cache_port') or 6379
            self.__db_name = self.config_get('cache_db_name') or 0

            self.db = redis.StrictRedis(host=self.__host,
                                        port=self.__port,
                                        db=self.__db_name)
        elif cache_backend == 'redis-cluster':
            self.__backend = cache_backend
            node1_host = self.config_get('node1_host')
            node1_port = self.config_get('node1_port')
            node2_host = self.config_get('node2_host')
            node2_port = self.config_get('node2_port')
            node3_host = self.config_get('node3_host')
            node3_port = self.config_get('node3_port')

            node4_host = self.config_get('node4_host')
            node4_port = self.config_get('node4_port')
            node5_host = self.config_get('node5_host')
            node5_port = self.config_get('node5_port')
            node6_host = self.config_get('node6_host')
            node6_port = self.config_get('node6_port')

            self.__host = (node1_host, node2_host, node3_host, node4_host,
                           node5_host, node6_host)
            self.__port = (node1_port, node2_port, node3_port, node4_port,
                           node5_port, node6_port)
            self.__db_name = 'cluster'

            cluster_nodes = [{
                'host': node1_host,
                'port': node1_port
            }, {
                'host': node2_host,
                'port': node2_port
            }, {
                'host': node3_host,
                'port': node3_port
            }, {
                'host': node4_host,
                'port': node4_port
            }, {
                'host': node5_host,
                'port': node5_port
            }, {
                'host': node6_host,
                'port': node6_port
            }]

            self.db = StrictRedisCluster(startup_nodes=cluster_nodes,
                                         decode_responses=True)

        self.set_value = self.db.set
        self.has_value = self.db.exists
        self.remove_value = self.db.delete

    def config_get(self, name):
        if isinstance(self.config, ConfigParser):
            return self.config.get('app:api', name)
        else:
            return self.config.get('app:api').get(name)

    def get(self, key):
        return self.db.get(key)

    def keys(self, prefix):
        keys = self.db.keys(prefix)
        return keys

    def put(self, key, value, ex=90000):
        self.set_value(key, value, ex)

    def remove(self, key):
        self.remove_value(key)

    def has(self, key):
        return self.has_value(key)

    def scan_iter(self, prefix=None):
        return [key for key in self.db.scan_iter(prefix)]

    def remove_pattern(self, prefix):
        for key in self.db.scan_iter(prefix):
            self.remove(key)

    def flushall(self):
        self.db.flushall()

    @property
    def backend(self):
        return self.__backend

    @property
    def host(self):
        return self.__host

    @property
    def port(self):
        return self.__port

    @property
    def db_name(self):
        return self.__db_name
Esempio n. 14
0
class RedisQueue(object):
    # def __init__(self, redis_type=None,**args):
    #
    # 	if redis_type == "cluster":
    # 		try:
    # 			self.rc = StrictRedisCluster(**args)
    # 		except Exception as e:
    # 			print("Connect Error!")
    # 	else:
    # 		try:
    # 			self.pool = redis.ConnectionPool(**args)
    # 			self.rc = redis.Redis(connection_pool=self.pool)
    # 		except Exception as e:
    # 			print("Connect Error!")

    def __init__(self, res):
        redis_type = res[1]
        if redis_type == "cluster":
            try:
                con = res[0]
                self.rc = StrictRedisCluster(**con)
            except Exception as e:
                print(e)
                print("Connect Error!")
        else:
            try:
                con = res[0]
                self.pool = redis.ConnectionPool(**con)
                self.rc = redis.Redis(connection_pool=self.pool)
            except Exception as e:
                print(e)
                print("Connect Error!")

    def get_all(self, key, block=True, timeout=None):
        for i in self.rc.keys():
            if key in str(i):
                type = self.rc.type(i)
                if type == 'string':
                    vals = self.rc.get(i)

                elif type == 'list':
                    vals = self.rc.lrange(i, 0, -1)

                elif type == 'set':
                    vals = self.rc.smembers(i)

                elif type == 'zset':
                    vals = self.rc.zrange(i, 0, -1)

                elif type == "hash":
                    vals = self.rc.hgetall(i)
                else:
                    print(type, i)
        return list(vals[0])

    def keys(self):
        keys = self.rc.keys()
        return keys

    def iskey(self, key):
        if self.rc.exists(key):
            return 1
        else:
            return 0

    def get(self, key):
        res = self.rc.get(key)
        nres = json.loads(res)
        return nres

    def put(self, key, value):
        new_value = json.dumps(value)
        res = self.rc.set(key, new_value)

    def delall(self, key):
        self.rc.delete(key)


# print(getredis('ak_sit'))
#RedisQueue(getredis('ak_cluster'))
Esempio n. 15
0
class RedisDB:
    def __init__(self,
                 ip_ports=None,
                 db=None,
                 user_pass=None,
                 url=None,
                 decode_responses=True,
                 service_name=None,
                 max_connections=32,
                 **kwargs):
        """
        redis的封装
        Args:
            ip_ports: ip:port 多个可写为列表或者逗号隔开 如 ip1:port1,ip2:port2 或 ["ip1:port1", "ip2:port2"]
            db:
            user_pass:
            url:
            decode_responses:
            service_name: 适用于redis哨兵模式
        """

        # 可能会改setting中的值,所以此处不能直接赋值为默认值,需要后加载赋值
        if ip_ports is None:
            ip_ports = setting.REDISDB_IP_PORTS
        if db is None:
            db = setting.REDISDB_DB
        if user_pass is None:
            user_pass = setting.REDISDB_USER_PASS
        if service_name is None:
            service_name = setting.REDISDB_SERVICE_NAME

        self._is_redis_cluster = False

        try:
            if not url:
                ip_ports = (ip_ports if isinstance(ip_ports, list) else
                            ip_ports.split(","))
                if len(ip_ports) > 1:
                    startup_nodes = []
                    for ip_port in ip_ports:
                        ip, port = ip_port.split(":")
                        startup_nodes.append({"host": ip, "port": port})

                    if service_name:
                        log.debug("使用redis哨兵模式")
                        hosts = [(node["host"], node["port"])
                                 for node in startup_nodes]
                        sentinel = Sentinel(hosts, socket_timeout=3, **kwargs)
                        self._redis = sentinel.master_for(
                            service_name,
                            password=user_pass,
                            db=db,
                            redis_class=redis.StrictRedis,
                            decode_responses=decode_responses,
                            max_connections=max_connections,
                            **kwargs)

                    else:
                        log.debug("使用redis集群模式")
                        self._redis = StrictRedisCluster(
                            startup_nodes=startup_nodes,
                            decode_responses=decode_responses,
                            password=user_pass,
                            max_connections=max_connections,
                            **kwargs)

                    self._is_redis_cluster = True
                else:
                    ip, port = ip_ports[0].split(":")
                    self._redis = redis.StrictRedis(
                        host=ip,
                        port=port,
                        db=db,
                        password=user_pass,
                        decode_responses=decode_responses,
                        max_connections=max_connections,
                        **kwargs)
            else:
                self._redis = redis.StrictRedis.from_url(
                    url, decode_responses=decode_responses)

        except Exception as e:
            raise
        else:
            if not url:
                log.debug("连接到redis数据库 %s db%s" % (ip_ports, db))
            else:
                log.debug("连接到redis数据库 %s" % (url))

        self._ip_ports = ip_ports
        self._db = db
        self._user_pass = user_pass
        self._url = url

    def __repr__(self):
        if self._url:
            return "<Redisdb url:{}>".format(self._url)

        return "<Redisdb ip_ports: {} db:{} user_pass:{}>".format(
            self._ip_ports, self._db, self._user_pass)

    @classmethod
    def from_url(cls, url):
        return cls(url=url)

    def sadd(self, table, values):
        """
        @summary: 使用无序set集合存储数据, 去重
        ---------
        @param table:
        @param values: 值; 支持list 或 单个值
        ---------
        @result: 若库中存在 返回0,否则入库,返回1。 批量添加返回None
        """

        if isinstance(values, list):
            pipe = self._redis.pipeline(
                transaction=True
            )  # redis-py默认在执行每次请求都会创建(连接池申请连接)和断开(归还连接池)一次连接操作,如果想要在一次请求中指定多个命令,则可以使用pipline实现一次请求指定多个命令,并且默认情况下一次pipline 是原子性操作。

            if not self._is_redis_cluster:
                pipe.multi()
            for value in values:
                pipe.sadd(table, value)
            pipe.execute()

        else:
            return self._redis.sadd(table, values)

    def sget(self, table, count=1, is_pop=True):
        """
        返回 list 如 ['1'] 或 []
        @param table:
        @param count:
        @param is_pop:
        @return:
        """
        datas = []
        if is_pop:
            count = count if count <= self.sget_count(
                table) else self.sget_count(table)
            if count:
                if count > 1:
                    pipe = self._redis.pipeline(
                        transaction=True
                    )  # redis-py默认在执行每次请求都会创建(连接池申请连接)和断开(归还连接池)一次连接操作,如果想要在一次请求中指定多个命令,则可以使用pipline实现一次请求指定多个命令,并且默认情况下一次pipline 是原子性操作。

                    if not self._is_redis_cluster:
                        pipe.multi()
                    while count:
                        pipe.spop(table)
                        count -= 1
                    datas = pipe.execute()

                else:
                    datas.append(self._redis.spop(table))

        else:
            datas = self._redis.srandmember(table, count)

        return datas

    def srem(self, table, values):
        """
        @summary: 移除集合中的指定元素
        ---------
        @param table:
        @param values: 一个或者列表
        ---------
        @result:
        """
        if isinstance(values, list):
            pipe = self._redis.pipeline(
                transaction=True
            )  # redis-py默认在执行每次请求都会创建(连接池申请连接)和断开(归还连接池)一次连接操作,如果想要在一次请求中指定多个命令,则可以使用pipline实现一次请求指定多个命令,并且默认情况下一次pipline 是原子性操作。

            if not self._is_redis_cluster:
                pipe.multi()
            for value in values:
                pipe.srem(table, value)
            pipe.execute()
        else:
            self._redis.srem(table, values)

    def sget_count(self, table):
        return self._redis.scard(table)

    def sdelete(self, table):
        """
        @summary: 删除set集合的大键(数据量大的表)
        删除大set键,使用sscan命令,每次扫描集合中500个元素,再用srem命令每次删除一个键
        若直接用delete命令,会导致Redis阻塞,出现故障切换和应用程序崩溃的故障。
        ---------
        @param table:
        ---------
        @result:
        """
        # 当 SCAN 命令的游标参数被设置为 0 时, 服务器将开始一次新的迭代, 而当服务器向用户返回值为 0 的游标时, 表示迭代已结束
        cursor = "0"
        while cursor != 0:
            cursor, data = self._redis.sscan(table, cursor=cursor, count=500)
            for item in data:
                # pipe.srem(table, item)
                self._redis.srem(table, item)

            # pipe.execute()

    def sismember(self, table, key):
        "Return a boolean indicating if ``value`` is a member of set ``name``"
        return self._redis.sismember(table, key)

    def zadd(self, table, values, prioritys=0):
        """
        @summary: 使用有序set集合存储数据, 去重(值存在更新)
        ---------
        @param table:
        @param values: 值; 支持list 或 单个值
        @param prioritys: 优先级; double类型,支持list 或 单个值。 根据此字段的值来排序, 值越小越优先。 可不传值,默认value的优先级为0
        ---------
        @result:若库中存在 返回0,否则入库,返回1。 批量添加返回 [0, 1 ...]
        """
        if isinstance(values, list):
            if not isinstance(prioritys, list):
                prioritys = [prioritys] * len(values)
            else:
                assert len(values) == len(prioritys), "values值要与prioritys值一一对应"

            pipe = self._redis.pipeline(transaction=True)

            if not self._is_redis_cluster:
                pipe.multi()
            for value, priority in zip(values, prioritys):
                pipe.zadd(table, priority, value)
            return pipe.execute()

        else:
            return self._redis.zadd(table, prioritys, values)

    def zget(self, table, count=1, is_pop=True):
        """
        @summary: 从有序set集合中获取数据 优先返回分数小的(优先级高的)
        ---------
        @param table:
        @param count: 数量 -1 返回全部数据
        @param is_pop:获取数据后,是否在原set集合中删除,默认是
        ---------
        @result: 列表
        """
        start_pos = 0  # 包含
        end_pos = count - 1 if count > 0 else count

        pipe = self._redis.pipeline(
            transaction=True
        )  # redis-py默认在执行每次请求都会创建(连接池申请连接)和断开(归还连接池)一次连接操作,如果想要在一次请求中指定多个命令,则可以使用pipline实现一次请求指定多个命令,并且默认情况下一次pipline 是原子性操作。

        if not self._is_redis_cluster:
            pipe.multi(
            )  # 标记事务的开始 参考 http://www.runoob.com/redis/redis-transactions.html
        pipe.zrange(table, start_pos, end_pos)  # 取值
        if is_pop:
            pipe.zremrangebyrank(table, start_pos, end_pos)  # 删除
        results, *count = pipe.execute()
        return results

    def zremrangebyscore(self, table, priority_min, priority_max):
        """
        根据分数移除成员 闭区间
        @param table:
        @param priority_min:
        @param priority_max:
        @return: 被移除的成员个数
        """
        return self._redis.zremrangebyscore(table, priority_min, priority_max)

    def zrangebyscore(self,
                      table,
                      priority_min,
                      priority_max,
                      count=None,
                      is_pop=True):
        """
        @summary: 返回指定分数区间的数据 闭区间
        ---------
        @param table:
        @param priority_min: 优先级越小越优先
        @param priority_max:
        @param count: 获取的数量,为空则表示分数区间内的全部数据
        @param is_pop: 是否删除
        ---------
        @result:
        """

        # 使用lua脚本, 保证操作的原子性
        lua = """
            local key = KEYS[1]
            local min_score = ARGV[2]
            local max_score = ARGV[3]
            local is_pop = ARGV[4]
            local count = ARGV[5]

            -- 取值
            local datas = nil
            if count then
                datas = redis.call('zrangebyscore', key, min_score, max_score, 'limit', 0, count)
            else
                datas = redis.call('zrangebyscore', key, min_score, max_score)
            end

            -- 删除redis中刚取到的值
            if (is_pop) then
                for i=1, #datas do
                    redis.call('zrem', key, datas[i])
                end
            end


            return datas

        """
        cmd = self._redis.register_script(lua)
        if count:
            res = cmd(keys=[table],
                      args=[table, priority_min, priority_max, is_pop, count])
        else:
            res = cmd(keys=[table],
                      args=[table, priority_min, priority_max, is_pop])

        return res

    def zrangebyscore_increase_score(self,
                                     table,
                                     priority_min,
                                     priority_max,
                                     increase_score,
                                     count=None):
        """
        @summary: 返回指定分数区间的数据 闭区间, 同时修改分数
        ---------
        @param table:
        @param priority_min: 最小分数
        @param priority_max: 最大分数
        @param increase_score: 分数值增量 正数则在原有的分数上叠加,负数则相减
        @param count: 获取的数量,为空则表示分数区间内的全部数据
        ---------
        @result:
        """

        # 使用lua脚本, 保证操作的原子性
        lua = """
            local key = KEYS[1]
            local min_score = ARGV[1]
            local max_score = ARGV[2]
            local increase_score = ARGV[3]
            local count = ARGV[4]

            -- 取值
            local datas = nil
            if count then
                datas = redis.call('zrangebyscore', key, min_score, max_score, 'limit', 0, count)
            else
                datas = redis.call('zrangebyscore', key, min_score, max_score)
            end

            --修改优先级
            for i=1, #datas do
                redis.call('zincrby', key, increase_score, datas[i])
            end

            return datas

        """
        cmd = self._redis.register_script(lua)
        if count:
            res = cmd(keys=[table],
                      args=[priority_min, priority_max, increase_score, count])
        else:
            res = cmd(keys=[table],
                      args=[priority_min, priority_max, increase_score])

        return res

    def zrangebyscore_set_score(self,
                                table,
                                priority_min,
                                priority_max,
                                score,
                                count=None):
        """
        @summary: 返回指定分数区间的数据 闭区间, 同时修改分数
        ---------
        @param table:
        @param priority_min: 最小分数
        @param priority_max: 最大分数
        @param score: 分数值
        @param count: 获取的数量,为空则表示分数区间内的全部数据
        ---------
        @result:
        """

        # 使用lua脚本, 保证操作的原子性
        lua = """
            local key = KEYS[1]
            local min_score = ARGV[1]
            local max_score = ARGV[2]
            local set_score = ARGV[3]
            local count = ARGV[4]

            -- 取值
            local datas = nil
            if count then
                datas = redis.call('zrangebyscore', key, min_score, max_score, 'withscores','limit', 0, count)
            else
                datas = redis.call('zrangebyscore', key, min_score, max_score, 'withscores')
            end

            local real_datas = {} -- 数据
            --修改优先级
            for i=1, #datas, 2 do
               local data = datas[i]
               local score = datas[i+1]

               table.insert(real_datas, data) -- 添加数据

               redis.call('zincrby', key, set_score - score, datas[i])
            end

            return real_datas

        """
        cmd = self._redis.register_script(lua)
        if count:
            res = cmd(keys=[table],
                      args=[priority_min, priority_max, score, count])
        else:
            res = cmd(keys=[table], args=[priority_min, priority_max, score])

        return res

    def zget_count(self, table, priority_min=None, priority_max=None):
        """
        @summary: 获取表数据的数量
        ---------
        @param table:
        @param priority_min:优先级范围 最小值(包含)
        @param priority_max:优先级范围 最大值(包含)
        ---------
        @result:
        """

        if priority_min != None and priority_max != None:
            return self._redis.zcount(table, priority_min, priority_max)
        else:
            return self._redis.zcard(table)

    def zrem(self, table, values):
        """
        @summary: 移除集合中的指定元素
        ---------
        @param table:
        @param values: 一个或者列表
        ---------
        @result:
        """
        if isinstance(values, list):
            pipe = self._redis.pipeline(
                transaction=True
            )  # redis-py默认在执行每次请求都会创建(连接池申请连接)和断开(归还连接池)一次连接操作,如果想要在一次请求中指定多个命令,则可以使用pipline实现一次请求指定多个命令,并且默认情况下一次pipline 是原子性操作。

            if not self._is_redis_cluster:
                pipe.multi()
            for value in values:
                pipe.zrem(table, value)
            pipe.execute()
        else:
            self._redis.zrem(table, values)

    def zexists(self, table, values):
        """
        利用zscore判断某元素是否存在
        @param values:
        @return:
        """
        is_exists = []

        if isinstance(values, list):
            pipe = self._redis.pipeline(
                transaction=True
            )  # redis-py默认在执行每次请求都会创建(连接池申请连接)和断开(归还连接池)一次连接操作,如果想要在一次请求中指定多个命令,则可以使用pipline实现一次请求指定多个命令,并且默认情况下一次pipline 是原子性操作。
            pipe.multi()
            for value in values:
                pipe.zscore(table, value)
            is_exists_temp = pipe.execute()
            for is_exist in is_exists_temp:
                if is_exist != None:
                    is_exists.append(1)
                else:
                    is_exists.append(0)

        else:
            is_exists = self._redis.zscore(table, values)
            is_exists = 1 if is_exists != None else 0

        return is_exists

    def lpush(self, table, values):
        if isinstance(values, list):
            pipe = self._redis.pipeline(
                transaction=True
            )  # redis-py默认在执行每次请求都会创建(连接池申请连接)和断开(归还连接池)一次连接操作,如果想要在一次请求中指定多个命令,则可以使用pipline实现一次请求指定多个命令,并且默认情况下一次pipline 是原子性操作。

            if not self._is_redis_cluster:
                pipe.multi()
            for value in values:
                pipe.rpush(table, value)
            pipe.execute()

        else:
            return self._redis.rpush(table, values)

    def lpop(self, table, count=1):
        """
        @summary:
        ---------
        @param table:
        @param count:
        ---------
        @result: count>1时返回列表
        """
        datas = None

        count = count if count <= self.lget_count(table) else self.lget_count(
            table)

        if count:
            if count > 1:
                pipe = self._redis.pipeline(
                    transaction=True
                )  # redis-py默认在执行每次请求都会创建(连接池申请连接)和断开(归还连接池)一次连接操作,如果想要在一次请求中指定多个命令,则可以使用pipline实现一次请求指定多个命令,并且默认情况下一次pipline 是原子性操作。

                if not self._is_redis_cluster:
                    pipe.multi()
                while count:
                    pipe.lpop(table)
                    count -= 1
                datas = pipe.execute()

            else:
                datas = self._redis.lpop(table)

        return datas

    def rpoplpush(self, from_table, to_table=None):
        """
        将列表 from_table 中的最后一个元素(尾元素)弹出,并返回给客户端。
        将 from_table 弹出的元素插入到列表 to_table ,作为 to_table 列表的的头元素。
        如果 from_table 和 to_table 相同,则列表中的表尾元素被移动到表头,并返回该元素,可以把这种特殊情况视作列表的旋转(rotation)操作
        @param from_table:
        @param to_table:
        @return:
        """

        if not to_table:
            to_table = from_table

        return self._redis.rpoplpush(from_table, to_table)

    def lget_count(self, table):
        return self._redis.llen(table)

    def lrem(self, table, value, num=0):
        return self._redis.lrem(table, value, num)

    def hset(self, table, key, value):
        """
        @summary:
        如果 key 不存在,一个新的哈希表被创建并进行 HSET 操作。
        如果域 field 已经存在于哈希表中,旧值将被覆盖
        ---------
        @param table:
        @param key:
        @param value:
        ---------
        @result: 1 新插入; 0 覆盖
        """

        return self._redis.hset(table, key, value)

    def hset_batch(self, table, datas):
        """
        批量插入
        Args:
            datas:
                [[key, value]]
        Returns:

        """
        pipe = self._redis.pipeline(transaction=True)

        if not self._is_redis_cluster:
            pipe.multi()
        for key, value in datas:
            pipe.hset(table, key, value)
        return pipe.execute()

    def hincrby(self, table, key, increment):
        return self._redis.hincrby(table, key, increment)

    def hget(self, table, key, is_pop=False):
        if not is_pop:
            return self._redis.hget(table, key)
        else:
            lua = """
                local key = KEYS[1]
                local field = ARGV[1]

                -- 取值
                local datas = redis.call('hget', key, field)
                -- 删除值
                redis.call('hdel', key, field)

                return datas

                    """
            cmd = self._redis.register_script(lua)
            res = cmd(keys=[table], args=[key])

            return res

    def hgetall(self, table):
        return self._redis.hgetall(table)

    def hexists(self, table, key):
        return self._redis.hexists(table, key)

    def hdel(self, table, *keys):
        """
        @summary: 删除对应的key 可传多个
        ---------
        @param table:
        @param *keys:
        ---------
        @result:
        """

        self._redis.hdel(table, *keys)

    def hget_count(self, table):
        return self._redis.hlen(table)

    def setbit(self, table, offsets, values):
        """
        设置字符串数组某一位的值, 返回之前的值
        @param table:
        @param offsets: 支持列表或单个值
        @param values: 支持列表或单个值
        @return: list / 单个值
        """
        if isinstance(offsets, list):
            if not isinstance(values, list):
                values = [values] * len(offsets)
            else:
                assert len(offsets) == len(values), "offsets值要与values值一一对应"

            pipe = self._redis.pipeline(
                transaction=True
            )  # redis-py默认在执行每次请求都会创建(连接池申请连接)和断开(归还连接池)一次连接操作,如果想要在一次请求中指定多个命令,则可以使用pipline实现一次请求指定多个命令,并且默认情况下一次pipline 是原子性操作。
            pipe.multi()

            for offset, value in zip(offsets, values):
                pipe.setbit(table, offset, value)

            return pipe.execute()

        else:
            return self._redis.setbit(table, offsets, values)

    def getbit(self, table, offsets):
        """
        取字符串数组某一位的值
        @param table:
        @param offsets: 支持列表
        @return: list / 单个值
        """
        if isinstance(offsets, list):
            pipe = self._redis.pipeline(
                transaction=True
            )  # redis-py默认在执行每次请求都会创建(连接池申请连接)和断开(归还连接池)一次连接操作,如果想要在一次请求中指定多个命令,则可以使用pipline实现一次请求指定多个命令,并且默认情况下一次pipline 是原子性操作。
            pipe.multi()
            for offset in offsets:
                pipe.getbit(table, offset)

            return pipe.execute()

        else:
            return self._redis.getbit(table, offsets)

    def bitcount(self, table):
        return self._redis.bitcount(table)

    def strset(self, table, value, **kwargs):
        return self._redis.set(table, value, **kwargs)

    def str_incrby(self, table, value):
        return self._redis.incrby(table, value)

    def strget(self, table):
        return self._redis.get(table)

    def strlen(self, table):
        return self._redis.strlen(table)

    def getkeys(self, regex):
        return self._redis.keys(regex)

    def exists_key(self, key):
        return self._redis.exists(key)

    def set_expire(self, key, seconds):
        """
        @summary: 设置过期时间
        ---------
        @param key:
        @param seconds: 秒
        ---------
        @result:
        """

        self._redis.expire(key, seconds)

    def clear(self, table):
        try:
            self._redis.delete(table)
        except Exception as e:
            log.error(e)

    def get_redis_obj(self):
        return self._redis
Esempio n. 16
0
#!/usr/bin/env python
# _*_coding:utf-8_*_
# Author: create by yang.hong
# Time: 2018-08-17 11:06
"""
集群模式下,匹配方式删除多个key
"""

from rediscluster import StrictRedisCluster

redis_cluster = [{'host': '172.28.246.12', 'port': '28000'},
                 {'host': '172.28.246.12', 'port': '28001'},
                 {'host': '172.28.246.12', 'port': '28002'},
                 {'host': '172.28.246.12', 'port': '28003'},
                 {'host': '172.28.246.12', 'port': '28004'},
                 {'host': '172.28.246.12', 'port': '28005'},
                 ]

r = StrictRedisCluster(startup_nodes=redis_cluster, decode_responses=True)
print r.keys(pattern="*mc_payChannel_router*")
r.delete(*r.keys(pattern="*mc_payChannel_router*"))
def read_seed_pool_redis():
    peer_ids_in_redis = []
    seeds_add_to_mysql = []
    file_ids = []
    redis_nodes = [
        {'host': SEED_POOL_REDIS_CLUSTER_HOST_1,
         'port': SEED_POOL_REDIS_CLUSTER_PORT_1},
        {'host': SEED_POOL_REDIS_CLUSTER_HOST_2,
         'port': SEED_POOL_REDIS_CLUSTER_PORT_2},
        {'host': SEED_POOL_REDIS_CLUSTER_HOST_3,
         'port': SEED_POOL_REDIS_CLUSTER_PORT_3}
    ]

    peer_ids_in_mysql = obtain_peer_id_from_table(Seed_pool)
    redis_connect = StrictRedisCluster(startup_nodes=redis_nodes)

    # get keys contain TSGO
    for key in redis_connect.keys():
        # key: {TSGO
        if key.find('TSGO') == 1:
            file_ids.append(key)

    if file_ids:
        # get file id, currently we only have one channel, so we only handle
        # file_ids[0]
        for file_id in file_ids:
            file_id_write_to_mysql = file_id.split("_")[2]
            # get value by key
            selected_seeds = redis_connect.smembers(file_id)
            log.logger.info('redis seeds num:{0}'.format(len(selected_seeds)))
            for seed in selected_seeds:
                # eval it from string to dictionary
                seed_info = eval(seed)
                peer_id = seed_info['peer_id']
                peer_ids_in_redis.append(peer_id)
                if peer_id not in peer_ids_in_mysql:
                    sdk_version = seed_info['version']
                    private_ip = seed_info['privateIP']
                    private_port = seed_info['privatePort']
                    public_ip = seed_info['publicIP']
                    public_port = seed_info['publicPort']
                    nat_type = seed_info['natType']
                    stun_ip = seed_info['stunIP']
                    isp_id = seed_info['isp_id']
                    province_id = seed_info['province_id']
                    ppc = seed_info['ppc']
                    cppc = seed_info['cppc']
                    seed_redis = (Seed_pool(peer_id=peer_id,
                                            sdk_version=sdk_version,
                                            public_ip=public_ip,
                                            public_port=public_port,
                                            private_ip=private_ip,
                                            private_port=private_port,
                                            nat_type=nat_type,
                                            stun_ip=stun_ip,
                                            isp_id=isp_id,
                                            province_id=province_id,
                                            ppc=ppc,
                                            cppc=cppc,
                                            file_id=file_id_write_to_mysql))
                    orm.session.add(seed_redis)
                    seeds_add_to_mysql.append(seed_redis)
        orm.session.commit()
        orm.session.close()
        log.logger.info('add {0} seeds to mysql'.format(len(seeds_add_to_mysql)))
        # del seeds expired(in mysql but not in redis)
        peer_ids_in_mysql_after_add = obtain_peer_id_from_table(Seed_pool)
        for peer in peer_ids_in_mysql_after_add:
            if peer not in peer_ids_in_redis:
                orm.session.query(Seed_pool).filter_by(
                    peer_id=peer).delete()
        orm.session.commit()
        orm.session.close()
    else:
        # if seeds list in redis is empty, clean MySQL
        log.logger.info('no seeds in redis so we clean table seed_pool ')
        orm.session.query(Seed_pool).filter().delete()
        orm.session.commit()
        orm.session.close()