Esempio n. 1
0
def migrate(srchost, dsthost, srccluster, dstcluster, db, flush):
    if srchost == dsthost:
        print 'Source and destination must be different.'
        return

    if srccluster:
        source_nodes = literal_eval(srchost)
        source = StrictRedisCluster(startup_nodes=source_nodes,
                                    decode_responses=True)
        logging.debug('source cluster info: %s', source.cluster_info())
    else:
        source = redis.Redis(srchost, db=db)
    if dstcluster:
        dest_nodes = literal_eval(dsthost)
        dest = StrictRedisCluster(startup_nodes=dest_nodes,
                                  decode_responses=True)
        logging.debug('dest cluster info: %s', dest.cluster_info())
    else:
        dest = redis.Redis(dsthost, db=db)

    if flush:
        dest.flushdb()

    if srccluster:
        representatives = {
            v['cluster_my_epoch']: k
            for k, v in source.cluster_info().items()
        }
        size = source.dbsize()
        size = sum(size[reprensentative]
                   for reprensentative in representatives.values())
    else:
        size = source.dbsize()

    if size == 0:
        print 'No keys found.'
        return

    progress_widgets = [
        '%d keys: ' % size,
        Percentage(), ' ',
        Bar(), ' ', ETA()
    ]
    pbar = ProgressBar(widgets=progress_widgets, maxval=size).start()

    COUNT = 2000  # scan size

    cnt = 0
    non_existing = 0
    already_existing = 0
    cursor = 0

    if srccluster:
        counter = 0
        keys = []
        # iterate all the keys
        for key in source.scan_iter(count=COUNT):
            counter += 1
            keys.append(key)

            if counter % COUNT == 0:
                already_existing, non_existing = handle_keys(
                    source, dest, keys, already_existing, non_existing)

                cnt += len(keys)
                pbar.update(min(size, cnt))
                keys = []

        # handle the remaining
        if len(keys) > 0:
            already_existing, non_existing = handle_keys(
                source, dest, keys, already_existing, non_existing)
            cnt += len(keys)
            pbar.update(min(size, cnt))

    else:
        while True:
            cursor, keys = source.scan(cursor, count=COUNT)
            already_existing, non_existing = handle_keys(
                source, dest, keys, already_existing, non_existing)

            if cursor == 0:
                break

            cnt += len(keys)
            pbar.update(min(size, cnt))

    pbar.finish()
    print 'Keys disappeared on source during scan:', non_existing
    print 'Keys already existing on destination:', already_existing
Esempio n. 2
0
class RedisCluster:
    def __init__(self):
        try:
            self.rc = StrictRedisCluster(startup_nodes=StartupNodesServer,
                                         decode_responses=True)
        except:
            traceback.print_exc()

    def count_keys(self):  # 查询当前库里有多少key
        return self.rc.dbsize()

    def exists_key(self, key):
        return self.rc.exists(key)

    def delete_key(self, key):
        self.rc.delete(key)

    def rename_key(self, key1, key2):
        self.rc.rename(key1, key2)

    # String操作
    def set_key_value(self, key, value):
        self.rc.set(key, value)

    def get_key_value(self, key):  # 没有对应key返回None
        return self.rc.get(key)

    # Hash操作
    def set_hash(self, key, mapping):  # mapping为字典, 已存在key会覆盖mapping
        self.rc.hmset(key, mapping)

    def delete_hash_field(self, key, field):  # 删除hash表中某个字段,无论字段是否存在
        self.rc.hdel(key, field)

    def exists_hash_field(self, key, field):  # 检查hash表中某个字段存在
        return self.rc.hexists(key, field)

    def get_hash_field(self, key, field):  # 获取hash表中指定字段的值, 没有返回None
        return self.rc.hget(key, field)

    def get_hash_all_field(self, key):  # 获取hash表中指定key所有字段和值,以字典形式,没有key返回空字典
        return self.rc.hgetall(key)

    def increase_hash_field(self, key, field,
                            increment):  # 为hash表key某个字段的整数型值增加increment
        self.rc.hincrby(key, field, increment)

    # List操作
    def rpush_into_lst(self, key, value):  # url从头至尾入列
        self.rc.rpush(key, value)

    def lpush_into_lst(self, key, value):  # url从尾至头入列
        self.rc.lpush(key, value)

    def lpop_lst_item(self, key):  # 从头取出列表第一个元素,没有返回None
        return self.rc.lpop(key)

    def blpop_lst_item(
            self, key):  # 从头取出列表第一个元素(元组形式,值为元祖[1], 元祖[0]为key名),并设置超时,超时返回None
        return self.rc.blpop(key, timeout=1)

    def rpop_lst_item(self, key):  # 从尾取出列表最后一个元素,没有返回None
        return self.rc.rpop(key)

    def brpop_lst_item(
            self,
            key):  # 从尾取出列表最后一个元素(元组形式,值为元祖[1], 元祖[0]为key名),并设置超时,超时返回None
        return self.rc.brpop(key, timeout=1)

    # Set操作
    def add_set(self, key, value):
        self.rc.sadd(key, value)

    def is_member(self, key, value):
        return self.rc.sismember(key, value)

    def pop_member(self, key):  # 随机移除一个值并返回该值,没有返回None
        return self.rc.spop(key)

    def pop_members(self, key, num):  # 随机取出num个值(非移除),列表形式返回这些值,没有返回空列表
        return self.rc.srandmember(key, num)

    def remove_member(self, key, value):  # 移除集合中指定元素
        self.rc.srem(key, value)

    def get_all_members(self, key):  # 返回集合中全部元素,不删除
        return self.rc.smembers(key)

    def remove_into(self, key1, key2, value):  # 把集合key1中value元素移入集合key2中
        self.rc.smove(key1, key2, value)

    def count_members(self, key):  # 计算集合中成员数量
        return self.rc.scard(key)
Esempio n. 3
0
class Redis:
    def __init__(self, ci):
        log.debug('create connection = %s', ci)
        t = ci.type
        self.t = t
        if t == 1:
            log.debug('create redis connection.')
            self.conn = StrictRedis(host=ci.host, port=ci.port, db=ci.db)
        elif t == 2:
            log.debug('create redis cluster connection.')
            nodes = json.loads(ci.host)
            pool = ClusterConnectionPool(startup_nodes=nodes)
            self.conn = StrictRedisCluster(connection_pool=pool, decode_responses=True)
        elif t == 3:
            log.debug('create redis connection from zookeeper.')
            client = zk.Client(hosts=ci.host, read_only=True)
            node = client.get(ci.path)
            arr = str(node[0], encoding='utf-8').split('\n')
            address = []
            for h in arr:
                if h is '':
                    continue
                a = h.split(':')
                address.append({'host': a[0], 'port': int(a[1])})
            pool = ClusterConnectionPool(startup_nodes=address)
            self.conn = StrictRedisCluster(connection_pool=pool, decode_responses=True)
        else:
            raise AttributeError('illegal ConnInfo type.')
        if self.test():
            self.ci = ci
            log.info('connect redis(%s) success', ci.host)

    def test(self):
        log.debug('test connect redis(%s)', self.conn)
        good = False
        try:
            result = self.conn.ping()
            if self.t == 1:
                good = result
            else:
                for k in result:
                    v = result[k]
                    log.debug('test [%s] result : %s', k, v)
                    if not v:
                        return False
                good = True
        except Exception as e:
            log.error(e)
        finally:
            log.debug('redis connection is good[%s]', good)
        return good

    def db_size(self):
        return self.conn.dbsize()

    def scan_iter(self, match='*', count=None):
        if match is not '*':
            match = '*' + match + '*'
        return self.conn.scan_iter(match=match, count=count)

    def get_str(self, key):
        log.debug('get str value by key: %s', key)
        return self.conn.get(key)

    def l_range(self, key):
        start = 0
        end = -1
        log.debug('get list value from %d to %d by key: %s', start, end, key)
        return self.conn.lrange(key, start, end)

    def z_range(self, key):
        start = 0
        end = -1
        log.debug('get sorted set value from %d to %d by key: %s', start, end, key)
        return self.conn.zrange(key, start, end)

    def s_members(self, key):
        log.debug('get set value by key: %s', key)
        return self.conn.smembers(key)

    def h_get_all(self, key):
        log.debug('get hash value by key: %s', key)
        return self.conn.hgetall(key)

    def get(self, t, k):
        f = self.__t_f_map__[t]
        return f(self, k)

    def type(self, key):
        log.debug('get type by key: %s', key)
        return self.conn.type(key)

    __t_f_map__ = {
        b'string': get_str,
        b'list': l_range,
        b'set': s_members,
        b'zset': z_range,
        b'hash': h_get_all,
    }
Esempio n. 4
0
'''

rc = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True)
redis_cluster_info = rc.cluster_info()
redis_nodes_info = rc.info()
redis_nodes_config = rc.config_get()
'''
        redis ip and port
'''

monitor_redis_node = "192.168.12.171:6379"
monitor_key = "used_memory"
DBsize = ["dbsize", "keys", "cluster_size"]

if monitor_key in redis_nodes_config[monitor_redis_node]:
    print redis_nodes_config[monitor_key]
elif monitor_key in monitor_cluster_key:
    print str(redis_cluster_info[monitor_redis_node][monitor_key])
elif monitor_key in DBsize:
    print rc.dbsize()[monitor_redis_node]
else:
    try:
        print redis_nodes_info[monitor_redis_node][monitor_key]
    except:
        print monitor_key + " is not found"

#print redis_nodes_config["192.168.12.172:6379"]
#print  redis_nodes_info["192.168.12.172:6379"]
#for k in redis_cluster_info:
#    print redis_cluster_info[k]
#    print k +" " + str(redis_cluster_info[k]["cluster_size"])