Esempio n. 1
0
    def get_master(self):
        print(self.redis_hosts)
        sentinel_hosts = []
        for redis_host in self.redis_hosts:
            redis_address_array = redis_host.split(':')
            redis_ip = redis_address_array[0]
            redis_port = redis_address_array[1]
            address = (redis_ip, redis_port)
            sentinel_hosts.append(address)

        print(sentinel_hosts)

        sentinel = Sentinel([('192.168.206.41', 26379),
                             ('192.168.206.42', 26379),
                             ('192.168.206.43', 26379)],
                            socket_timeout=0.1)
        print(sentinel.discover_master('mymaster'))
        master = sentinel.discover_master('mymaster')
        self.master_ip = master[0]
        self.master_port = master[1]
        print(str(master[0]) + ' is master')
        self.ip_master = {
            "addresses": [{
                "ip": self.master_ip
            }],
            "ports": [{
                "port": int(self.master_port),
                "protocol": "TCP"
            }]
        }
        return True
Esempio n. 2
0
    def connect(self, write=True):
        """
        Create a redis connection with connection pool.
        """
        self.log.debug("connect called: write=%s", write)

        sentinel_timeout = self._options.get("SENTINEL_TIMEOUT", 1)
        password = self._options.get("PASSWORD", None)
        sentinel = Sentinel(
            self._sentinel_hosts, socket_timeout=sentinel_timeout, password=password
        )

        if write:
            host, port = sentinel.discover_master(self._master_name)
        else:
            try:
                host, port = random.choice(  # nosec
                    sentinel.discover_slaves(self._master_name)
                )
            except IndexError:
                self.log.debug("no slaves are available. using master for read.")
                host, port = sentinel.discover_master(self._master_name)

        if password:
            connection_url = f"redis://:{password}@{host}:{port}/{self._database_name}"
        else:
            connection_url = f"redis://{host}:{port}/{self._database_name}"
        self.log.debug("Connecting to: %s", connection_url)
        return self.connection_factory.connect(connection_url)
class RedisSentinelMessageBroker(RedisMessageBroker):
    """ Redis Sentinel Message broker. 
    """
    def __init__(self, **kwargs):
        self.init_sentinel(**kwargs)
        RedisMessageBroker.__init__(self, **kwargs)

    def init_sentinel(self, **kwargs):
        if not kwargs.has_key('sentinel_servers'):
            raise ConfigurationError('Kewyword sentinel_servers is missing.')
        if not kwargs.has_key('sentinel_master'):
            raise ConfigurationError('Kewyword sentinel_master is missing.')

        # Enable sentinel support
        self.__sentinel_servers = kwargs['sentinel_servers']
        self.__sentinel_master = kwargs['sentinel_master']
        del kwargs['sentinel_servers']
        del kwargs['sentinel_master']
        self.__sentinel = Sentinel(self.__sentinel_servers, **kwargs)

    def configure_redis(self, **kwargs):
        return self.__sentinel.master_for(self.__sentinel_master)

    def get_connection_port(self):
        return self.__sentinel.discover_master(self.__sentinel_master)[1]

    def get_connection_host(self):
        return self.__sentinel.discover_master(self.__sentinel_master)[0]
Esempio n. 4
0
def redis_populate():
    """Function to populate keys in Redis Server"""
    # client = redis.StrictRedis(host=configs["redis_host"], port=configs["redis_port"])
    # sentinel = Sentinel(
    #   [
    #     ("redis-cloudflare-node-0.redis-cloudflare-headless.dns-proxy.svc.cluster.local", 26379),
    #     ("redis-cloudflare-node-1.redis-cloudflare-headless.dns-proxy.svc.cluster.local", 26379),
    #     ("redis-cloudflare-node-2.redis-cloudflare-headless.dns-proxy.svc.cluster.local", 26379)
    #   ],
    # socket_timeout=0.1)

    from redis.sentinel import Sentinel

    sentinel = Sentinel([
        ('redis-cloudflare', 26379),
    ],
                        sentinel_kwargs={'password': '******'},
                        password='******')

    sentinel.discover_master('mymaster')

    client = sentinel.master_for('mymaster', socket_timeout=0.5)
    for i in range(1000000):
        key = 'key' + str(i)
        # value='value'+str(i)
        value = str(
            i
        ) + '::: https://www.google.co.uk/#sclient=psy-ab&hl=en&source=hp&q=ASUSTeK+Computer+INC.+Model+M4A78LT-M+manual&pbx=1&oq=ASUSTeK+Computer+INC.+Model+M4A78LT-M+manual&aq=f&aqi=&aql=&gs_sm=3&gs_upl=52765l57528l0l57848l8l8l0l0l0l0l2413l3989l8-1.1l2l0&bav=on.2,or.r_gc.r_pw.,cf.osb&fp=3d6c1d1d0a5ea45f&biw=1262&bih=879'
        client.set(key, value)
        print(key, value)
Esempio n. 5
0
    def connect(self, index=0, write=True):
        """
        Creates a redis connection with connection pool.
        """
        master_name, sentinel_hosts, db = self.parse_connection_string(self._connection_string)

        sentinel_timeout = self._options.get('SENTINEL_TIMEOUT', 1)
        sentinel = Sentinel(sentinel_hosts, socket_timeout=sentinel_timeout)

        if write:
            host, port = sentinel.discover_master(master_name)
        else:
            host, port = random.choice([sentinel.discover_master(master_name)] + sentinel.discover_slaves(master_name))

        kwargs = {
            "db": db,
            "parser_class": self.parser_class,
            "password": self._options.get('PASSWORD', None),
        }

        kwargs.update({'host': host, 'port': port, 'connection_class': Connection})

        if 'SOCKET_TIMEOUT' in self._options:
            kwargs.update({'socket_timeout': int(self._options['SOCKET_TIMEOUT'])})

        kwargs.update(self._pool_cls_kwargs)

        connection_pool = get_or_create_connection_pool(self._pool_cls, **kwargs)
        connection = Redis(connection_pool=connection_pool)
        return connection
Esempio n. 6
0
def redis_populate():
    """Function to populate keys in Redis Server"""
    # client = redis.StrictRedis(host=configs["redis_host"], port=configs["redis_port"])
    # sentinel = Sentinel(
    #   [
    #     ("redis-cloudflare-node-0.redis-cloudflare-headless.dns-proxy.svc.cluster.local", 26379),
    #     ("redis-cloudflare-node-1.redis-cloudflare-headless.dns-proxy.svc.cluster.local", 26379),
    #     ("redis-cloudflare-node-2.redis-cloudflare-headless.dns-proxy.svc.cluster.local", 26379)
    #   ],
    # socket_timeout=0.1)

    from redis.sentinel import Sentinel

    sentinel = Sentinel([
        ('redis-cloudflare', 26379),
    ],
                        sentinel_kwargs={'password': '******'},
                        password='******')

    sentinel.discover_master('mymaster')

    client = sentinel.master_for('mymaster', socket_timeout=0.5)
    i = 93218
    key = 'key' + str(i)
    value = 'value' + str(i)
    result = client.get(key)
    print(result)
Esempio n. 7
0
def healthz():
    sentinel = Sentinel([('redis-0.redis', 26379)], socket_timeout=0.1)
    try:
        sentinel.discover_master("redis")
        return "ok"
    except MasterNotFoundError:
        logger.info("Redis master not yet ready")
        raise
    def save_to_redis(self):
        '''
        Save JSON dump into Redis storage
        as covid_data_ru and covid_data_en objects
        '''

        status = False
        data = {}

        for language in ('ru', 'en'):
            data[language] = self.create_geojson(language)

        try:

            logger.info('Connecting to Redis Sentinel')

            sentinel_client = Sentinel(
                [('redis-sentinel', 26379)], socket_timeout=0.1)

            logger.info('Discovering Redis Master')
            sentinel_client.discover_master(REDIS_MASTER)
            master = sentinel_client.master_for(
                REDIS_MASTER, socket_timeout=0.1)

            logger.info('Discovering Redis Slave')
            slave = sentinel_client.slave_for(REDIS_MASTER, socket_timeout=0.1)

            for language in ('ru', 'en'):

                logger.info(f'Saving {language.upper()} covid_data into Slave')
                status = slave.set('covid_data_' + language,
                                   json.dumps(data[language], ensure_ascii=False))

                if not status:
                    return False

            return status

        except:
            logger.warning(
                'Can not proceed with Redis. Saving JSON as a file.')

            try:
                for language in ('ru', 'en'):
                    with open(language + '.json', 'w') as outfile:
                        logger.info(f'Saving {language.upper()} as {language}.json')
                        json.dump(data[language], outfile, ensure_ascii=False)
            except:
                logger.warning('Can not save data as a file.')

            return False
Esempio n. 9
0
    def connect(self, index=0, write=True):
        """
        Creates a redis connection with connection pool.
        """
        master_name, sentinel_hosts, db = self.parse_connection_string(self._connection_string)

        sentinel_timeout = self._options.get("SENTINEL_TIMEOUT", 1)
        sentinel = Sentinel(sentinel_hosts, socket_timeout=sentinel_timeout)

        if write:
            host, port = sentinel.discover_master(master_name)
        else:
            host, port = random.choice([sentinel.discover_master(master_name)] + sentinel.discover_slaves(master_name))

        return self.connection_factory.connect(host, port, db)
Esempio n. 10
0
def get_redis_master(redis_sentinel_nodes):
    nodes_list = redis_sentinel_nodes.split(',')
    sentinel = Sentinel([tuple(nodes_list[0].split(':'))], socket_timeout=0.1)
    master = sentinel.discover_master('mymaster')
    if not master:
        sys.exit(1)
    return list(master)
Esempio n. 11
0
class RedisCluster(object):
    def __init__(self, sentinel_hosts, service_name, **connection_kwargs):
        self.service_name = service_name
        self.sentinel = Sentinel(sentinel_hosts, **connection_kwargs)
        self.master = None
        self.master_host_port = None
        self.reconfigure()
        self.sentinel_threads = {}
        self.listen()

    def reconfigure(self):
        try:
            self.master_host_port = self.sentinel.discover_master(self.service_name)
            self.master = self.sentinel.master_for(self.service_name)
            log.info(f"Reconfigured master to {self.master_host_port}")
        except Exception as ex:
            log.error(f"Error while reconfiguring. {ex.args[0]}")

    def listen(self):
        def on_new_master(workerThread):
            self.reconfigure()

        for sentinel in self.sentinel.sentinels:
            sentinel_host = sentinel.connection_pool.connection_kwargs["host"]
            self.sentinel_threads[sentinel_host] = MyPubSubWorkerThread(
                sentinel, on_new_master, msg_sleep_time=0.001, daemon=True
            )
            self.sentinel_threads[sentinel_host].start()
Esempio n. 12
0
    def add_queues_length_stats(self):
        if len(self.queues) == 0:
            return
        sentinel_list = [(node.get('host', 'localhost'),
                          node.get('sentinel_port', 26379))
                         for node in self.node_list]
        sentinel = Sentinel(sentinel_list, socket_timeout=5)
        master_host = ''
        queues_length_map = {}
        for queue in self.queues:
            queues_length_map[queue] = -1
        try:
            master_host, master_port = sentinel.discover_master(
                self.master_name)
            master_conn = sentinel.master_for(self.master_name,
                                              db=self.db,
                                              password=self.password)
            for queue in self.queues:
                try:
                    queues_length_map[queue] = master_conn.llen(queue)
                except Exception:
                    continue
        except Exception:
            pass

        for queue in self.queues:
            self.add_gauge_value('Redis_Queues/%s' % queue,
                                 None,
                                 queues_length_map[queue],
                                 count=1)
Esempio n. 13
0
 def __init__(self,
              host=None,
              port=6379,
              db=0,
              password=None,
              sentinels=None,
              service_name=None,
              decode_responses=True,
              socket_timeout=None,
              log=None,
              *args,
              **kwargs):
     if sentinels:
         sentinel = Sentinel(sentinels=sentinels)
         master = sentinel.discover_master(service_name=service_name)
         host, port = master
     self.server = {
         'host': host,
         'port': port,
         'db': db,
         'password': password,
         'decode_responses': decode_responses,
         'socket_timeout': socket_timeout
     }
     self.log = log if log else SimpleLog()
     self.pool = ConnectionPool(*args, **self.server, **kwargs)
     Redis.__init__(self, connection_pool=self.pool)
     # Redis.__init__(self, *args, **self.server, **kwargs)
     self.cache_result = {}
Esempio n. 14
0
def getSentinelConnection(sentinelHosts):
    sentinel = Sentinel(sentinelHosts, socket_timeout=0.1)
    master = sentinel.discover_master('mymaster')
    r = redis.StrictRedis(master[0],
                          port=int(master[1]),
                          db=int(config["redisDb"]))
    return r
Esempio n. 15
0
    def __get_master(self, update=False):
        master_ip = self.node.get_ip()
        master_port = self.node.get_port()
        master_node_id = self.node.get_node_id()

        hosts = self.__get_hosts(refresh=update)

        if len(hosts) > 0:
            master_ip = hosts[0]['ip']
            master_node_id = hosts[0]['node_id']
        
        if os.path.isfile(Constants.MASTER_FILE):
            fd = open(Constants.MASTER_FILE, "r")
            temp_ip, temp_port = fd.readline().split(' ')
            fd.close()
            if temp_ip in hosts:
                master_ip, master_port = temp_ip, temp_port
            else:
                os.remove(Constants.MASTER_FILE)

        for host in hosts:
            sentinel = Sentinel([(host["ip"], Constants.SENTINEL_PORT)], socket_timeout=0.1)
            try:
                ip, port = sentinel.discover_master(Constants.REDIS_SENTINEL_NAME)
                for ht in hosts:
                    if ip == ht['ip']:
                        master_ip, master_port = ip, port
                        break
            except:
                pass
        return master_ip, master_port
Esempio n. 16
0
def check_sso_auth(user_id, brand_id, trans_type):
    sentinel = Sentinel([('120.76.154.208', 26379)], socket_timeout=0.5)
    log.d(sentinel.discover_master('mymaster'))
    redis_obj = sentinel.master_for('mymaster', socket_timeout=0.1)

    db_obj = PostgreSQLConnector()
    sso_cls = SSOAuth(redis_obj, db_obj)
    return sso_cls.check_auth(user_id, brand_id, trans_type)
Esempio n. 17
0
def redis_connect(sentinel_addr, sentinel_port):

    sentinel = Sentinel([(sentinel_addr, sentinel_port)], socket_timeout=0.1)
    master_ip = sentinel.discover_master('mymaster')[0]
    master_port = sentinel.discover_master('mymaster')[1]

    print("Found master on {}:{} ".format(master_ip, master_port))

    slaves = sentinel.discover_slaves('mymaster')

    print("Found slaves on:")

    for slave in slaves:
        print("     {}:{}".format(slave[0], slave[1]))

    conn = redis.StrictRedis(host=master_ip, port=master_port, db=0)
    return conn
Esempio n. 18
0
def getRedisClient():
    ipaddrs = config['redis']['redis_ipaddrs']
    arr = ipaddrs.split(",")
    arr1 = []
    for s in arr:
        ipaddr = s.split(":")
        t = (ipaddr[0], int(ipaddr[1]))
        arr1.append(t)
    sentinel = Sentinel(arr1,
                        socket_timeout=config['redis']['redis_socket_timeout'],
                        password=config['redis']['redis_password'])
    master_name = config['redis']['redis_master_name']
    sentinel.discover_master(master_name)
    sentinel.discover_slaves(master_name)
    master = sentinel.master_for(
        master_name, socket_timeout=config['redis']['redis_socket_timeout'])
    return master
Esempio n. 19
0
def check_sso_auth(user_id, brand_id, trans_type ):
    sentinel = Sentinel([('120.76.154.208', 26379)], socket_timeout=0.5)
    log.d(sentinel.discover_master('mymaster'))
    redis_obj = sentinel.master_for('mymaster', socket_timeout=0.1)

    db_obj = PostgreSQLConnector()
    sso_cls = SSOAuth(redis_obj,db_obj)
    return sso_cls.check_auth(user_id, brand_id, trans_type)
Esempio n. 20
0
def sso_log(user_id, brand_id, trans_type, data_type,data_content,log_time):
    sentinel = Sentinel([('112.74.198.224', 26379)], socket_timeout=0.5)
    log.d(sentinel.discover_master('mymaster'))
    redis_obj = sentinel.master_for('mymaster', socket_timeout=0.1)

    db_obj = PostgreSQLConnector()
    sso_cls = SSOAuth(redis_obj,db_obj)
    return sso_cls.send_log(user_id, brand_id, trans_type, data_type,data_content,log_time)
def discover_master(s: Sentinel):
    master = None
    try:
        master = s.discover_master(redis_master_name)
    except Exception as Err:
        pass

    return master or "No master found."
Esempio n. 22
0
def sso_log(user_id, brand_id, trans_type, data_type, data_content, log_time):
    sentinel = Sentinel([('112.74.198.224', 26379)], socket_timeout=0.5)
    log.d(sentinel.discover_master('mymaster'))
    redis_obj = sentinel.master_for('mymaster', socket_timeout=0.1)

    db_obj = PostgreSQLConnector()
    sso_cls = SSOAuth(redis_obj, db_obj)
    return sso_cls.send_log(user_id, brand_id, trans_type, data_type,
                            data_content, log_time)
Esempio n. 23
0
    def connect(self, index=0, write=True):
        """
        Creates a redis connection with connection pool.
        """
        master_name, sentinel_hosts, db = self.parse_connection_string(
            self._connection_string)

        sentinel_timeout = self._options.get('SENTINEL_TIMEOUT', 1)
        sentinel = Sentinel(sentinel_hosts, socket_timeout=sentinel_timeout)

        if write:
            host, port = sentinel.discover_master(master_name)
        else:
            host, port = random.choice(
                [sentinel.discover_master(master_name)] +
                sentinel.discover_slaves(master_name))

        return self.connection_factory.connect(host, port, db)
Esempio n. 24
0
def get_redis_url():
    if hasattr(config, 'ETCD_HOST'):
        return config.HERA_REDIS_SENTINEL
    sentinel_list = [tuple(i.split(':')) for i in get_etcd_setting('HERA_REDIS_SENTINEL').split(',')]
    redis_db = get_etcd_setting('HERA_REDIS_DB')
    sentinel = Sentinel(sentinel_list, socket_timeout=0.1)
    master_name = get_etcd_setting('HERA_REDIS_MASTERNAME')
    return 'redis://{}/{}'.format(
        ':'.join(map(str, sentinel.discover_master(master_name))),
        redis_db
    )
Esempio n. 25
0
    def new_new(_, cls, broker_url, *args, **kwargs):
        scheme = urlparse(broker_url).scheme
        if scheme == "redis-sentinel":
            from rpaas.tasks import app

            opts = app.conf.BROKER_TRANSPORT_OPTIONS
            s = Sentinel(opts["sentinels"], password=opts["password"])
            host, port = s.discover_master(opts["service_name"])
            return RedisBroker("redis://:{}@{}:{}".format(opts["password"], host, port))
        else:
            old_new(cls, broker_url, *args, **kwargs)
Esempio n. 26
0
    def _redis(self):
        if getattr(settings, 'EXPERIMENTS_REDIS_SENTINELS', None):
            sentinel = Sentinel(settings.EXPERIMENTS_REDIS_SENTINELS, socket_timeout=settings.EXPERIMENTS_REDIS_SENTINELS_TIMEOUT)
            host, port = sentinel.discover_master(settings.EXPERIMENTS_REDIS_MASTER_NAME)
        else:
            host = getattr(settings, 'EXPERIMENTS_REDIS_HOST', 'localhost')
            port = getattr(settings, 'EXPERIMENTS_REDIS_PORT', 6379)

        password = getattr(settings, 'EXPERIMENTS_REDIS_PASSWORD', None)
        db = getattr(settings, 'EXPERIMENTS_REDIS_DB', 0)

        return redis.Redis(host=host, port=port, password=password, db=db)
Esempio n. 27
0
def init_redis():
    sentinel_list = [("127.0.0.1",26379),("127.0.0.1",26380),("127.0.0.1",23681)]
    sentinel = Sentinel(sentinel_list,socket_timeout=0.5)

    master = sentinel.discover_master("mymaster")
    print(master)

    slave = sentinel.discover_slaves("mymaster")
    print(slave)

    redis_conn = sentinel.master_for('mymaster', socket_timeout=0.5)
    return redis_conn
Esempio n. 28
0
    def _redis(self):
        if getattr(settings, 'EXPERIMENTS_REDIS_SENTINELS', None):
            sentinel = Sentinel(settings.EXPERIMENTS_REDIS_SENTINELS, socket_timeout=settings.EXPERIMENTS_REDIS_SENTINELS_TIMEOUT)
            host, port = sentinel.discover_master(settings.EXPERIMENTS_REDIS_MASTER_NAME)
        else:
            host = getattr(settings, 'EXPERIMENTS_REDIS_HOST', 'localhost')
            port = getattr(settings, 'EXPERIMENTS_REDIS_PORT', 6379)

        password = getattr(settings, 'EXPERIMENTS_REDIS_PASSWORD', None)
        db = getattr(settings, 'EXPERIMENTS_REDIS_DB', 0)

        return redis.Redis(host=host, port=port, password=password, db=db)
Esempio n. 29
0
class redisSentinelHelper():
    def __init__(self,sentinel_list,service_name,db):
        self.sentinel = Sentinel(sentinel_list,socket_timeout=0.5)
        self.service_name = service_name
        self.db = db

    def get_master_redis(self):
        return self.sentinel.discover_master(self.service_name)

    def get_slave_redis(self):
        return self.sentinel.discover_slaves(self.service_name)

    def set_key(self,key,value):
        master = self.sentinel.master_for(
            service_name=self.service_name,
            socket_timeout=0.5,
            db=self.db
        )
        return master.set(key,value)


    def set_hash_key(self,hkey,key,value):
        master = self.sentinel.master_for(
            service_name=self.service_name,
            socket_timeout=0.5,
            db=self.db
        )
        return master.hset(hkey,key,value)

    def get_key(self,key):
        slave = self.sentinel.slave_for(
            service_name=self.service_name,
            socket_timeout=0.5,
            db=self.db
        )
        return slave.get(key)

    def get_hash_key(self,hkey,hey):
        slave = self.sentinel.slave_for(
            service_name=self.service_name,
            socket_timeout=0.5,
            db=self.db
        )
        return slave.hget(hkey,hey)

    def delete(self,key):
        master = self.sentinel.master_for(
            service_name=self.service_name,
            socket_timeout=0.5,
            db=self.db
        )
        return master.delete(key)
Esempio n. 30
0
    def connect(self, index=0, write=True):
        """
        Creates a redis connection with connection pool.
        """
        master_name, sentinel_hosts, db = self.parse_connection_string(
            self._connection_string)

        sentinel_timeout = self._options.get('SENTINEL_TIMEOUT', 1)
        sentinel = Sentinel(sentinel_hosts, socket_timeout=sentinel_timeout)

        if write:
            host, port = sentinel.discover_master(master_name)
        else:
            host, port = random.choice(
                [sentinel.discover_master(master_name)] +
                sentinel.discover_slaves(master_name))

        kwargs = {
            "db": db,
            "parser_class": self.parser_class,
            "password": self._options.get('PASSWORD', None),
        }

        kwargs.update({
            'host': host,
            'port': port,
            'connection_class': Connection
        })

        if 'SOCKET_TIMEOUT' in self._options:
            kwargs.update(
                {'socket_timeout': int(self._options['SOCKET_TIMEOUT'])})

        kwargs.update(self._pool_cls_kwargs)

        connection_pool = get_or_create_connection_pool(
            self._pool_cls, **kwargs)
        connection = Redis(connection_pool=connection_pool)
        return connection
Esempio n. 31
0
def main(system, logger):
    """
    Returns the IP address of the current Redis Master
    """

    sentinel = Sentinel(
        [(config.SENTINEL_IP, config.PORT)], socket_timeout=0.1)
    master = sentinel.discover_master(config.REDIS_MASTER)

    if master[0] == config.REDIS_MASTER_IP:
        return int(1)
    else:
        return int(0)
Esempio n. 32
0
    def connect(self, index=0, write=True):
        """
        Creates a redis connection with connection pool.
        """
        master_name, sentinel_hosts, db = self.parse_connection_string(self._connection_string)

        sentinel_timeout = self._options.get('SENTINEL_TIMEOUT', 1)
        sentinel = Sentinel(sentinel_hosts, socket_timeout=sentinel_timeout)

        if write:
            host, port = sentinel.discover_master(master_name)
        else:
            host, port = random.choice([sentinel.discover_master(master_name)] + sentinel.discover_slaves(master_name))

        sentinel_options = self._options.copy()
        sentinel_options['CONNECTION_POOL_CLASS'] = 'redis.sentinel.SentinelConnectionPool'
        sentinel_options['CONNECTION_POOL_KWARGS'] = {
            'service_name': master_name,
            'sentinel_manager': sentinel}
        self.connection_factory = get_connection_factory(options=sentinel_options)

        return self.connection_factory.connect(host, port, db)
Esempio n. 33
0
 def new_new(_, cls, broker_url, *args, **kwargs):
     scheme = urlparse(broker_url).scheme
     if scheme == 'redis-sentinel':
         from rpaas.tasks import app
         opts = app.conf.BROKER_TRANSPORT_OPTIONS
         s = Sentinel(
             opts['sentinels'],
             password=opts['password'],
         )
         host, port = s.discover_master(opts['service_name'])
         return RedisBroker('redis://:{}@{}:{}'.format(
             opts['password'], host, port))
     else:
         old_new(cls, broker_url, *args, **kwargs)
Esempio n. 34
0
def fetch_redis_info(conf):
    info = {}

    # 获取redis状态信息(0 dead, 1 master, -1 slave)
    try:
        r = redis.Redis(host=conf['host'],
                        port=conf['port'],
                        password=conf['password'],
                        socket_connect_timeout=5)
        for k, v in r.info().items():
            if k in conf['redis_info'].keys():
                info[k] = v
            elif k.startswith('db'):
                for i in ['keys', 'expires']:
                    info[k + '_' + i] = v[i]
            elif k == 'role':
                if v == 'master':
                    info['redis_alive'] = 1
                else:
                    info['redis_alive'] = -1

        if info['maxmemory'] > 0:
            info['mem_used_ratio'] = round(
                float(info['used_memory']) / float(info['maxmemory']) * 100, 2)
        else:
            info['mem_used_ratio'] = 0

        for k, v in r.info('commandstats').items():
            if k + '_calls' in conf['redis_info'].keys():
                info[k + '_calls'] = v['calls']

    except redis.RedisError as e:
        collectd.error('redis %s:%s connection error!' %
                       (conf['host'], conf['port']))
        info['redis_alive'] = 0

    #  获取sentinel状态信息 (0 dead, 1 leader, -1 leaf)
    try:
        s = Sentinel([(conf['host'], conf['sentinel_port'])],
                     socket_timeout=0.1)
        if conf['host'] == s.discover_master(conf['sentinel_name'])[0]:
            info['sentinel_alive'] = 1
        else:
            info['sentinel_alive'] = -1
    except redis.RedisError as e:
        collectd.error('sentinel %s:%s connection error!' %
                       (conf['host'], conf['sentinel_port']))
        info['sentinel_alive'] = 0

    return info
Esempio n. 35
0
 def __init__(self, sentinels, service, **kwargs):
     sentinel = Sentinel(sentinels, **kwargs)
     try:
         master = sentinel.discover_master(service)
     except MasterNotFoundError:
         raise
     try:
         slaves = sentinel.discover_slaves(service)
     except SlaveNotFoundError:
         self._slaves = None
     if version_info[0] < 3:
         super(AutoRedisSentinel, self).__init__(master, slaves=slaves, **kwargs)
     else:
         super().__init__(master, slaves=slaves, **kwargs)
Esempio n. 36
0
 def __init__(self, sentinels, service, **kwargs):
     sentinel = Sentinel(sentinels, **kwargs)
     try:
         master = sentinel.discover_master(service)
     except MasterNotFoundError:
         raise
     try:
         slaves = sentinel.discover_slaves(service)
     except SlaveNotFoundError:
         self._slaves = None
     if version_info[0] < 3:
         super(AutoRedisSentinel, self).__init__(master, slaves=slaves, **kwargs)
     else:
         super().__init__(master, slaves=slaves, **kwargs)
Esempio n. 37
0
    def run(self, terms, inject=None, **kwargs):
        values = []
        terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject)

        for term in terms:
            params = self.parse_params(term)

            try:
                conn = Sentinel(params['server'], socket_timeout=1)
                res = conn.discover_master(params['name'])
                if res is None:
                    res = ""
                values.append(":".join((str(v) for v in res)))
            except Exception, e:
                print(e)
Esempio n. 38
0
def connect():
    server_1 = '192.168.11.101'
    server_2 = '192.168.11.111'
    server_3 = '192.168.11.112'
    password = '******'

    connection = Sentinel([(server_1, 26379), (server_2, 26379), (server_3, 26379)])
    host, port = connection.discover_master('redis-cluster')
    redis_client = StrictRedis(
        host=host,
        port=port,
        password=password
    )
    print(host)
    print(redis_client.set('city', 'feni'))
Esempio n. 39
0
def get_redis_connection(sentinels=None, host=None, port=None, db_num=0):
    """ Connect to redis using sentinels if defined or directly using given host and port.
        :arg sentinels  list of tuples (host, port)
        :arg host   redis host
        :arg port   redis port
        :arg db_num     redis db number
        :return redis.Redis
        :raise redis.RedisError"""
    if sentinels is not None:
        logging.debug('Asking sentinels for master address')
        sentinel_conn = Sentinel(sentinels, socket_timeout=0.2)
        host, port = sentinel_conn.discover_master(conf.sentinel_master_name)
    logging.debug('Connecting to redis at ' + str(host) +
                  ':' + str(port) + ' db=' + str(db_num))
    return redis.Redis(host=host, port=port, db=db_num)
Esempio n. 40
0
    else:
        return str_to_bool(os_var)


REDIS_MASTER_IP = None
REDIS_MASTER_PORT = None

REDIS_SENTINEL_ENABLED = bool_from_env('REDIS_SENTINEL_ENABLED', False)
if REDIS_SENTINEL_ENABLED:
    from redis.sentinel import Sentinel

    # REDIS_SENTINEL_HOSTS env variable format: host_1:port;host_2:port
    REDIS_SENTINEL_HOSTS = os.environ['REDIS_SENTINEL_HOSTS'].split(';')
    REDIS_CLUSTER_NAME = os.environ['REDIS_CLUSTER_NAME']

    sentinel = Sentinel(
        [tuple(s_host.split(':')) for s_host in REDIS_SENTINEL_HOSTS],
        socket_timeout=float(
            os.environ.get('REDIS_SENTINEL_SOCKET_TIMEOUT', 0.2)
        )
    )
    REDIS_MASTER_IP, REDIS_MASTER_PORT = sentinel.discover_master(
        REDIS_CLUSTER_NAME
    )

REDIS_HOST = REDIS_MASTER_IP or os.environ.get('REDIS_HOST', '127.0.0.1')
REDIS_PORT = REDIS_MASTER_PORT or int(os.environ.get('REDIS_PORT', 6379))
REDIS_DB = int(os.environ.get('REDIS_DB', 0))
REDIS_PASSWORD = os.environ.get('REDIS_PASSWORD', None)
REDIS_SSL = bool_from_env('REDIS_SSL', False)
Esempio n. 41
0
def getSentinelConnection(sentinelHosts):
	sentinel = Sentinel(sentinelHosts, socket_timeout=0.1)
	master = sentinel.discover_master('mymaster')
	r = redis.StrictRedis(master[0], port=int(master[1]), db=int(config["redisDb"]))
	return r
Esempio n. 42
0
###########################################################
# 1.确保 所有哨兵端口都开启

# 2.所有的节点端口开启

# 3.配置文件 bind 不能为127.0.0.1 应该是 0.0.0.0
from redis.sentinel import Sentinel

# 连接哨兵服务器(主机名也可以用域名)
sentinel = Sentinel([('192.168.56.101', 26379)], socket_timeout=0.5)



# 获取主服务器地址
master = sentinel.discover_master('mymaster')
print(master)
# 输出:('172.31.0.2', 5001)


# 获取从服务器地址
slave = sentinel.discover_slaves('mymaster')
print(slave)
# 输出:[('172.31.3', 5001), ('172.31.0.4', 5001), ('172.31.0.5', 5001)]



# 获取主服务器进行写入
master = sentinel.master_for('mymaster', socket_timeout=0.5, db=15)
w_ret = master.set('foo', 'bar')
# 输出:True
#     print 'error starting slave instance.'
#     sys.exit(99)
raw_input('./start-slave.sh')
b = StrictRedis('localhost', 44411)
wait_for_redis(b)

# err = subprocess.call('./start-sentinel.sh')
# if err:
#     print 'error starting sentinel instance.'
#     sys.exit(99)
raw_input('./start-sentinel.sh')
c = StrictRedis('localhost', 55511)
wait_for_redis(c)

sentinel = Sentinel([('localhost', 55511)], socket_timeout=0.1)
if sentinel.discover_master(NAME)[1] != 33311:
    print 'unexpected situation: expecting port 33311 to be the master'
    sys.exit(99)

print """>> Getting an High Availability connection through the sentinel.
"""
mha = sentinel.master_for(NAME)
if int(mha.config_get()['port']) != 33311:
    print 'unexpected master. expecting master to be on port 33311.'
    sys.exit(99)

assert mha.ping() and a.ping()

print """>> Currently: (MASTER:{})<==(SLAVES:{})""".format(sentinel.discover_master(NAME),
                                                           sentinel.discover_slaves(NAME))
class agent(object): 
    def __init__(self):
        pass

    #by default,load the agent cnfig file
    def load_config(self,config_file='./conf/agent.yml'):
        self.config_file=config_file
        with open(self.config_file,'r') as fd: 
            self.config = yaml.load(fd)
        return  self.config
    
    def logger(self,path):
    	self.logger=logging.getLogger()
    	self.logger.setLevel(logging.WARNING)
    	fh=logging.FileHandler(path)
    	formatter = logging.Formatter("%(asctime)-15s %(filename)s [%(levelname)-8s] %(message)s")
    	fh.setFormatter(formatter)
    	self.logger.addHandler(fh)
    	return self.logger

    #load the twemproxy config file by agent_conf_sn we specified 
    def  load_twem_config(self,conf_sn):
         self.config=self.load_config()
         self.twem_config_file=self.config['agents'][conf_sn]['twem_config']
         self.twem_config=self.load_config(self.twem_config_file)
         return self.twem_config
     
    #restart the twemproxy 
    def twem_restart(self,conf_sn):  # conf_sn(config file session name) is 'cli1','cli2' etc. we can regard him as  primary key in sql. 
        self.load_config()
        cmd=self.config['agents'][conf_sn]['twem_cmd']
        rs,rt = commands.getstatusoutput(cmd)  # 'rs' means result, 'rt' mean return
        self.logger.warning("%s"%rt)
        return  rt  

    def update_twem_master(self,conf_sn,sv_alias,new_ip,new_port):
        self.load_twem_config(conf_sn)    
        self.twem_servers=self.twem_config['twem1']['servers']
        self.logger.warning("NO,%s:%s twem_master != sentinel_master,Begin update twemproxy "%(conf_sn,sv_alias))

        for i in range(len(self.twem_servers)):
            if ( self.twem_servers[i].split(" ")[1] == sv_alias ):
                self.logger.warning("twem old master:%s"%self.twem_servers[i].split(" ")[0].split(":"))
                #print "[",datetime.now(),"]","old_master :",self.twem_servers[i].split(" ")[0].split(":")
                weight=self.twem_servers[i].split(" ")[0].split(":")[2]
                new_addr=new_ip+':'+str(new_port)+':'+weight+' '+sv_alias
                self.logger.warning("twem new master:%s"%new_addr)
	        #print "[",datetime.now(),"]","new_master",new_addr 
                self.twem_servers[i]=new_addr
    
        with open(self.twem_config_file,'w') as stream:
            yaml.dump(self.twem_config,stream,allow_unicode=True,default_flow_style=False)

        self.twem_restart(conf_sn)

    #return the twemproxy's master addr (IP:PORT)
    def get_twem_master(self,conf_sn,sv_alias):
        self.load_twem_config(conf_sn)
        self.twem_servers=self.twem_config['twem1']['servers']
        for i in range(len(self.twem_servers)):
            if( self.twem_servers[i].split(" ")[1] == sv_alias ):
                self.twem_master=self.twem_servers[i].split(" ")[0].split(':')[0]+':'+self.twem_servers[i].split(" ")[0].split(':')[1]
        #print 'twemproxy_master:',self.twem_master
        self.logger.info("get twemproxy %s:%s master:%s"%(conf_sn,sv_alias,self.twem_master))
        return self.twem_master


    #retun the sentinel_master addr 
    def get_sentinel_master(self,conf_sn,sv_alias):
        stats=1
        self.load_config()
        self.sentinel_host=self.config['agents'][conf_sn]['sentinel_host']
        self.sentinel_port=self.config['agents'][conf_sn]['sentinel_port']
        try:
            self.sentinel=Sentinel([(self.sentinel_host,self.sentinel_port)],socket_timeout=1)
        except :
            self.logger.error('connection to Sentinel %s:%s failed '%(self.sentinel_host,self.sentinel_port))

        while stats:
            try:
                self.sentinel_master=self.sentinel.discover_master(sv_alias)
                stats=0
            except:
                time.sleep(0.5)
                self.logger.error('get %s sentinel %s:%s failed %d times'%(conf_sn,self.sentinel_host,self.sentinel_port,stats))
                stats = stats + 1
		while (stats == 5):
                    stats=0
		    self.logger.error('retry failed,skip this sentinel %s:%s,please check it'%(self.sentinel_host,self.sentinel_port))
        
        self.sentinel_master=self.sentinel_master[0]+':'+str(self.sentinel_master[1])
        if (len(self.sentinel_master)<7):
            return False
            self.logger.error("get sentinel %s,%s master Failed"%(conf_sn,sv_alias))
        else:
            self.logger.info("get sentinel %s,%s master:%s"%(conf_sn,sv_alias,self.sentinel_master))
            return self.sentinel_master
  
    def __del__(self):
        pass
signal.signal(signal.SIGINT,signal_handler)

try:
    parser = OptionParser()
    parser.add_option("-n",dest='numofkeys')
    (options,args) = parser.parse_args()

    if len(sys.argv) == 1:
        print('please suppply sentinel ipaddrs')
        sys.exit(1)

    sentinel = Sentinel(
        [(arg.split(':')[0],arg.split(':')[1]) for arg in args[1:]],
        socket_timeout = 0.5
    )
    print('detected master is %s' % str(sentinel.discover_master('mymaster')))
    time.sleep(5)
    master = sentinel.master_for('mymaster',socket_timeout = 0.1)
    #master = redis.StrictRedis(host='172.17.0.7') 
except:
    print(sys.exc_info()[0])
    sys.exit(1)

print("key insertion to commence in 3s ...")
time.sleep(3)

value = datetime.datetime.now()
key,value = keygen.generate_kv_pair()

num = int(options.numofkeys) if options.numofkeys else -1
count = 0
Esempio n. 46
0
import time
from redis.sentinel import Sentinel

sentinel = Sentinel([('192.168.11.35', 26379),('192.168.11.37', 26379),('192.168.11.37', 26379)], socket_timeout=0.1)
for i in range(1,1000):
  print "ciclo " + str(i)
  try:
    print "master: " + str(sentinel.discover_master('mymaster'))
    print "slave/s: " + str(sentinel.discover_slaves('mymaster'))

    master = sentinel.master_for('mymaster', socket_timeout=0.1)
    slave = sentinel.slave_for('mymaster', socket_timeout=0.1)
  except:
    print "sentinel eligiendo a un master"

  clave = "clave" + str(i)
  valor = "valor" + str(i)

  try:
    print "master.set " + clave + " " + valor 
    master.set(clave, valor)

    print "master.get " + clave + " = " + str(master.get(clave))

    print "master.set " + clave + " = " + str(slave.get(clave))

    print "master.delete " + clave
    master.delete(clave)
  except:
    print "cluster en modo solo lectura"
#!/usr/bin/python

from redis.sentinel import Sentinel
sentinel = Sentinel([('localhost', 19001)], socket_timeout=0.1)
master = sentinel.discover_master('redis-cluster')
slaves = sentinel.discover_slaves('redis-cluster')

print ""
print "Master(" + str(master) + ")"
print ""
print "Slaves(" + str(len(slaves)) + ") Nodes(" + str(slaves) + ")"
print ""
Esempio n. 48
0
class Database:
    """
    Provide an interface using which to communicate with a Redis server.

    A new connection to the server is created by each class instance.

    Custom exception hierarchy:
    Error
     +-- ConnectionError
     +-- WaitTimeoutError
    """

    socket_timeout = 1
    pubsubs = {}
    sentinel = None

    class Error(Exception):
        pass
        # print "Error"

    class ConnectionError(Error):
        pass
        # print "ConnectionError"

    class WaitTimeoutError(Error):
        def __str__(self):
            return self.msg
        def __init__(self, msg, key, value, timeout):

            msg = str(msg) # in the event msg was an Exception, for example
            self.msg = msg
            self.key = key
            self.value = value
            self.timeout = timeout

            super(RedisClient.WaitTimeoutError, self).__init__(self, msg) # this raises the exception
            # Note: In Python 2.x, self.__class__ should not be used in place of base class name.

    def state_connection(self):
        """
        Return a dict containing keys and values providing information about
        the state of the connection to the Redis server.
        """
        if self.logger:
            self.logger.debug("state_connection")

        # conn_kwargs = self._pool.connection_kwargs
        # server_addr = (conn_kwargs["host"], conn_kwargs["port"])

        # state = {"Server address": "{}:{}".format(*server_addr),}
        state = {}

        # Do ping test
        try:
            # connection = redis.Redis(connection_pool=self._pool)
            state["Server pingable"] = self.redis.ping() # returns True
        except redis.exceptions.ConnectionError: # as e:
            #self.logger.error("{} Console Redis ping failure: {}".format(tools.cm_name(), e))
                # intentionally not self.logger.exception
            state["Server pingable"] = False

        # # Test availability of a key
        # try:
        #     self["MONO_SV"]
        # except self.ConnectionError: # as e:
        #     #self.logger.error("{} Console Redis test failure: {}".format(tools.cm_name(), e))
        #         # intentionally not self.logger.exception
        #     state["Test result"] = "Failed"
        # else:
        #     state["Test result"] = "Passed"

        return state

    def state_server(self):
        """Return a dict containing keys and values providing some properties
        about the the Redis server."""
        if self.logger:
            self.logger.debug("state_server")

        try:
            # connection = redis.Redis(connection_pool=self._pool)
            info = self.redis.info()
        except redis.exceptions.ConnectionError as e:
            if self.logger:
                self.logger.error("Redis info acquisition failure: {}".format(e))
            # intentionally not self.logger.exception
            state = {}
        else:
            state = {"Number of connected clients": info["connected_clients"],
                     "Redis version": info["redis_version"],
                     "Uptime": datetime.datetime.utcnow() - datetime.datetime.utcfromtimestamp(time.time() - info["uptime_in_seconds"]),
                     }

        return state

    def __init__(self,
                 host="localhost",
                 port=6379,
                 db=0, 
                 password=None, 
                 sentinels=None,
                 master=None,
                 settings=None,
                 logger=False):
        """Initialize the client."""

        # Save passed-in variables
        self.host =      host
        self.port =      port
        self.db =        db
        self.password =  password 
        self.sentinels = sentinels
        self.master =    master
        self.logger =    logger

        # Use the RAPD CONTROL_DATABASE_SETTINGS object
        if settings:
            # Sentinel
            if settings["REDIS_CONNECTION"] == "sentinel":
                self.host =      None
                self.port =      None
                self.db =        None
                self.password =  settings.get("REDIS_PASSWORD", None) 
                self.sentinels = settings["REDIS_SENTINEL_HOSTS"] 
                self.master =    settings["REDIS_MASTER_NAME"] 
            # Standard
            else:
                self.host =      settings["REDIS_HOST"]
                self.port =      settings["REDIS_PORT"]
                self.db =        settings["REDIS_DB"]
                self.password =  settings.get("REDIS_PASSWORD", None) 
                self.sentinels = None
                self.master =    None

        self._ConnectionError_last_log_time = float("-inf")

        # Connect to server
        self._connect_lock = threading.Lock()
        self._connect() # "with self._connect_lock:" is not necessary in __init__
        atexit.register(self._disconnect)

    ###############
    # ADMIN Methods
    ###############

    def _connect(self):
        """Connect to server."""

        # self.logger.debug(tools.cm_name())

        socket_timeout = self.socket_timeout

        # Sentinel connection
        if self.sentinels:
            self.sentinel = Sentinel(self.sentinels)

            # Assign master connection to self.redis
            self.redis = self.sentinel.master_for(self.master)

        # Standard connection
        else:
            attempt_count = 0
            while attempt_count < CONNECTION_ATTEMPT_LIMIT:
                try:
                    # Connect
                    self.redis = redis.Redis(host=self.host,
                                             port=self.port,
                                             db=self.db,
                                             password=self.password)
                    # self._pool = redis.ConnectionPool(host=self.host, port=self.port, db=0)
                    
                    # Make sure redis server is up
                    _ = self.redis.ping()
                    break
                    # self._connection = redis.Redis(host, socket_timeout=socket_timeout)
                    # redis module reconnects automatically as needed.
                    # socket_timeout is not set to a higher value because it blocks.
                except redis.exceptions.ConnectionError as e:
                    print "except"
                    if self.logger:
                        self.logger.debug("Connection error {}".format(e))
                    time.sleep(1)
                    attempt_count += 1
                    # self._raise_ConnectionError(e)

        # # Make sure redis server is up
        # try:
        #     _ = self.redis.ping()
        # except redis.sentinel.MasterNotFoundError as error:
        #     self._raise_ConnectionError(error)

    def _disconnect(self):
        """Disconnect from server."""

        # self.logger.debug(tools.cm_name())

        try:
            pool = self._pool
        except AttributeError:
            pass
        else:
            pool.disconnect()
            #self.logger.info("{} Disconnected Redis client from Redis server {host}:{port}.".format(tools.cm_name(), **connection.connection_pool.connection_kwargs))

    def _raise_ConnectionError(self, exc):
        """
        Raise ConnectionError exception with an error message corresponding
        to the provided exception.
        """

        # TODO: Implement functionality of _raise_ConnectionError directly into ConnectionError exception instead. Delete method definition.

        err_msg = str(exc)
        if err_msg: err_msg = ": {}".format(err_msg)
        err_msg = "Redis connection error{}".format(err_msg)

        # Conditionally log error
        # log_interval = backend.settings.redis["Console"]["ConnectionError log interval"]
        # if time.time() > (self._ConnectionError_last_log_time + log_interval):
        #     self._ConnectionError_last_log_time = time.time()
        #     logger = self.logger.info
        # else:
        if self.logger:
            logger = self.logger.debug
            logger("{}".format(err_msg))

        raise self.ConnectionError(err_msg)

    def discover_master(self):
        """
        Return the master instance (host, ip)
        """

        if self.sentinel and self.master:
            attempts = 0
            while attempts < ATTEMPT_LIMIT:
                try:
                    attempts += 1
                    master = self.sentinel.discover_master(self.master)
                    break
                except redis.sentinel.MasterNotFoundError as e:
                    # Pause for specified time
                    print "try %d" % attempts
                    time.sleep(ATTEMPT_PAUSE)
            else:
                self._raise_ConnectionError(e)

            return master 
        else:
            self._raise_ConnectionError("Sentinels not properly defined")

    def discover_slaves(self):
        """
        Return the slave instances [(host, ip),]
        """

        if self.sentinel and self.master:
            attempts = 0
            while attempts < ATTEMPT_LIMIT:
                try:
                    attempts += 1
                    master = self.sentinel.discover_slaves(self.master)
                    break
                except redis.sentinel.SlaveNotFoundError as e:
                    # Pause for specified time
                    print "try %d" % attempts
                    time.sleep(ATTEMPT_PAUSE)
            else:
                self._raise_ConnectionError(e)

            return master 
        else:
            self._raise_ConnectionError("Sentinels not properly defined")
    
    #############
    # PIPELINE Methods
    #############
    
    def pipeline(self):
        """return a pipeline instance"""
        return self.redis.pipeline()

    #############
    # GET Methods
    #############

    def __getitem__(self, key, return_dict=False):
        """
        Return the value of the specified key(s).

        key can be either a str, or a tuple or list of strs. It is recommended
        that key(s) be present in backend.redis_console_keys.keys.

        `transform` indicates whether to apply transformations to the value
        before returning it, as defined in the self._transform_value method.

        `return_dict` indicates whether to return values in a dict. If True,
        the keys in the returned dict match the specified key(s).
        """

        # Get value(s)
        if isinstance(key, str):
            return self.get(key, return_dict)
        elif isinstance(key, tuple) or isinstance(key, list):
            return self.mget(key, return_dict)
    get = __getitem__

    @connectionErrorWrapper
    def get(self, key, return_dict=False):
        """
        Return the value of `key`.

        `return_dict` indicates whether to return values in a dict. If True,
        the keys in the returned dict match the specified key(s).
        """

        # self.logger.debug("get key:{} return_dict:{}".format(key, return_dict))

        # Retrieve value
        value = self.redis.get(key)
        
        # Wrap in dict if requested
        return ({key: value} if return_dict else value)

    # def get(self, key, return_dict=False):
    #     """
    #     Return the value of `key`.

    #     `return_dict` indicates whether to return values in a dict. If True,
    #     the keys in the returned dict match the specified key(s).
    #     """

    #     # self.logger.debug("get key:{} return_dict:{}".format(key, return_dict))

    #     # Retrieve value
    #     attempts = 0
    #     while attempts < ATTEMPT_LIMIT:
    #         try:
    #             attempts += 1
    #             value = self.redis.get(key)
    #             break
    #         except redis.exceptions.ConnectionError as e:
    #             # Pause for specified time
    #             print "try %d" % attempts
    #             time.sleep(ATTEMPT_PAUSE)
    #     else:
    #         self._raise_ConnectionError(e)

    #     return ({key: value} if return_dict else value)

    @connectionErrorWrapper
    def mget(self, keys, return_dict=False):
        """
        Return the values of the sequence `keys`.

        `transform` indicates whether to apply transformations to the value
        before returning it, as defined in the self._transform_value method.

        `return_dict` indicates whether to return values in a dict. If True,
        the keys in the returned dict match the specified key(s).
        """

        # Retrieve values
        values = self.redis.mget(keys)

        return (dict(zip(keys,values)) if return_dict else values)

    # def mget(self, keys, return_dict=False):
    #     """
    #     Return the values of the sequence `keys`.

    #     `transform` indicates whether to apply transformations to the value
    #     before returning it, as defined in the self._transform_value method.

    #     `return_dict` indicates whether to return values in a dict. If True,
    #     the keys in the returned dict match the specified key(s).
    #     """

    #     # self.logger.log(5, "{} keys:{} transform:{} return_dict:{}".format(tools.cm_name(), keys, transform, return_dict))

    #     # Retrieve values
    #     try:
    #         connection = redis.Redis(connection_pool=self._pool)
    #         values = connection.mget(keys)
    #     except redis.exceptions.ConnectionError as e:
    #         self._raise_ConnectionError(e)
    #     # self.logger.log(5 if transform else logging.DEBUG, "{} keys:{} values:{}".format(tools.cm_name(), keys, repr(values)))

    #     # Transform and return values
    #     # if transform: values = self._transform_value(keys, values)
    #     return (dict(zip(keys,values)) if return_dict else values)

    #############
    # SET Methods
    #############

    @connectionErrorWrapper
    def set(self, key, value):
        """
        Set the indicated key value pair.
        """
        if self.logger:
            self.logger.debug("set key:{} value:{}".format(key, value))

        self.redis.set(key, str(value))

    # def set(self, key, value):
    #     """
    #     Set the indicated key value pair.
    #     """

    #     self.logger.debug("set key:{} value:{}".format(key, value))

    #     # Set
    #     attempts = 0
    #     while attempts < ATTEMPT_LIMIT:
    #         try:
    #             attempts += 1
    #             self.redis.set(key, str(value))
    #             break
    #         except redis.exceptions.ConnectionError as error:
    #             # Pause for specified time
    #             print "try %d" % attempts
    #             time.sleep(ATTEMPT_PAUSE)
    #     else:
    #         self._raise_ConnectionError(error)

    __setitem__ = set

    @connectionErrorWrapper
    def mset(self, mapping):
        """
        Set the key value pairs which are in the dict `mapping`.
        """

        # self.logger.debug("{} mapping:{}".format(tools.cm_name(), mapping))

        # Get the mapping ready
        mapping = {k: str(v) for k, v in mapping.items()}

        # Set in redis
        self.redis.mset(mapping)


    # def mset(self, mapping):
    #     """
    #     Set the key value pairs which are in the dict `mapping`.
    #     """

    #     # self.logger.debug("{} mapping:{}".format(tools.cm_name(), mapping))

    #     mapping = {k: str(v) for k, v in mapping.items()}
    #     # Set
    #     attempts = 0
    #     while attempts < ATTEMPT_LIMIT:
    #         try:
    #             attempts += 1
    #             self.redis.mset(mapping)
    #             break
    #         except redis.exceptions.ConnectionError as error:
    #             # Pause for specified time
    #             # print "try %d" % attempts
    #             time.sleep(ATTEMPT_PAUSE)
    #     else:
    #         self._raise_ConnectionError(error)
    @connectionErrorWrapper
    def setex(self, key, expire_time, value):
        """
        Set the indicated key value pair with expire time.
        """
        #if self.logger:
        #    self.logger.debug("set key:{} value:{}".format(key, value))

        self.redis.setex(key, int(expire_time), value)
    
    #############
    # KEY Methods
    #############
    @connectionErrorWrapper
    def keys(self, template):
        """Searches for keys fitting template."""
        #self.logger.debug("set key:{} value:{}".format(key, value))
        return self.redis.keys(template)
    
    @connectionErrorWrapper
    def delete(self, key):
        """Delete a key"""
        #self.logger.debug("set key:{} value:{}".format(key, value))
        return self.redis.delete(key)


    ##############
    # LIST Methods
    ##############
    @connectionErrorWrapper
    def llen(self, key):
        """
        LLEN get length of list
        """
        value = self.redis.llen(key)
        return value

    @connectionErrorWrapper
    def lpop(self, key):
        """
        LPOP a value off a given list
        """
        value = self.redis.lpop(key)
        return value
    
    @connectionErrorWrapper
    def lpush(self, key, value):
        """
        LPUSH a values onto a given list
        """
        self.redis.lpush(key, value)

    @connectionErrorWrapper
    def brpop(self, key):
        """
        RPOP a value off a given list
        """
        value = self.redis.brpop(key)
        return value
    @connectionErrorWrapper
    def rpop(self, key):
        """
        RPOP a value off a given list
        """
        value = self.redis.rpop(key)
        return value
    
    @connectionErrorWrapper
    def rpoplpush(self, list1, list2):
        """
        RPOPLPUSH pop a value off a given list and push on another list
        """
        value = self.redis.rpoplpush(list1, list2)
        #return value
               
    ##############
    # HASH Methods
    ##############
    @connectionErrorWrapper
    def hget(self, key, field):
        """
        HGET field on a key
        """
        return self.redis.hget(key, field)

    @connectionErrorWrapper
    def hgetall(self, key):
        """
        HGETALL a mapping on a key
        """
        return self.redis.hgetall(key)

    @connectionErrorWrapper
    def hmset(self, key, mapping):
        """
        HMSET a key with the attached mapping
        """
        self.redis.hmset(key, mapping)

    @connectionErrorWrapper
    def hset(self, key, field, value):
        """
        Sets field in the hash stored at key to value.
        """
        self.redis.hset(key, field, value)

    ################
    # PUBSUB Methods
    ################
    @connectionErrorWrapper
    def publish(self, key, value):
        """
        Publish a value on a given key
        """

        # print "publish {} {}".format(key, value)
        self.redis.publish(key, value)

    ####################
    # EXPIRATION Methods
    ####################
    @connectionErrorWrapper
    def expire(self, key, seconds):
        """
        Set a timeout on key
        """

        self.redis.expire(key, seconds)



    # def publish(self, key, value):
    #     """
    #     Publish a value on a given key
    #     """

    #     # print "publish {} {}".format(key, value)

    #     attempts = 0
    #     while attempts < ATTEMPT_LIMIT:
    #         try:
    #             attempts += 1
    #             self.redis.publish(key, value)
    #             break
    #         except redis.exceptions.ConnectionError as error:
    #             # Pause for specified time
    #             print "try %d" % attempts
    #             time.sleep(ATTEMPT_PAUSE)
    #     else:
    #         self._raise_ConnectionError(error)

    @connectionErrorWrapper
    def get_message(self, id):
        """
        Get message on pubsub connection
        """

        message = self.pubsubs[id].get_message()

        return message

    # def get_message(self, id):
    #     """
    #     Get message on pubsub connection
    #     """

    #     attempts = 0
    #     while attempts < ATTEMPT_LIMIT:
    #         try:
    #             attempts += 1
    #             message = self.pubsubs[id].get_message()
    #             break
    #         except redis.exceptions.ConnectionError as error:
    #             # Pause for specified time
    #             print "try %d" % attempts
    #             time.sleep(ATTEMPT_PAUSE)
    #     else:
    #         self._raise_ConnectionError(error)

    #     return message

    def get_pubsub(self):
        """
        Returns a pubsub connection - no error is triggered if no connection
        """

        # Create the pubsub object
        ps = self.redis.pubsub()

        # Give it an id
        id = len(self.pubsubs) + 1
        ps._id = id

        # Store a reference
        self.pubsubs[id] = ps

        # Return the id
        return id

    @connectionErrorWrapper
    def psubscribe(self, id=False, pattern=False):
        """
        Add pattern subscription to a pubsub object and return id of pubsub object
        """

        if pattern:

            # No id passed in - create pubsub object
            if not id:
                id = self.get_pubsub()

            self.pubsubs[id].psubscribe(pattern)

            return id

        else:
            raise TypeError("pattern must be specified")

    # def psubscribe(self, id=False, pattern=False):
    #     """
    #     Add pattern subscription to a pubsub object and return id of pubsub object
    #     """

    #     # No id passed in - create pubsub object
    #     if not id:
    #         id = self.get_pubsub()

    #     attempts = 0
    #     while attempts < ATTEMPT_LIMIT:
    #         try:
    #             attempts += 1
    #             self.pubsubs[id].psubscribe(pattern)
    #             break
    #         except redis.exceptions.ConnectionError as error:
    #             # Pause for specified time
    #             # print "try %d" % attempts
    #             time.sleep(ATTEMPT_PAUSE)
    #     else:
    #         self._raise_ConnectionError(error)

    #     return id

    @connectionErrorWrapper
    def subscribe(self, id=None, channel=None):
        """
        Add subscription to a pubsub object and return id of pubsub object
        """

        if channel:

            # No id passed in - create pubsub object
            if not id:
                id = self.get_pubsub()

            self.pubsubs[id].subscribe(channel)

            return id

        else:
            raise TypeError("channel must be specified")