def connect(self, index=0, write=True): """ Creates a redis connection with connection pool. """ master_name, sentinel_hosts, db = self.parse_connection_string(self._connection_string) sentinel_timeout = self._options.get('SENTINEL_TIMEOUT', 1) sentinel = Sentinel(sentinel_hosts, socket_timeout=sentinel_timeout) if write: host, port = sentinel.discover_master(master_name) else: host, port = random.choice([sentinel.discover_master(master_name)] + sentinel.discover_slaves(master_name)) kwargs = { "db": db, "parser_class": self.parser_class, "password": self._options.get('PASSWORD', None), } kwargs.update({'host': host, 'port': port, 'connection_class': Connection}) if 'SOCKET_TIMEOUT' in self._options: kwargs.update({'socket_timeout': int(self._options['SOCKET_TIMEOUT'])}) kwargs.update(self._pool_cls_kwargs) connection_pool = get_or_create_connection_pool(self._pool_cls, **kwargs) connection = Redis(connection_pool=connection_pool) return connection
def connect(self, write=True): """ Create a redis connection with connection pool. """ self.log.debug("connect called: write=%s", write) sentinel_timeout = self._options.get("SENTINEL_TIMEOUT", 1) password = self._options.get("PASSWORD", None) sentinel = Sentinel( self._sentinel_hosts, socket_timeout=sentinel_timeout, password=password ) if write: host, port = sentinel.discover_master(self._master_name) else: try: host, port = random.choice( # nosec sentinel.discover_slaves(self._master_name) ) except IndexError: self.log.debug("no slaves are available. using master for read.") host, port = sentinel.discover_master(self._master_name) if password: connection_url = f"redis://:{password}@{host}:{port}/{self._database_name}" else: connection_url = f"redis://{host}:{port}/{self._database_name}" self.log.debug("Connecting to: %s", connection_url) return self.connection_factory.connect(connection_url)
def discover_slave(s: Sentinel): slave = None try: slave = s.discover_slaves(redis_master_name) except Exception as Err: pass return slave or "No slave found."
def getRedisClient(): ipaddrs = config['redis']['redis_ipaddrs'] arr = ipaddrs.split(",") arr1 = [] for s in arr: ipaddr = s.split(":") t = (ipaddr[0], int(ipaddr[1])) arr1.append(t) sentinel = Sentinel(arr1, socket_timeout=config['redis']['redis_socket_timeout'], password=config['redis']['redis_password']) master_name = config['redis']['redis_master_name'] sentinel.discover_master(master_name) sentinel.discover_slaves(master_name) master = sentinel.master_for( master_name, socket_timeout=config['redis']['redis_socket_timeout']) return master
def init_redis(): sentinel_list = [("127.0.0.1",26379),("127.0.0.1",26380),("127.0.0.1",23681)] sentinel = Sentinel(sentinel_list,socket_timeout=0.5) master = sentinel.discover_master("mymaster") print(master) slave = sentinel.discover_slaves("mymaster") print(slave) redis_conn = sentinel.master_for('mymaster', socket_timeout=0.5) return redis_conn
class redisSentinelHelper(): def __init__(self,sentinel_list,service_name,db): self.sentinel = Sentinel(sentinel_list,socket_timeout=0.5) self.service_name = service_name self.db = db def get_master_redis(self): return self.sentinel.discover_master(self.service_name) def get_slave_redis(self): return self.sentinel.discover_slaves(self.service_name) def set_key(self,key,value): master = self.sentinel.master_for( service_name=self.service_name, socket_timeout=0.5, db=self.db ) return master.set(key,value) def set_hash_key(self,hkey,key,value): master = self.sentinel.master_for( service_name=self.service_name, socket_timeout=0.5, db=self.db ) return master.hset(hkey,key,value) def get_key(self,key): slave = self.sentinel.slave_for( service_name=self.service_name, socket_timeout=0.5, db=self.db ) return slave.get(key) def get_hash_key(self,hkey,hey): slave = self.sentinel.slave_for( service_name=self.service_name, socket_timeout=0.5, db=self.db ) return slave.hget(hkey,hey) def delete(self,key): master = self.sentinel.master_for( service_name=self.service_name, socket_timeout=0.5, db=self.db ) return master.delete(key)
def __init__(self, sentinels, service, **kwargs): sentinel = Sentinel(sentinels, **kwargs) try: master = sentinel.discover_master(service) except MasterNotFoundError: raise try: slaves = sentinel.discover_slaves(service) except SlaveNotFoundError: self._slaves = None if version_info[0] < 3: super(AutoRedisSentinel, self).__init__(master, slaves=slaves, **kwargs) else: super().__init__(master, slaves=slaves, **kwargs)
def __init__(self, sentinels, service, **kwargs): sentinel = Sentinel(sentinels, **kwargs) try: master = sentinel.discover_master(service) except MasterNotFoundError: raise try: slaves = sentinel.discover_slaves(service) except SlaveNotFoundError: self._slaves = None if version_info[0] < 3: super(AutoRedisSentinel, self).__init__(master, slaves=slaves, **kwargs) else: super().__init__(master, slaves=slaves, **kwargs)
def connect(self, index=0, write=True): """ Creates a redis connection with connection pool. """ master_name, sentinel_hosts, db = self.parse_connection_string(self._connection_string) sentinel_timeout = self._options.get("SENTINEL_TIMEOUT", 1) sentinel = Sentinel(sentinel_hosts, socket_timeout=sentinel_timeout) if write: host, port = sentinel.discover_master(master_name) else: host, port = random.choice([sentinel.discover_master(master_name)] + sentinel.discover_slaves(master_name)) return self.connection_factory.connect(host, port, db)
def get_sentinel(refresh=False): """get a sentinel connection object""" global SENTINEL if refresh : SENTINEL = None if not SENTINEL: try: SENTINEL = Sentinel( SENTINEL_LIST, # password=REDIS_AUTH, decode_responses=True, socket_timeout=0.1) assert SENTINEL.discover_master(REDIS_MASTER_NAME) assert SENTINEL.discover_slaves(REDIS_MASTER_NAME) except Exception as e: print(e) return SENTINEL
def redis_connect(sentinel_addr, sentinel_port): sentinel = Sentinel([(sentinel_addr, sentinel_port)], socket_timeout=0.1) master_ip = sentinel.discover_master('mymaster')[0] master_port = sentinel.discover_master('mymaster')[1] print("Found master on {}:{} ".format(master_ip, master_port)) slaves = sentinel.discover_slaves('mymaster') print("Found slaves on:") for slave in slaves: print(" {}:{}".format(slave[0], slave[1])) conn = redis.StrictRedis(host=master_ip, port=master_port, db=0) return conn
def connect(self, index=0, write=True): """ Creates a redis connection with connection pool. """ master_name, sentinel_hosts, db = self.parse_connection_string( self._connection_string) sentinel_timeout = self._options.get('SENTINEL_TIMEOUT', 1) sentinel = Sentinel(sentinel_hosts, socket_timeout=sentinel_timeout) if write: host, port = sentinel.discover_master(master_name) else: host, port = random.choice( [sentinel.discover_master(master_name)] + sentinel.discover_slaves(master_name)) return self.connection_factory.connect(host, port, db)
def index(): try: stream = StringIO() sentinel = Sentinel([('redis-sentinel', 26379)], socket_timeout=0.1) dm = sentinel.discover_master('dpf') ds = sentinel.discover_slaves('dpf') print('master:', dm, file=stream) print('slaves:', ds, file=stream) master = sentinel.master_for('dpf', socket_timeout=0.1) master.set('foo', time.asctime()) slave = sentinel.slave_for('dpf', socket_timeout=0.1) print('get from redis:', slave.get('foo'), file=stream) except MasterNotFoundError as e: print('redis master is GONE, waiting for failover...', e, file=stream) except Exception as e: print('exception happened, connection lost?', e, file=stream) return '{}\n'.format(stream.getvalue())
def connect(self, index=0, write=True): """ Creates a redis connection with connection pool. """ master_name, sentinel_hosts, db = self.parse_connection_string(self._connection_string) sentinel_timeout = self._options.get('SENTINEL_TIMEOUT', 1) sentinel = Sentinel(sentinel_hosts, socket_timeout=sentinel_timeout) if write: host, port = sentinel.discover_master(master_name) else: host, port = random.choice([sentinel.discover_master(master_name)] + sentinel.discover_slaves(master_name)) sentinel_options = self._options.copy() sentinel_options['CONNECTION_POOL_CLASS'] = 'redis.sentinel.SentinelConnectionPool' sentinel_options['CONNECTION_POOL_KWARGS'] = { 'service_name': master_name, 'sentinel_manager': sentinel} self.connection_factory = get_connection_factory(options=sentinel_options) return self.connection_factory.connect(host, port, db)
def connect(self, index=0, write=True): """ Creates a redis connection with connection pool. """ master_name, sentinel_hosts, db = self.parse_connection_string( self._connection_string) sentinel_timeout = self._options.get('SENTINEL_TIMEOUT', 1) sentinel = Sentinel(sentinel_hosts, socket_timeout=sentinel_timeout) if write: host, port = sentinel.discover_master(master_name) else: host, port = random.choice( [sentinel.discover_master(master_name)] + sentinel.discover_slaves(master_name)) kwargs = { "db": db, "parser_class": self.parser_class, "password": self._options.get('PASSWORD', None), } kwargs.update({ 'host': host, 'port': port, 'connection_class': Connection }) if 'SOCKET_TIMEOUT' in self._options: kwargs.update( {'socket_timeout': int(self._options['SOCKET_TIMEOUT'])}) kwargs.update(self._pool_cls_kwargs) connection_pool = get_or_create_connection_pool( self._pool_cls, **kwargs) connection = Redis(connection_pool=connection_pool) return connection
class RedisSentinel(object): """docstring for RedisSentinel""" def __init__(self, sentinel_addrs): super(RedisSentinel, self).__init__() self.m_sentinel = Sentinel(sentinel_addrs) self.m_sentinelIns = self.m_sentinel.sentinels[0] self.m_masters = self.get_masters() self.m_slaves = self.get_slaves() def get_masters(self): masters = {} masterDetail = self.m_sentinelIns.sentinel_masters() for name in masterDetail: masters[name] = (masterDetail[name]['ip'], masterDetail[name]['port']) return masters def get_slaves(self): slaves = {} for name in self.m_masters: slave = self.m_sentinel.discover_slaves(name) if len(slave) > 0: slaves[name] = (slave[0][0], slave[0][1]) return slaves
import time import random import traceback from redis.sentinel import Sentinel sentinel = Sentinel([("redis-sentinel-1", 26379), ("redis-sentinel-2", 26379), ("redis-sentinel-3", 26379)], socket_timeout=0.1) while True: try: # 发现查看下master和slave master, slave = sentinel.discover_master('mymaster'), sentinel.discover_slaves("mymaster") print("master: %s, slave: %s" % (master, slave)) # 获取master m = sentinel.master_for("mymaster", socket_timeout=0.1, password='******') s = sentinel.slave_for("mymaster", socket_timeout=0.1, password="******") value = random.randint(1, 1000) m.set("test_int", value) r = s.get("test_int") print("set test_int: %s, get test_int: %s" % (value, r)) print("沉睡中...") except: print(traceback.format_exc()) time.sleep(5)
目前支持的鉴权方式有 "simple" 和 "hmacsha1" 两种,默认使用 "simple"鉴权。 所有方法均可添加关键字参数sign_type修改鉴权方式。 """ import redis,random import kdl,requests from redis.sentinel import Sentinel sentinel = Sentinel([('192.168.17.65', 26379), ('192.168.17.66', 26379), ('192.168.17.67', 26379) ], socket_timeout=0.5) # 查看master节点 master = sentinel.discover_master('ida_redis_master') # 查看slave 节点 slave = sentinel.discover_slaves('ida_redis_master') # 连接数据库 rds = sentinel.master_for('ida_redis_master', socket_timeout=0.5, db=7, decode_responses=True) # rds = redis.StrictRedis(host='192.168.17.60', port=6378, db=7, decode_responses=True) def get_proxy_from_redis(): try: one_proxy = rds.randomkey() username = "******" password = "******" proxies = { "http": "http://%(user)s:%(pwd)s@%(ip)s/" % {'user': username, 'pwd': password, 'ip': one_proxy}, "https": "http://%(user)s:%(pwd)s@%(ip)s/" % {'user': username, 'pwd': password, 'ip': one_proxy} } return proxies except Exception as e:
import time from redis.sentinel import Sentinel sentinel = Sentinel([('192.168.11.35', 26379), ('192.168.11.37', 26379), ('192.168.11.37', 26379)], socket_timeout=0.1) for i in range(1, 1000): print "ciclo " + str(i) try: print "master: " + str(sentinel.discover_master('mymaster')) print "slave/s: " + str(sentinel.discover_slaves('mymaster')) master = sentinel.master_for('mymaster', socket_timeout=0.1) slave = sentinel.slave_for('mymaster', socket_timeout=0.1) except: print "sentinel eligiendo a un master" clave = "clave" + str(i) valor = "valor" + str(i) try: print "master.set " + clave + " " + valor master.set(clave, valor) print "master.get " + clave + " = " + str(master.get(clave)) print "master.set " + clave + " = " + str(slave.get(clave)) print "master.delete " + clave master.delete(clave) except:
class Database: """ Provide an interface using which to communicate with a Redis server. A new connection to the server is created by each class instance. Custom exception hierarchy: Error +-- ConnectionError +-- WaitTimeoutError """ socket_timeout = 1 pubsubs = {} sentinel = None class Error(Exception): pass # print "Error" class ConnectionError(Error): pass # print "ConnectionError" class WaitTimeoutError(Error): def __str__(self): return self.msg def __init__(self, msg, key, value, timeout): msg = str(msg) # in the event msg was an Exception, for example self.msg = msg self.key = key self.value = value self.timeout = timeout super(RedisClient.WaitTimeoutError, self).__init__(self, msg) # this raises the exception # Note: In Python 2.x, self.__class__ should not be used in place of base class name. def state_connection(self): """ Return a dict containing keys and values providing information about the state of the connection to the Redis server. """ if self.logger: self.logger.debug("state_connection") # conn_kwargs = self._pool.connection_kwargs # server_addr = (conn_kwargs["host"], conn_kwargs["port"]) # state = {"Server address": "{}:{}".format(*server_addr),} state = {} # Do ping test try: # connection = redis.Redis(connection_pool=self._pool) state["Server pingable"] = self.redis.ping() # returns True except redis.exceptions.ConnectionError: # as e: #self.logger.error("{} Console Redis ping failure: {}".format(tools.cm_name(), e)) # intentionally not self.logger.exception state["Server pingable"] = False # # Test availability of a key # try: # self["MONO_SV"] # except self.ConnectionError: # as e: # #self.logger.error("{} Console Redis test failure: {}".format(tools.cm_name(), e)) # # intentionally not self.logger.exception # state["Test result"] = "Failed" # else: # state["Test result"] = "Passed" return state def state_server(self): """Return a dict containing keys and values providing some properties about the the Redis server.""" if self.logger: self.logger.debug("state_server") try: # connection = redis.Redis(connection_pool=self._pool) info = self.redis.info() except redis.exceptions.ConnectionError as e: if self.logger: self.logger.error("Redis info acquisition failure: {}".format(e)) # intentionally not self.logger.exception state = {} else: state = {"Number of connected clients": info["connected_clients"], "Redis version": info["redis_version"], "Uptime": datetime.datetime.utcnow() - datetime.datetime.utcfromtimestamp(time.time() - info["uptime_in_seconds"]), } return state def __init__(self, host="localhost", port=6379, db=0, password=None, sentinels=None, master=None, settings=None, logger=False): """Initialize the client.""" # Save passed-in variables self.host = host self.port = port self.db = db self.password = password self.sentinels = sentinels self.master = master self.logger = logger # Use the RAPD CONTROL_DATABASE_SETTINGS object if settings: # Sentinel if settings["REDIS_CONNECTION"] == "sentinel": self.host = None self.port = None self.db = None self.password = settings.get("REDIS_PASSWORD", None) self.sentinels = settings["REDIS_SENTINEL_HOSTS"] self.master = settings["REDIS_MASTER_NAME"] # Standard else: self.host = settings["REDIS_HOST"] self.port = settings["REDIS_PORT"] self.db = settings["REDIS_DB"] self.password = settings.get("REDIS_PASSWORD", None) self.sentinels = None self.master = None self._ConnectionError_last_log_time = float("-inf") # Connect to server self._connect_lock = threading.Lock() self._connect() # "with self._connect_lock:" is not necessary in __init__ atexit.register(self._disconnect) ############### # ADMIN Methods ############### def _connect(self): """Connect to server.""" # self.logger.debug(tools.cm_name()) socket_timeout = self.socket_timeout # Sentinel connection if self.sentinels: self.sentinel = Sentinel(self.sentinels) # Assign master connection to self.redis self.redis = self.sentinel.master_for(self.master) # Standard connection else: attempt_count = 0 while attempt_count < CONNECTION_ATTEMPT_LIMIT: try: # Connect self.redis = redis.Redis(host=self.host, port=self.port, db=self.db, password=self.password) # self._pool = redis.ConnectionPool(host=self.host, port=self.port, db=0) # Make sure redis server is up _ = self.redis.ping() break # self._connection = redis.Redis(host, socket_timeout=socket_timeout) # redis module reconnects automatically as needed. # socket_timeout is not set to a higher value because it blocks. except redis.exceptions.ConnectionError as e: print "except" if self.logger: self.logger.debug("Connection error {}".format(e)) time.sleep(1) attempt_count += 1 # self._raise_ConnectionError(e) # # Make sure redis server is up # try: # _ = self.redis.ping() # except redis.sentinel.MasterNotFoundError as error: # self._raise_ConnectionError(error) def _disconnect(self): """Disconnect from server.""" # self.logger.debug(tools.cm_name()) try: pool = self._pool except AttributeError: pass else: pool.disconnect() #self.logger.info("{} Disconnected Redis client from Redis server {host}:{port}.".format(tools.cm_name(), **connection.connection_pool.connection_kwargs)) def _raise_ConnectionError(self, exc): """ Raise ConnectionError exception with an error message corresponding to the provided exception. """ # TODO: Implement functionality of _raise_ConnectionError directly into ConnectionError exception instead. Delete method definition. err_msg = str(exc) if err_msg: err_msg = ": {}".format(err_msg) err_msg = "Redis connection error{}".format(err_msg) # Conditionally log error # log_interval = backend.settings.redis["Console"]["ConnectionError log interval"] # if time.time() > (self._ConnectionError_last_log_time + log_interval): # self._ConnectionError_last_log_time = time.time() # logger = self.logger.info # else: if self.logger: logger = self.logger.debug logger("{}".format(err_msg)) raise self.ConnectionError(err_msg) def discover_master(self): """ Return the master instance (host, ip) """ if self.sentinel and self.master: attempts = 0 while attempts < ATTEMPT_LIMIT: try: attempts += 1 master = self.sentinel.discover_master(self.master) break except redis.sentinel.MasterNotFoundError as e: # Pause for specified time print "try %d" % attempts time.sleep(ATTEMPT_PAUSE) else: self._raise_ConnectionError(e) return master else: self._raise_ConnectionError("Sentinels not properly defined") def discover_slaves(self): """ Return the slave instances [(host, ip),] """ if self.sentinel and self.master: attempts = 0 while attempts < ATTEMPT_LIMIT: try: attempts += 1 master = self.sentinel.discover_slaves(self.master) break except redis.sentinel.SlaveNotFoundError as e: # Pause for specified time print "try %d" % attempts time.sleep(ATTEMPT_PAUSE) else: self._raise_ConnectionError(e) return master else: self._raise_ConnectionError("Sentinels not properly defined") ############# # PIPELINE Methods ############# def pipeline(self): """return a pipeline instance""" return self.redis.pipeline() ############# # GET Methods ############# def __getitem__(self, key, return_dict=False): """ Return the value of the specified key(s). key can be either a str, or a tuple or list of strs. It is recommended that key(s) be present in backend.redis_console_keys.keys. `transform` indicates whether to apply transformations to the value before returning it, as defined in the self._transform_value method. `return_dict` indicates whether to return values in a dict. If True, the keys in the returned dict match the specified key(s). """ # Get value(s) if isinstance(key, str): return self.get(key, return_dict) elif isinstance(key, tuple) or isinstance(key, list): return self.mget(key, return_dict) get = __getitem__ @connectionErrorWrapper def get(self, key, return_dict=False): """ Return the value of `key`. `return_dict` indicates whether to return values in a dict. If True, the keys in the returned dict match the specified key(s). """ # self.logger.debug("get key:{} return_dict:{}".format(key, return_dict)) # Retrieve value value = self.redis.get(key) # Wrap in dict if requested return ({key: value} if return_dict else value) # def get(self, key, return_dict=False): # """ # Return the value of `key`. # `return_dict` indicates whether to return values in a dict. If True, # the keys in the returned dict match the specified key(s). # """ # # self.logger.debug("get key:{} return_dict:{}".format(key, return_dict)) # # Retrieve value # attempts = 0 # while attempts < ATTEMPT_LIMIT: # try: # attempts += 1 # value = self.redis.get(key) # break # except redis.exceptions.ConnectionError as e: # # Pause for specified time # print "try %d" % attempts # time.sleep(ATTEMPT_PAUSE) # else: # self._raise_ConnectionError(e) # return ({key: value} if return_dict else value) @connectionErrorWrapper def mget(self, keys, return_dict=False): """ Return the values of the sequence `keys`. `transform` indicates whether to apply transformations to the value before returning it, as defined in the self._transform_value method. `return_dict` indicates whether to return values in a dict. If True, the keys in the returned dict match the specified key(s). """ # Retrieve values values = self.redis.mget(keys) return (dict(zip(keys,values)) if return_dict else values) # def mget(self, keys, return_dict=False): # """ # Return the values of the sequence `keys`. # `transform` indicates whether to apply transformations to the value # before returning it, as defined in the self._transform_value method. # `return_dict` indicates whether to return values in a dict. If True, # the keys in the returned dict match the specified key(s). # """ # # self.logger.log(5, "{} keys:{} transform:{} return_dict:{}".format(tools.cm_name(), keys, transform, return_dict)) # # Retrieve values # try: # connection = redis.Redis(connection_pool=self._pool) # values = connection.mget(keys) # except redis.exceptions.ConnectionError as e: # self._raise_ConnectionError(e) # # self.logger.log(5 if transform else logging.DEBUG, "{} keys:{} values:{}".format(tools.cm_name(), keys, repr(values))) # # Transform and return values # # if transform: values = self._transform_value(keys, values) # return (dict(zip(keys,values)) if return_dict else values) ############# # SET Methods ############# @connectionErrorWrapper def set(self, key, value): """ Set the indicated key value pair. """ if self.logger: self.logger.debug("set key:{} value:{}".format(key, value)) self.redis.set(key, str(value)) # def set(self, key, value): # """ # Set the indicated key value pair. # """ # self.logger.debug("set key:{} value:{}".format(key, value)) # # Set # attempts = 0 # while attempts < ATTEMPT_LIMIT: # try: # attempts += 1 # self.redis.set(key, str(value)) # break # except redis.exceptions.ConnectionError as error: # # Pause for specified time # print "try %d" % attempts # time.sleep(ATTEMPT_PAUSE) # else: # self._raise_ConnectionError(error) __setitem__ = set @connectionErrorWrapper def mset(self, mapping): """ Set the key value pairs which are in the dict `mapping`. """ # self.logger.debug("{} mapping:{}".format(tools.cm_name(), mapping)) # Get the mapping ready mapping = {k: str(v) for k, v in mapping.items()} # Set in redis self.redis.mset(mapping) # def mset(self, mapping): # """ # Set the key value pairs which are in the dict `mapping`. # """ # # self.logger.debug("{} mapping:{}".format(tools.cm_name(), mapping)) # mapping = {k: str(v) for k, v in mapping.items()} # # Set # attempts = 0 # while attempts < ATTEMPT_LIMIT: # try: # attempts += 1 # self.redis.mset(mapping) # break # except redis.exceptions.ConnectionError as error: # # Pause for specified time # # print "try %d" % attempts # time.sleep(ATTEMPT_PAUSE) # else: # self._raise_ConnectionError(error) @connectionErrorWrapper def setex(self, key, expire_time, value): """ Set the indicated key value pair with expire time. """ #if self.logger: # self.logger.debug("set key:{} value:{}".format(key, value)) self.redis.setex(key, int(expire_time), value) ############# # KEY Methods ############# @connectionErrorWrapper def keys(self, template): """Searches for keys fitting template.""" #self.logger.debug("set key:{} value:{}".format(key, value)) return self.redis.keys(template) @connectionErrorWrapper def delete(self, key): """Delete a key""" #self.logger.debug("set key:{} value:{}".format(key, value)) return self.redis.delete(key) ############## # LIST Methods ############## @connectionErrorWrapper def llen(self, key): """ LLEN get length of list """ value = self.redis.llen(key) return value @connectionErrorWrapper def lpop(self, key): """ LPOP a value off a given list """ value = self.redis.lpop(key) return value @connectionErrorWrapper def lpush(self, key, value): """ LPUSH a values onto a given list """ self.redis.lpush(key, value) @connectionErrorWrapper def brpop(self, key): """ RPOP a value off a given list """ value = self.redis.brpop(key) return value @connectionErrorWrapper def rpop(self, key): """ RPOP a value off a given list """ value = self.redis.rpop(key) return value @connectionErrorWrapper def rpoplpush(self, list1, list2): """ RPOPLPUSH pop a value off a given list and push on another list """ value = self.redis.rpoplpush(list1, list2) #return value ############## # HASH Methods ############## @connectionErrorWrapper def hget(self, key, field): """ HGET field on a key """ return self.redis.hget(key, field) @connectionErrorWrapper def hgetall(self, key): """ HGETALL a mapping on a key """ return self.redis.hgetall(key) @connectionErrorWrapper def hmset(self, key, mapping): """ HMSET a key with the attached mapping """ self.redis.hmset(key, mapping) @connectionErrorWrapper def hset(self, key, field, value): """ Sets field in the hash stored at key to value. """ self.redis.hset(key, field, value) ################ # PUBSUB Methods ################ @connectionErrorWrapper def publish(self, key, value): """ Publish a value on a given key """ # print "publish {} {}".format(key, value) self.redis.publish(key, value) #################### # EXPIRATION Methods #################### @connectionErrorWrapper def expire(self, key, seconds): """ Set a timeout on key """ self.redis.expire(key, seconds) # def publish(self, key, value): # """ # Publish a value on a given key # """ # # print "publish {} {}".format(key, value) # attempts = 0 # while attempts < ATTEMPT_LIMIT: # try: # attempts += 1 # self.redis.publish(key, value) # break # except redis.exceptions.ConnectionError as error: # # Pause for specified time # print "try %d" % attempts # time.sleep(ATTEMPT_PAUSE) # else: # self._raise_ConnectionError(error) @connectionErrorWrapper def get_message(self, id): """ Get message on pubsub connection """ message = self.pubsubs[id].get_message() return message # def get_message(self, id): # """ # Get message on pubsub connection # """ # attempts = 0 # while attempts < ATTEMPT_LIMIT: # try: # attempts += 1 # message = self.pubsubs[id].get_message() # break # except redis.exceptions.ConnectionError as error: # # Pause for specified time # print "try %d" % attempts # time.sleep(ATTEMPT_PAUSE) # else: # self._raise_ConnectionError(error) # return message def get_pubsub(self): """ Returns a pubsub connection - no error is triggered if no connection """ # Create the pubsub object ps = self.redis.pubsub() # Give it an id id = len(self.pubsubs) + 1 ps._id = id # Store a reference self.pubsubs[id] = ps # Return the id return id @connectionErrorWrapper def psubscribe(self, id=False, pattern=False): """ Add pattern subscription to a pubsub object and return id of pubsub object """ if pattern: # No id passed in - create pubsub object if not id: id = self.get_pubsub() self.pubsubs[id].psubscribe(pattern) return id else: raise TypeError("pattern must be specified") # def psubscribe(self, id=False, pattern=False): # """ # Add pattern subscription to a pubsub object and return id of pubsub object # """ # # No id passed in - create pubsub object # if not id: # id = self.get_pubsub() # attempts = 0 # while attempts < ATTEMPT_LIMIT: # try: # attempts += 1 # self.pubsubs[id].psubscribe(pattern) # break # except redis.exceptions.ConnectionError as error: # # Pause for specified time # # print "try %d" % attempts # time.sleep(ATTEMPT_PAUSE) # else: # self._raise_ConnectionError(error) # return id @connectionErrorWrapper def subscribe(self, id=None, channel=None): """ Add subscription to a pubsub object and return id of pubsub object """ if channel: # No id passed in - create pubsub object if not id: id = self.get_pubsub() self.pubsubs[id].subscribe(channel) return id else: raise TypeError("channel must be specified")
class SentinelConnectionFactory(ConnectionFactory): # Creates Sentinel Client from connection params # It does not cache anything def __init__(self, options): super().__init__(options) pool_cls_path = options.get( 'CONNECTION_POOL_CLASS', 'redis.sentinel.SentinelConnectionPool', ) self.pool_cls = djredis_util.load_class(pool_cls_path) self.pool_cls_kwargs = options.get('CONNECTION_POOL_KWARGS', {}) redis_client_cls_path = options.get( 'REDIS_CLIENT_CLASS', 'redis.client.StrictRedis', ) self.redis_client_cls = djredis_util.load_class(redis_client_cls_path) self.redis_client_cls_kwargs = options.get('REDIS_CLIENT_KWARGS', {}) self.service_name = options.get('SENTINEL_SERVICE_NAME', None) if not self.service_name: raise ImproperlyConfigured( 'SentinelClient requires SENTINEL_SERVICE_NAME in OPTIONS') # Get sentinels servers from options (even though it's not an option...) self.sentinels = options.get('SENTINELS', []) self.options = options # Sentinel Connection Pool is not cached, not indexed by URL, # so params are constant for each connection self.sentinel_conn_pool_cls_kwargs = { 'parser_class': self.get_parser_cls() } password = self.options.get('PASSWORD', None) if password: self.sentinel_conn_pool_cls_kwargs['password'] = password socket_timeout = self.options.get('SOCKET_TIMEOUT', None) if socket_timeout: assert isinstance( socket_timeout, (int, float)), 'Socket timeout should be float or integer' self.sentinel_conn_pool_cls_kwargs[ 'socket_timeout'] = socket_timeout socket_connect_timeout = self.options.get('SOCKET_CONNECT_TIMEOUT', None) if socket_connect_timeout: assert isinstance( socket_connect_timeout, (int, float)), 'Socket connect timeout should be float or integer' self.sentinel_conn_pool_cls_kwargs[ 'socket_connect_timeout'] = socket_connect_timeout # Actual Sentinel client, it is responsible of creating the StrictRedis clients self._sentinel = Sentinel(self.sentinels, **self.sentinel_conn_pool_cls_kwargs) # Returns a list of current slaves self._has_slaves = len( self._sentinel.discover_slaves(self.service_name)) def has_slaves(self): return self._has_slaves def connect_master(self): """ Given a basic connection parameters and sentinel client, return a new master connection. :raises MasterNotFoundError: if no master available then raises this """ return self._sentinel.master_for(self.service_name, **self.redis_client_cls_kwargs) def connect_slave(self, force_slave=False): """ Given a basic connection parameters and sentinel client, return a new slave connection if available, master's if not :raises SlaveNotFoundError: it automatically fallback to master if not slaves available, if nobody available then raises this """ if self.has_slaves() or force_slave: return self._sentinel.slave_for(self.service_name, **self.redis_client_cls_kwargs) else: # If the cluster had no slaves when creating the pool # then no need for callbacks and unnecessary discoveries, fall back # to master directly return self.connect_master()
import hashlib from rediscluster import StrictRedisCluster from SIT_tool.pyflask.lib.readconfig import getredis sentinel = Sentinel([('gis-ass-oms-gk1.cachesit.sfdc.com.cn', 8001), ('gis-ass-oms-gk2.cachesit.sfdc.com.cn', 8001), ('gis-ass-oms-gk3.cachesit.sfdc.com.cn', 8001)], socket_timeout=1) # 获取主服务器地址 master = sentinel.discover_master('GIS_ASS_OMS_REDIS_GK_C01') print(master) # 输出:('192.168.196.132', 6379) # 获取从服务器地址 slave = sentinel.discover_slaves('GIS_ASS_OMS_REDIS_GK_C01') print(slave) # 输出:[('192.168.196.129', 6379)] # 获取主服务器进行写入 master = sentinel.master_for('GIS_ASS_OMS_REDIS_GK_C01', socket_timeout=0.5, password='******', db=2) w_ret = master.set('foo', 'bar') # 输出:True # 获取从服务器进行读取(默认是round-roubin,随机从多个slave服务中读取数据) slave = sentinel.slave_for('GIS_ASS_OMS_REDIS_GK_C01', socket_timeout=0.5, password='******',
import time from redis.sentinel import Sentinel sentinel = Sentinel([('192.168.11.35', 26379),('192.168.11.37', 26379),('192.168.11.37', 26379)], socket_timeout=0.1) for i in range(1,1000): print "ciclo " + str(i) try: print "master: " + str(sentinel.discover_master('mymaster')) print "slave/s: " + str(sentinel.discover_slaves('mymaster')) master = sentinel.master_for('mymaster', socket_timeout=0.1) slave = sentinel.slave_for('mymaster', socket_timeout=0.1) except: print "sentinel eligiendo a un master" clave = "clave" + str(i) valor = "valor" + str(i) try: print "master.set " + clave + " " + valor master.set(clave, valor) print "master.get " + clave + " = " + str(master.get(clave)) print "master.set " + clave + " = " + str(slave.get(clave)) print "master.delete " + clave master.delete(clave) except: print "cluster en modo solo lectura"
from redis.sentinel import Sentinel sentinel = Sentinel([("127.0.0.1", 26379), ("127.0.0.1", 26380), ("127.0.0.1", 26381)], socket_timeout=0.1) print(sentinel.discover_master("mymaster")) print(sentinel.discover_slaves("mymaster"))
sentinel = Sentinel([('localhost', 55511)], socket_timeout=0.1) if sentinel.discover_master(NAME)[1] != 33311: print 'unexpected situation: expecting port 33311 to be the master' sys.exit(99) print """>> Getting an High Availability connection through the sentinel. """ mha = sentinel.master_for(NAME) if int(mha.config_get()['port']) != 33311: print 'unexpected master. expecting master to be on port 33311.' sys.exit(99) assert mha.ping() and a.ping() print """>> Currently: (MASTER:{})<==(SLAVES:{})""".format(sentinel.discover_master(NAME), sentinel.discover_slaves(NAME)) print """>> Stopping A instance (current master) """ # err = subprocess.call('./stop-master.sh') # if err: # print 'error stopping master instance.' # sys.exit(99) raw_input('./stop-master.sh') print ">> waiting for A to stop." try: while a.ping(): pass except ConnectionError: print '>> instance A stopped.'
# https://www.throwable.club/2019/10/07/redis-server-sentinel-install-guide/ import pprint from redis.sentinel import Sentinel sentinel = Sentinel([('127.0.0.1', '26379')]) master = sentinel.discover_master("doge-master") slaver = sentinel.discover_slaves("doge-master") pprint.pprint(master) pprint.pprint(slaver) master = sentinel.master_for("doge-master") slaver = sentinel.slave_for("doge-master") pprint.pprint(master.set("a", 1)) pprint.pprint(slaver.info("server").get("tcp_port"))
import redis import time from redis.sentinel import Sentinel sentinel = Sentinel([('192.168.55.31', 26379),('192.168.55.32', 26379),('192.168.55.33', 26379)],socket_timeout=0.5) master = sentinel.discover_master('sdemo') print(master) slave = sentinel.discover_slaves('sdemo') print(slave) for i in range(1, 1000): key = "k%d" % i value = "v%d" % i try: master = sentinel.master_for('sdemo', socket_timeout=0.5, password='******', db=0) slave = sentinel.slave_for('sdemo', socket_timeout=0.5, password='******', db=0) master.set(key, value) ret = slave.get(key) print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime((time.time()))), ret) except Exception as e: print(str(e)) time.sleep(1)
from redis.sentinel import Sentinel, MasterNotFoundError while True: import time try: sentinel = Sentinel([('redis-sentinel', 26379)], socket_timeout=0.1) dm = sentinel.discover_master('dpf') ds = sentinel.discover_slaves('dpf') print('master:', dm, '====>', 'slaves', ds) master = sentinel.master_for('dpf', socket_timeout=0.1) master.set('foo', time.asctime()) slave = sentinel.slave_for('dpf', socket_timeout=0.1) print('get from redis:', slave.get('foo')) except MasterNotFoundError as e: print('redis master is GONE, waiting for failover...', e) except Exception as e: print('exception happened, connection lost?', e) time.sleep(1.0)
if __name__ == "__main__": # change logging config logging.basicConfig(level=logging.DEBUG, format='[%(asctime)s.%(msecs)03d][%(filename)s:%(lineno)d][%(levelname)s]%(message)s', datefmt='%Y-%m-%d %H:%M:%S') logger = logging.getLogger(__file__) # redis sentinels = [ ('redis-sentinel-1', 26379), ('redis-sentinel-2', 26379), ('redis-sentinel-3', 26379) ] while True: try: sentinel = Sentinel(sentinels, socket_timeout=0.1) logger.debug('Cluster Master: %s', str(sentinel.discover_master('mymaster'))) logger.debug('Cluster Slave: %s', str(sentinel.discover_slaves('mymaster'))) master = sentinel.master_for('mymaster', socket_timeout=0.1, password='******') slave = sentinel.slave_for('mymaster', socket_timeout=0.1, password='******') master.set('timestamp', datetime.now().strftime('%Y-%m-%d %H:%M:%S')) logger.debug('Reading key(timestamp): %s', slave.get('timestamp')) except Exception as ex: logger.error(str(ex)) finally: time.sleep(1)
class Database: """ Provide an interface using which to communicate with a Redis server. A new connection to the server is created by each class instance. Custom exception hierarchy: Error +-- ConnectionError +-- WaitTimeoutError """ socket_timeout = 1 pubsubs = {} sentinel = None class Error(Exception): pass # print "Error" class ConnectionError(Error): pass # print "ConnectionError" class WaitTimeoutError(Error): def __str__(self): return self.msg def __init__(self, msg, key, value, timeout): msg = str(msg) # in the event msg was an Exception, for example self.msg = msg self.key = key self.value = value self.timeout = timeout super(RedisClient.WaitTimeoutError, self).__init__(self, msg) # this raises the exception # Note: In Python 2.x, self.__class__ should not be used in place of base class name. def state_connection(self): """ Return a dict containing keys and values providing information about the state of the connection to the Redis server. """ if self.logger: self.logger.debug("state_connection") # conn_kwargs = self._pool.connection_kwargs # server_addr = (conn_kwargs["host"], conn_kwargs["port"]) # state = {"Server address": "{}:{}".format(*server_addr),} state = {} # Do ping test try: # connection = redis.Redis(connection_pool=self._pool) state["Server pingable"] = self.redis.ping() # returns True except redis.exceptions.ConnectionError: # as e: #self.logger.error("{} Console Redis ping failure: {}".format(tools.cm_name(), e)) # intentionally not self.logger.exception state["Server pingable"] = False # # Test availability of a key # try: # self["MONO_SV"] # except self.ConnectionError: # as e: # #self.logger.error("{} Console Redis test failure: {}".format(tools.cm_name(), e)) # # intentionally not self.logger.exception # state["Test result"] = "Failed" # else: # state["Test result"] = "Passed" return state def state_server(self): """Return a dict containing keys and values providing some properties about the the Redis server.""" if self.logger: self.logger.debug("state_server") try: # connection = redis.Redis(connection_pool=self._pool) info = self.redis.info() except redis.exceptions.ConnectionError as e: if self.logger: self.logger.error( "Redis info acquisition failure: {}".format(e)) # intentionally not self.logger.exception state = {} else: state = { "Number of connected clients": info["connected_clients"], "Redis version": info["redis_version"], "Uptime": datetime.datetime.utcnow() - datetime.datetime.utcfromtimestamp(time.time() - info["uptime_in_seconds"]), } return state def __init__(self, host="localhost", port=6379, db=0, password=None, sentinels=None, master=None, settings=None, logger=False): """Initialize the client.""" # Save passed-in variables self.host = host self.port = port self.db = db self.password = password self.sentinels = sentinels self.master = master self.logger = logger # Use the RAPD CONTROL_DATABASE_SETTINGS object if settings: # Sentinel if settings["REDIS_CONNECTION"] == "sentinel": self.host = None self.port = None self.db = None self.password = settings.get("REDIS_PASSWORD", None) self.sentinels = settings["REDIS_SENTINEL_HOSTS"] self.master = settings["REDIS_MASTER_NAME"] # Standard else: self.host = settings["REDIS_HOST"] self.port = settings["REDIS_PORT"] self.db = settings["REDIS_DB"] self.password = settings.get("REDIS_PASSWORD", None) self.sentinels = None self.master = None self._ConnectionError_last_log_time = float("-inf") # Connect to server self._connect_lock = threading.Lock() self._connect( ) # "with self._connect_lock:" is not necessary in __init__ atexit.register(self._disconnect) ############### # ADMIN Methods ############### def _connect(self): """Connect to server.""" # self.logger.debug(tools.cm_name()) socket_timeout = self.socket_timeout # Sentinel connection if self.sentinels: self.sentinel = Sentinel(self.sentinels) # Assign master connection to self.redis self.redis = self.sentinel.master_for(self.master) # Standard connection else: attempt_count = 0 while attempt_count < CONNECTION_ATTEMPT_LIMIT: try: # Connect self.redis = redis.Redis(host=self.host, port=self.port, db=self.db, password=self.password) # self._pool = redis.ConnectionPool(host=self.host, port=self.port, db=0) # Make sure redis server is up _ = self.redis.ping() break # self._connection = redis.Redis(host, socket_timeout=socket_timeout) # redis module reconnects automatically as needed. # socket_timeout is not set to a higher value because it blocks. except redis.exceptions.ConnectionError as e: print "except" if self.logger: self.logger.debug("Connection error {}".format(e)) time.sleep(1) attempt_count += 1 # self._raise_ConnectionError(e) # # Make sure redis server is up # try: # _ = self.redis.ping() # except redis.sentinel.MasterNotFoundError as error: # self._raise_ConnectionError(error) def _disconnect(self): """Disconnect from server.""" # self.logger.debug(tools.cm_name()) try: pool = self._pool except AttributeError: pass else: pool.disconnect() #self.logger.info("{} Disconnected Redis client from Redis server {host}:{port}.".format(tools.cm_name(), **connection.connection_pool.connection_kwargs)) def _raise_ConnectionError(self, exc): """ Raise ConnectionError exception with an error message corresponding to the provided exception. """ # TODO: Implement functionality of _raise_ConnectionError directly into ConnectionError exception instead. Delete method definition. err_msg = str(exc) if err_msg: err_msg = ": {}".format(err_msg) err_msg = "Redis connection error{}".format(err_msg) # Conditionally log error # log_interval = backend.settings.redis["Console"]["ConnectionError log interval"] # if time.time() > (self._ConnectionError_last_log_time + log_interval): # self._ConnectionError_last_log_time = time.time() # logger = self.logger.info # else: if self.logger: logger = self.logger.debug logger("{}".format(err_msg)) raise self.ConnectionError(err_msg) def discover_master(self): """ Return the master instance (host, ip) """ if self.sentinel and self.master: attempts = 0 while attempts < ATTEMPT_LIMIT: try: attempts += 1 master = self.sentinel.discover_master(self.master) break except redis.sentinel.MasterNotFoundError as e: # Pause for specified time print "try %d" % attempts time.sleep(ATTEMPT_PAUSE) else: self._raise_ConnectionError(e) return master else: self._raise_ConnectionError("Sentinels not properly defined") def discover_slaves(self): """ Return the slave instances [(host, ip),] """ if self.sentinel and self.master: attempts = 0 while attempts < ATTEMPT_LIMIT: try: attempts += 1 master = self.sentinel.discover_slaves(self.master) break except redis.sentinel.SlaveNotFoundError as e: # Pause for specified time print "try %d" % attempts time.sleep(ATTEMPT_PAUSE) else: self._raise_ConnectionError(e) return master else: self._raise_ConnectionError("Sentinels not properly defined") ############# # PIPELINE Methods ############# def pipeline(self): """return a pipeline instance""" return self.redis.pipeline() ############# # GET Methods ############# def __getitem__(self, key, return_dict=False): """ Return the value of the specified key(s). key can be either a str, or a tuple or list of strs. It is recommended that key(s) be present in backend.redis_console_keys.keys. `transform` indicates whether to apply transformations to the value before returning it, as defined in the self._transform_value method. `return_dict` indicates whether to return values in a dict. If True, the keys in the returned dict match the specified key(s). """ # Get value(s) if isinstance(key, str): return self.get(key, return_dict) elif isinstance(key, tuple) or isinstance(key, list): return self.mget(key, return_dict) get = __getitem__ @connectionErrorWrapper def get(self, key, return_dict=False): """ Return the value of `key`. `return_dict` indicates whether to return values in a dict. If True, the keys in the returned dict match the specified key(s). """ # self.logger.debug("get key:{} return_dict:{}".format(key, return_dict)) # Retrieve value value = self.redis.get(key) # Wrap in dict if requested return ({key: value} if return_dict else value) # def get(self, key, return_dict=False): # """ # Return the value of `key`. # `return_dict` indicates whether to return values in a dict. If True, # the keys in the returned dict match the specified key(s). # """ # # self.logger.debug("get key:{} return_dict:{}".format(key, return_dict)) # # Retrieve value # attempts = 0 # while attempts < ATTEMPT_LIMIT: # try: # attempts += 1 # value = self.redis.get(key) # break # except redis.exceptions.ConnectionError as e: # # Pause for specified time # print "try %d" % attempts # time.sleep(ATTEMPT_PAUSE) # else: # self._raise_ConnectionError(e) # return ({key: value} if return_dict else value) @connectionErrorWrapper def mget(self, keys, return_dict=False): """ Return the values of the sequence `keys`. `transform` indicates whether to apply transformations to the value before returning it, as defined in the self._transform_value method. `return_dict` indicates whether to return values in a dict. If True, the keys in the returned dict match the specified key(s). """ # Retrieve values values = self.redis.mget(keys) return (dict(zip(keys, values)) if return_dict else values) # def mget(self, keys, return_dict=False): # """ # Return the values of the sequence `keys`. # `transform` indicates whether to apply transformations to the value # before returning it, as defined in the self._transform_value method. # `return_dict` indicates whether to return values in a dict. If True, # the keys in the returned dict match the specified key(s). # """ # # self.logger.log(5, "{} keys:{} transform:{} return_dict:{}".format(tools.cm_name(), keys, transform, return_dict)) # # Retrieve values # try: # connection = redis.Redis(connection_pool=self._pool) # values = connection.mget(keys) # except redis.exceptions.ConnectionError as e: # self._raise_ConnectionError(e) # # self.logger.log(5 if transform else logging.DEBUG, "{} keys:{} values:{}".format(tools.cm_name(), keys, repr(values))) # # Transform and return values # # if transform: values = self._transform_value(keys, values) # return (dict(zip(keys,values)) if return_dict else values) ############# # SET Methods ############# @connectionErrorWrapper def set(self, key, value): """ Set the indicated key value pair. """ if self.logger: self.logger.debug("set key:{} value:{}".format(key, value)) self.redis.set(key, str(value)) # def set(self, key, value): # """ # Set the indicated key value pair. # """ # self.logger.debug("set key:{} value:{}".format(key, value)) # # Set # attempts = 0 # while attempts < ATTEMPT_LIMIT: # try: # attempts += 1 # self.redis.set(key, str(value)) # break # except redis.exceptions.ConnectionError as error: # # Pause for specified time # print "try %d" % attempts # time.sleep(ATTEMPT_PAUSE) # else: # self._raise_ConnectionError(error) __setitem__ = set @connectionErrorWrapper def mset(self, mapping): """ Set the key value pairs which are in the dict `mapping`. """ # self.logger.debug("{} mapping:{}".format(tools.cm_name(), mapping)) # Get the mapping ready mapping = {k: str(v) for k, v in mapping.items()} # Set in redis self.redis.mset(mapping) # def mset(self, mapping): # """ # Set the key value pairs which are in the dict `mapping`. # """ # # self.logger.debug("{} mapping:{}".format(tools.cm_name(), mapping)) # mapping = {k: str(v) for k, v in mapping.items()} # # Set # attempts = 0 # while attempts < ATTEMPT_LIMIT: # try: # attempts += 1 # self.redis.mset(mapping) # break # except redis.exceptions.ConnectionError as error: # # Pause for specified time # # print "try %d" % attempts # time.sleep(ATTEMPT_PAUSE) # else: # self._raise_ConnectionError(error) @connectionErrorWrapper def setex(self, key, expire_time, value): """ Set the indicated key value pair with expire time. """ #if self.logger: # self.logger.debug("set key:{} value:{}".format(key, value)) self.redis.setex(key, int(expire_time), value) ############# # KEY Methods ############# @connectionErrorWrapper def keys(self, template): """Searches for keys fitting template.""" #self.logger.debug("set key:{} value:{}".format(key, value)) return self.redis.keys(template) @connectionErrorWrapper def delete(self, key): """Delete a key""" #self.logger.debug("set key:{} value:{}".format(key, value)) return self.redis.delete(key) ############## # LIST Methods ############## @connectionErrorWrapper def llen(self, key): """ LLEN get length of list """ value = self.redis.llen(key) return value @connectionErrorWrapper def lpop(self, key): """ LPOP a value off a given list """ value = self.redis.lpop(key) return value @connectionErrorWrapper def lpush(self, key, value): """ LPUSH a values onto a given list """ self.redis.lpush(key, value) @connectionErrorWrapper def brpop(self, key): """ RPOP a value off a given list """ value = self.redis.brpop(key) return value @connectionErrorWrapper def rpop(self, key): """ RPOP a value off a given list """ value = self.redis.rpop(key) return value @connectionErrorWrapper def rpoplpush(self, list1, list2): """ RPOPLPUSH pop a value off a given list and push on another list """ value = self.redis.rpoplpush(list1, list2) #return value ############## # HASH Methods ############## @connectionErrorWrapper def hget(self, key, field): """ HGET field on a key """ return self.redis.hget(key, field) @connectionErrorWrapper def hgetall(self, key): """ HGETALL a mapping on a key """ return self.redis.hgetall(key) @connectionErrorWrapper def hmset(self, key, mapping): """ HMSET a key with the attached mapping """ self.redis.hmset(key, mapping) @connectionErrorWrapper def hset(self, key, field, value): """ Sets field in the hash stored at key to value. """ self.redis.hset(key, field, value) ################ # PUBSUB Methods ################ @connectionErrorWrapper def publish(self, key, value): """ Publish a value on a given key """ # print "publish {} {}".format(key, value) self.redis.publish(key, value) #################### # EXPIRATION Methods #################### @connectionErrorWrapper def expire(self, key, seconds): """ Set a timeout on key """ self.redis.expire(key, seconds) # def publish(self, key, value): # """ # Publish a value on a given key # """ # # print "publish {} {}".format(key, value) # attempts = 0 # while attempts < ATTEMPT_LIMIT: # try: # attempts += 1 # self.redis.publish(key, value) # break # except redis.exceptions.ConnectionError as error: # # Pause for specified time # print "try %d" % attempts # time.sleep(ATTEMPT_PAUSE) # else: # self._raise_ConnectionError(error) @connectionErrorWrapper def get_message(self, id): """ Get message on pubsub connection """ message = self.pubsubs[id].get_message() return message # def get_message(self, id): # """ # Get message on pubsub connection # """ # attempts = 0 # while attempts < ATTEMPT_LIMIT: # try: # attempts += 1 # message = self.pubsubs[id].get_message() # break # except redis.exceptions.ConnectionError as error: # # Pause for specified time # print "try %d" % attempts # time.sleep(ATTEMPT_PAUSE) # else: # self._raise_ConnectionError(error) # return message def get_pubsub(self): """ Returns a pubsub connection - no error is triggered if no connection """ # Create the pubsub object ps = self.redis.pubsub() # Give it an id id = len(self.pubsubs) + 1 ps._id = id # Store a reference self.pubsubs[id] = ps # Return the id return id @connectionErrorWrapper def psubscribe(self, id=False, pattern=False): """ Add pattern subscription to a pubsub object and return id of pubsub object """ if pattern: # No id passed in - create pubsub object if not id: id = self.get_pubsub() self.pubsubs[id].psubscribe(pattern) return id else: raise TypeError("pattern must be specified") # def psubscribe(self, id=False, pattern=False): # """ # Add pattern subscription to a pubsub object and return id of pubsub object # """ # # No id passed in - create pubsub object # if not id: # id = self.get_pubsub() # attempts = 0 # while attempts < ATTEMPT_LIMIT: # try: # attempts += 1 # self.pubsubs[id].psubscribe(pattern) # break # except redis.exceptions.ConnectionError as error: # # Pause for specified time # # print "try %d" % attempts # time.sleep(ATTEMPT_PAUSE) # else: # self._raise_ConnectionError(error) # return id @connectionErrorWrapper def subscribe(self, id=None, channel=None): """ Add subscription to a pubsub object and return id of pubsub object """ if channel: # No id passed in - create pubsub object if not id: id = self.get_pubsub() self.pubsubs[id].subscribe(channel) return id else: raise TypeError("channel must be specified")
# 3.配置文件 bind 不能为127.0.0.1 应该是 0.0.0.0 from redis.sentinel import Sentinel # 连接哨兵服务器(主机名也可以用域名) sentinel = Sentinel([('192.168.56.101', 26379)], socket_timeout=0.5) # 获取主服务器地址 master = sentinel.discover_master('mymaster') print(master) # 输出:('172.31.0.2', 5001) # 获取从服务器地址 slave = sentinel.discover_slaves('mymaster') print(slave) # 输出:[('172.31.3', 5001), ('172.31.0.4', 5001), ('172.31.0.5', 5001)] # 获取主服务器进行写入 master = sentinel.master_for('mymaster', socket_timeout=0.5, db=15) w_ret = master.set('foo', 'bar') # 输出:True # # 获取从服务器进行读取(默认是round-roubin) slave = sentinel.slave_for('mymaster', socket_timeout=0.5, db=15) r_ret = slave.get('foo') print(r_ret)
# -*- coding: utf-8 -*- from redis.sentinel import Sentinel REDIS_SENTINEL_SERVICE_NAME = 'mymaster' # 指定sentinel的地址和端口号 sentinel = Sentinel([('192.168.1.24', 26382), ('192.168.1.24', 26383), ('192.168.1.24', 26384), ('192.168.1.24', 26385)], socket_timeout=0.01) # 测试,获取以下主库和从库的信息 print sentinel.discover_master(REDIS_SENTINEL_SERVICE_NAME) print sentinel.discover_slaves('mymaster') ##配置读写分离 #写节点 master = sentinel.master_for(REDIS_SENTINEL_SERVICE_NAME, socket_timeout=0.01) #读节点 slave = sentinel.slave_for(REDIS_SENTINEL_SERVICE_NAME, socket_timeout=0.01) ###读写分离测试 key print master.set('oldboy', '123') print slave.get('oldboy')
# -*- coding:utf-8 -*- from redis.sentinel import Sentinel # 连接哨兵服务器(主机名也可以用域名) sentinel = Sentinel([('172.31.0.2', 5001), ('172.31.0.3', 5001), ('172.31.0.4', 5001), ('172.31.0.5', 5001)], socket_timeout=10000) # 获取主服务器地址 master = sentinel.discover_master('mymaster') print(master) # 输出:('172.31.0.2', 5001) # 获取从服务器地址 slave = sentinel.discover_slaves('mymaster') print(slave) # 输出:[('172.31.3', 5001), ('172.31.0.4', 5001), ('172.31.0.5', 5001)] # 获取主服务器进行写入 master = sentinel.master_for('mymaster', socket_timeout=0.5, password='******', db=15) w_ret = master.set('foo', 'bar') # 输出:True # # 获取从服务器进行读取(默认是round-roubin) slave = sentinel.slave_for('mymaster', socket_timeout=0.5, password='******',
#!/usr/bin/python from redis.sentinel import Sentinel sentinel = Sentinel([('localhost', 19001)], socket_timeout=0.1) master = sentinel.discover_master('redis-cluster') slaves = sentinel.discover_slaves('redis-cluster') print "" print "Master(" + str(master) + ")" print "" print "Slaves(" + str(len(slaves)) + ") Nodes(" + str(slaves) + ")" print ""
from redis.sentinel import Sentinel from config import config sentinel = Sentinel([ ('192.168.39.57',26379), ('192.168.39.110',26379), ], socket_timeout=0.5, ) # 获取主服务器地址 master = sentinel.discover_master(config.REDIS['sentinel']['master']) print(master) # 获取从服务器地址 slave = sentinel.discover_slaves(config.REDIS['sentinel']['master']) print(slave) # 获取主服务器进行写入mq def write_mq(theme, msg): master_write = sentinel.master_for(config.REDIS['sentinel']['master'], socket_timeout=0.5, password=config.REDIS['password'], db=config.REDIS['database'] ) return master_write.set(theme, msg) # 获取从服务器进行读取 def read_mq(theme):
from redis.sentinel import Sentinel from redisgraph import Node, Edge, Graph sentinel = Sentinel([('localhost', 26379)]) #sentinel = Sentinel([('io-madstat-prod-redis-redis-ha.redis', 26379)], socket_timeout=0.1) print "MASTERS > {0}".format(sentinel.discover_master('mymaster')) print "SLAVES > {0}".format(sentinel.discover_slaves('mymaster')) slave = sentinel.slave_for('mymaster', socket_timeout=0.3) redis_graph = Graph('bulk', slave) query = "MATCH (t:Tag {name: 'odin'}) RETURN t" result = redis_graph.query(query) result.pretty_print()
import redis # pip install redis from redis.sentinel import Sentinel import random bind_ip = "localhost" master_port = 7000 slave_ports = [7001, 7002] master_name = "mymaster" sentinel_hosts = [(bind_ip, 5000), (bind_ip, 5001), (bind_ip, 5002)] # TODO: intend to split reads and writes master = redis.Redis(host=bind_ip, port=master_port, db=0) clients = [redis.Redis(host=bind_ip, port=port) for port in slave_ports] master.set("foo", "bar") assert "bar" == master.get("foo"), "get foo not return bar" client = random.choice(clients) assert "bar" == client.get("foo"), "get foo not return bar" master.delete("foo") assert None == client.get("foo"), "get foo not return None" print("test set get delete ok") sentinel = Sentinel(sentinel_hosts, socket_timeout=0.1) print(sentinel.discover_master(master_name)) print(sentinel.discover_slaves(master_name)) master = sentinel.master_for(master_name, socket_timeout=0.1) slave = sentinel.slave_for(master_name, socket_timeout=0.1) master.set('foo1', 'bar1') print(slave.get('foo1'))
def add_master_slave_stats(self): master_normal = 1 switch_over = 0 slaves_list = [] master_host = '' sentinel_list = [(node.get('host', 'localhost'), node.get('sentinel_port', 26379)) for node in self.node_list] sentinel = Sentinel(sentinel_list, socket_timeout=5) try: master_host, master_port = sentinel.discover_master( self.master_name) last_master = '' with open(self.tmp_master_file, 'r') as f: last_master = f.readline().strip() if last_master != master_host: switch_over = 1 with open(self.tmp_master_file, 'w') as f: f.write(master_host) except MasterNotFoundError: master_normal = 0 try: master_conn = sentinel.master_for(self.master_name, db=self.db, password=self.password) set_data = random.randint(0, 10) if master_conn.set('newrelic_redis_cluster_agent', set_data): if master_conn.get('newrelic_redis_cluster_agent') != str( set_data): master_normal = 0 else: master_normal = 0 except MasterNotFoundError: master_normal = 0 slaves_list = sentinel.discover_slaves(self.master_name) slaves_hosts = [slave[0] for slave in slaves_list] for node in self.node_list: host = node.get('host', 'localhost') status = 0 if host == master_host: status = 2 if host in slaves_hosts: status = 1 self.add_gauge_value('Redis_Cluster/ClusterStatus/%s' % host, None, status, count=1) self.add_gauge_value('Redis_Cluster/SlavesNum', None, len(slaves_list), count=1) self.add_gauge_value('Redis_Cluster/MasterStatus', None, master_normal, count=1) self.add_gauge_value('Redis_Cluster/SwitchOver', None, switch_over, count=1)