def connect(self, index=0, write=True):
"""
Creates a redis connection with connection pool.
"""
master_name, sentinel_hosts, db = self.parse_connection_string(self._connection_string)
sentinel_timeout = self._options.get('SENTINEL_TIMEOUT', 1)
sentinel = Sentinel(sentinel_hosts, socket_timeout=sentinel_timeout)
if write:
host, port = sentinel.discover_master(master_name)
else:
host, port = random.choice([sentinel.discover_master(master_name)] + sentinel.discover_slaves(master_name))
kwargs = {
"db": db,
"parser_class": self.parser_class,
"password": self._options.get('PASSWORD', None),
}
kwargs.update({'host': host, 'port': port, 'connection_class': Connection})
if 'SOCKET_TIMEOUT' in self._options:
kwargs.update({'socket_timeout': int(self._options['SOCKET_TIMEOUT'])})
kwargs.update(self._pool_cls_kwargs)
connection_pool = get_or_create_connection_pool(self._pool_cls, **kwargs)
connection = Redis(connection_pool=connection_pool)
return connection
def connect(self, write=True):
"""
Create a redis connection with connection pool.
"""
self.log.debug("connect called: write=%s", write)
sentinel_timeout = self._options.get("SENTINEL_TIMEOUT", 1)
password = self._options.get("PASSWORD", None)
sentinel = Sentinel(
self._sentinel_hosts, socket_timeout=sentinel_timeout, password=password
)
if write:
host, port = sentinel.discover_master(self._master_name)
else:
try:
host, port = random.choice( # nosec
sentinel.discover_slaves(self._master_name)
)
except IndexError:
self.log.debug("no slaves are available. using master for read.")
host, port = sentinel.discover_master(self._master_name)
if password:
connection_url = f"redis://:{password}@{host}:{port}/{self._database_name}"
else:
connection_url = f"redis://{host}:{port}/{self._database_name}"
self.log.debug("Connecting to: %s", connection_url)
return self.connection_factory.connect(connection_url)
def discover_slave(s: Sentinel):
slave = None
try:
slave = s.discover_slaves(redis_master_name)
except Exception as Err:
pass
return slave or "No slave found."
def getRedisClient():
ipaddrs = config['redis']['redis_ipaddrs']
arr = ipaddrs.split(",")
arr1 = []
for s in arr:
ipaddr = s.split(":")
t = (ipaddr[0], int(ipaddr[1]))
arr1.append(t)
sentinel = Sentinel(arr1,
socket_timeout=config['redis']['redis_socket_timeout'],
password=config['redis']['redis_password'])
master_name = config['redis']['redis_master_name']
sentinel.discover_master(master_name)
sentinel.discover_slaves(master_name)
master = sentinel.master_for(
master_name, socket_timeout=config['redis']['redis_socket_timeout'])
return master
def init_redis():
sentinel_list = [("127.0.0.1",26379),("127.0.0.1",26380),("127.0.0.1",23681)]
sentinel = Sentinel(sentinel_list,socket_timeout=0.5)
master = sentinel.discover_master("mymaster")
print(master)
slave = sentinel.discover_slaves("mymaster")
print(slave)
redis_conn = sentinel.master_for('mymaster', socket_timeout=0.5)
return redis_conn
class redisSentinelHelper():
def __init__(self,sentinel_list,service_name,db):
self.sentinel = Sentinel(sentinel_list,socket_timeout=0.5)
self.service_name = service_name
self.db = db
def get_master_redis(self):
return self.sentinel.discover_master(self.service_name)
def get_slave_redis(self):
return self.sentinel.discover_slaves(self.service_name)
def set_key(self,key,value):
master = self.sentinel.master_for(
service_name=self.service_name,
socket_timeout=0.5,
db=self.db
)
return master.set(key,value)
def set_hash_key(self,hkey,key,value):
master = self.sentinel.master_for(
service_name=self.service_name,
socket_timeout=0.5,
db=self.db
)
return master.hset(hkey,key,value)
def get_key(self,key):
slave = self.sentinel.slave_for(
service_name=self.service_name,
socket_timeout=0.5,
db=self.db
)
return slave.get(key)
def get_hash_key(self,hkey,hey):
slave = self.sentinel.slave_for(
service_name=self.service_name,
socket_timeout=0.5,
db=self.db
)
return slave.hget(hkey,hey)
def delete(self,key):
master = self.sentinel.master_for(
service_name=self.service_name,
socket_timeout=0.5,
db=self.db
)
return master.delete(key)
def __init__(self, sentinels, service, **kwargs):
sentinel = Sentinel(sentinels, **kwargs)
try:
master = sentinel.discover_master(service)
except MasterNotFoundError:
raise
try:
slaves = sentinel.discover_slaves(service)
except SlaveNotFoundError:
self._slaves = None
if version_info[0] < 3:
super(AutoRedisSentinel, self).__init__(master, slaves=slaves, **kwargs)
else:
super().__init__(master, slaves=slaves, **kwargs)
def __init__(self, sentinels, service, **kwargs):
sentinel = Sentinel(sentinels, **kwargs)
try:
master = sentinel.discover_master(service)
except MasterNotFoundError:
raise
try:
slaves = sentinel.discover_slaves(service)
except SlaveNotFoundError:
self._slaves = None
if version_info[0] < 3:
super(AutoRedisSentinel, self).__init__(master, slaves=slaves, **kwargs)
else:
super().__init__(master, slaves=slaves, **kwargs)
def connect(self, index=0, write=True):
"""
Creates a redis connection with connection pool.
"""
master_name, sentinel_hosts, db = self.parse_connection_string(self._connection_string)
sentinel_timeout = self._options.get("SENTINEL_TIMEOUT", 1)
sentinel = Sentinel(sentinel_hosts, socket_timeout=sentinel_timeout)
if write:
host, port = sentinel.discover_master(master_name)
else:
host, port = random.choice([sentinel.discover_master(master_name)] + sentinel.discover_slaves(master_name))
return self.connection_factory.connect(host, port, db)
def get_sentinel(refresh=False):
"""get a sentinel connection object"""
global SENTINEL
if refresh :
SENTINEL = None
if not SENTINEL:
try:
SENTINEL = Sentinel(
SENTINEL_LIST,
# password=REDIS_AUTH,
decode_responses=True,
socket_timeout=0.1)
assert SENTINEL.discover_master(REDIS_MASTER_NAME)
assert SENTINEL.discover_slaves(REDIS_MASTER_NAME)
except Exception as e:
print(e)
return SENTINEL
def redis_connect(sentinel_addr, sentinel_port):
sentinel = Sentinel([(sentinel_addr, sentinel_port)], socket_timeout=0.1)
master_ip = sentinel.discover_master('mymaster')[0]
master_port = sentinel.discover_master('mymaster')[1]
print("Found master on {}:{} ".format(master_ip, master_port))
slaves = sentinel.discover_slaves('mymaster')
print("Found slaves on:")
for slave in slaves:
print(" {}:{}".format(slave[0], slave[1]))
conn = redis.StrictRedis(host=master_ip, port=master_port, db=0)
return conn
def connect(self, index=0, write=True):
"""
Creates a redis connection with connection pool.
"""
master_name, sentinel_hosts, db = self.parse_connection_string(
self._connection_string)
sentinel_timeout = self._options.get('SENTINEL_TIMEOUT', 1)
sentinel = Sentinel(sentinel_hosts, socket_timeout=sentinel_timeout)
if write:
host, port = sentinel.discover_master(master_name)
else:
host, port = random.choice(
[sentinel.discover_master(master_name)] +
sentinel.discover_slaves(master_name))
return self.connection_factory.connect(host, port, db)
def index():
try:
stream = StringIO()
sentinel = Sentinel([('redis-sentinel', 26379)], socket_timeout=0.1)
dm = sentinel.discover_master('dpf')
ds = sentinel.discover_slaves('dpf')
print('master:', dm, file=stream)
print('slaves:', ds, file=stream)
master = sentinel.master_for('dpf', socket_timeout=0.1)
master.set('foo', time.asctime())
slave = sentinel.slave_for('dpf', socket_timeout=0.1)
print('get from redis:', slave.get('foo'), file=stream)
except MasterNotFoundError as e:
print('redis master is GONE, waiting for failover...', e, file=stream)
except Exception as e:
print('exception happened, connection lost?', e, file=stream)
return '{}\n'.format(stream.getvalue())
def connect(self, index=0, write=True):
"""
Creates a redis connection with connection pool.
"""
master_name, sentinel_hosts, db = self.parse_connection_string(self._connection_string)
sentinel_timeout = self._options.get('SENTINEL_TIMEOUT', 1)
sentinel = Sentinel(sentinel_hosts, socket_timeout=sentinel_timeout)
if write:
host, port = sentinel.discover_master(master_name)
else:
host, port = random.choice([sentinel.discover_master(master_name)] + sentinel.discover_slaves(master_name))
sentinel_options = self._options.copy()
sentinel_options['CONNECTION_POOL_CLASS'] = 'redis.sentinel.SentinelConnectionPool'
sentinel_options['CONNECTION_POOL_KWARGS'] = {
'service_name': master_name,
'sentinel_manager': sentinel}
self.connection_factory = get_connection_factory(options=sentinel_options)
return self.connection_factory.connect(host, port, db)
def connect(self, index=0, write=True):
"""
Creates a redis connection with connection pool.
"""
master_name, sentinel_hosts, db = self.parse_connection_string(
self._connection_string)
sentinel_timeout = self._options.get('SENTINEL_TIMEOUT', 1)
sentinel = Sentinel(sentinel_hosts, socket_timeout=sentinel_timeout)
if write:
host, port = sentinel.discover_master(master_name)
else:
host, port = random.choice(
[sentinel.discover_master(master_name)] +
sentinel.discover_slaves(master_name))
kwargs = {
"db": db,
"parser_class": self.parser_class,
"password": self._options.get('PASSWORD', None),
}
kwargs.update({
'host': host,
'port': port,
'connection_class': Connection
})
if 'SOCKET_TIMEOUT' in self._options:
kwargs.update(
{'socket_timeout': int(self._options['SOCKET_TIMEOUT'])})
kwargs.update(self._pool_cls_kwargs)
connection_pool = get_or_create_connection_pool(
self._pool_cls, **kwargs)
connection = Redis(connection_pool=connection_pool)
return connection
class RedisSentinel(object):
"""docstring for RedisSentinel"""
def __init__(self, sentinel_addrs):
super(RedisSentinel, self).__init__()
self.m_sentinel = Sentinel(sentinel_addrs)
self.m_sentinelIns = self.m_sentinel.sentinels[0]
self.m_masters = self.get_masters()
self.m_slaves = self.get_slaves()
def get_masters(self):
masters = {}
masterDetail = self.m_sentinelIns.sentinel_masters()
for name in masterDetail:
masters[name] = (masterDetail[name]['ip'],
masterDetail[name]['port'])
return masters
def get_slaves(self):
slaves = {}
for name in self.m_masters:
slave = self.m_sentinel.discover_slaves(name)
if len(slave) > 0:
slaves[name] = (slave[0][0], slave[0][1])
return slaves
import time
import random
import traceback
from redis.sentinel import Sentinel
sentinel = Sentinel([("redis-sentinel-1", 26379), ("redis-sentinel-2", 26379), ("redis-sentinel-3", 26379)],
socket_timeout=0.1)
while True:
try:
# 发现查看下master和slave
master, slave = sentinel.discover_master('mymaster'), sentinel.discover_slaves("mymaster")
print("master: %s, slave: %s" % (master, slave))
# 获取master
m = sentinel.master_for("mymaster", socket_timeout=0.1, password='******')
s = sentinel.slave_for("mymaster", socket_timeout=0.1, password="******")
value = random.randint(1, 1000)
m.set("test_int", value)
r = s.get("test_int")
print("set test_int: %s, get test_int: %s" % (value, r))
print("沉睡中...")
except:
print(traceback.format_exc())
time.sleep(5)
目前支持的鉴权方式有 "simple" 和 "hmacsha1" 两种，默认使用 "simple"鉴权。
所有方法均可添加关键字参数sign_type修改鉴权方式。
"""
import redis,random
import kdl,requests
from redis.sentinel import Sentinel
sentinel = Sentinel([('192.168.17.65', 26379),
('192.168.17.66', 26379),
('192.168.17.67', 26379)
], socket_timeout=0.5)
# 查看master节点
master = sentinel.discover_master('ida_redis_master')
# 查看slave 节点
slave = sentinel.discover_slaves('ida_redis_master')
# 连接数据库
rds = sentinel.master_for('ida_redis_master', socket_timeout=0.5, db=7, decode_responses=True)
# rds = redis.StrictRedis(host='192.168.17.60', port=6378, db=7, decode_responses=True)
def get_proxy_from_redis():
try:
one_proxy = rds.randomkey()
username = "******"
password = "******"
proxies = {
"http": "http://%(user)s:%(pwd)s@%(ip)s/" % {'user': username, 'pwd': password, 'ip': one_proxy},
"https": "http://%(user)s:%(pwd)s@%(ip)s/" % {'user': username, 'pwd': password, 'ip': one_proxy}
}
return proxies
except Exception as e:
import time
from redis.sentinel import Sentinel
sentinel = Sentinel([('192.168.11.35', 26379), ('192.168.11.37', 26379),
('192.168.11.37', 26379)],
socket_timeout=0.1)
for i in range(1, 1000):
print "ciclo " + str(i)
try:
print "master: " + str(sentinel.discover_master('mymaster'))
print "slave/s: " + str(sentinel.discover_slaves('mymaster'))
master = sentinel.master_for('mymaster', socket_timeout=0.1)
slave = sentinel.slave_for('mymaster', socket_timeout=0.1)
except:
print "sentinel eligiendo a un master"
clave = "clave" + str(i)
valor = "valor" + str(i)
try:
print "master.set " + clave + " " + valor
master.set(clave, valor)
print "master.get " + clave + " = " + str(master.get(clave))
print "master.set " + clave + " = " + str(slave.get(clave))
print "master.delete " + clave
master.delete(clave)
except:
class Database:
"""
Provide an interface using which to communicate with a Redis server.
A new connection to the server is created by each class instance.
Custom exception hierarchy:
Error
+-- ConnectionError
+-- WaitTimeoutError
"""
socket_timeout = 1
pubsubs = {}
sentinel = None
class Error(Exception):
pass
# print "Error"
class ConnectionError(Error):
pass
# print "ConnectionError"
class WaitTimeoutError(Error):
def __str__(self):
return self.msg
def __init__(self, msg, key, value, timeout):
msg = str(msg) # in the event msg was an Exception, for example
self.msg = msg
self.key = key
self.value = value
self.timeout = timeout
super(RedisClient.WaitTimeoutError, self).__init__(self, msg) # this raises the exception
# Note: In Python 2.x, self.__class__ should not be used in place of base class name.
def state_connection(self):
"""
Return a dict containing keys and values providing information about
the state of the connection to the Redis server.
"""
if self.logger:
self.logger.debug("state_connection")
# conn_kwargs = self._pool.connection_kwargs
# server_addr = (conn_kwargs["host"], conn_kwargs["port"])
# state = {"Server address": "{}:{}".format(*server_addr),}
state = {}
# Do ping test
try:
# connection = redis.Redis(connection_pool=self._pool)
state["Server pingable"] = self.redis.ping() # returns True
except redis.exceptions.ConnectionError: # as e:
#self.logger.error("{} Console Redis ping failure: {}".format(tools.cm_name(), e))
# intentionally not self.logger.exception
state["Server pingable"] = False
# # Test availability of a key
# try:
# self["MONO_SV"]
# except self.ConnectionError: # as e:
# #self.logger.error("{} Console Redis test failure: {}".format(tools.cm_name(), e))
# # intentionally not self.logger.exception
# state["Test result"] = "Failed"
# else:
# state["Test result"] = "Passed"
return state
def state_server(self):
"""Return a dict containing keys and values providing some properties
about the the Redis server."""
if self.logger:
self.logger.debug("state_server")
try:
# connection = redis.Redis(connection_pool=self._pool)
info = self.redis.info()
except redis.exceptions.ConnectionError as e:
if self.logger:
self.logger.error("Redis info acquisition failure: {}".format(e))
# intentionally not self.logger.exception
state = {}
else:
state = {"Number of connected clients": info["connected_clients"],
"Redis version": info["redis_version"],
"Uptime": datetime.datetime.utcnow() - datetime.datetime.utcfromtimestamp(time.time() - info["uptime_in_seconds"]),
}
return state
def __init__(self,
host="localhost",
port=6379,
db=0,
password=None,
sentinels=None,
master=None,
settings=None,
logger=False):
"""Initialize the client."""
# Save passed-in variables
self.host = host
self.port = port
self.db = db
self.password = password
self.sentinels = sentinels
self.master = master
self.logger = logger
# Use the RAPD CONTROL_DATABASE_SETTINGS object
if settings:
# Sentinel
if settings["REDIS_CONNECTION"] == "sentinel":
self.host = None
self.port = None
self.db = None
self.password = settings.get("REDIS_PASSWORD", None)
self.sentinels = settings["REDIS_SENTINEL_HOSTS"]
self.master = settings["REDIS_MASTER_NAME"]
# Standard
else:
self.host = settings["REDIS_HOST"]
self.port = settings["REDIS_PORT"]
self.db = settings["REDIS_DB"]
self.password = settings.get("REDIS_PASSWORD", None)
self.sentinels = None
self.master = None
self._ConnectionError_last_log_time = float("-inf")
# Connect to server
self._connect_lock = threading.Lock()
self._connect() # "with self._connect_lock:" is not necessary in __init__
atexit.register(self._disconnect)
###############
# ADMIN Methods
###############
def _connect(self):
"""Connect to server."""
# self.logger.debug(tools.cm_name())
socket_timeout = self.socket_timeout
# Sentinel connection
if self.sentinels:
self.sentinel = Sentinel(self.sentinels)
# Assign master connection to self.redis
self.redis = self.sentinel.master_for(self.master)
# Standard connection
else:
attempt_count = 0
while attempt_count < CONNECTION_ATTEMPT_LIMIT:
try:
# Connect
self.redis = redis.Redis(host=self.host,
port=self.port,
db=self.db,
password=self.password)
# self._pool = redis.ConnectionPool(host=self.host, port=self.port, db=0)
# Make sure redis server is up
_ = self.redis.ping()
break
# self._connection = redis.Redis(host, socket_timeout=socket_timeout)
# redis module reconnects automatically as needed.
# socket_timeout is not set to a higher value because it blocks.
except redis.exceptions.ConnectionError as e:
print "except"
if self.logger:
self.logger.debug("Connection error {}".format(e))
time.sleep(1)
attempt_count += 1
# self._raise_ConnectionError(e)
# # Make sure redis server is up
# try:
# _ = self.redis.ping()
# except redis.sentinel.MasterNotFoundError as error:
# self._raise_ConnectionError(error)
def _disconnect(self):
"""Disconnect from server."""
# self.logger.debug(tools.cm_name())
try:
pool = self._pool
except AttributeError:
pass
else:
pool.disconnect()
#self.logger.info("{} Disconnected Redis client from Redis server {host}:{port}.".format(tools.cm_name(), **connection.connection_pool.connection_kwargs))
def _raise_ConnectionError(self, exc):
"""
Raise ConnectionError exception with an error message corresponding
to the provided exception.
"""
# TODO: Implement functionality of _raise_ConnectionError directly into ConnectionError exception instead. Delete method definition.
err_msg = str(exc)
if err_msg: err_msg = ": {}".format(err_msg)
err_msg = "Redis connection error{}".format(err_msg)
# Conditionally log error
# log_interval = backend.settings.redis["Console"]["ConnectionError log interval"]
# if time.time() > (self._ConnectionError_last_log_time + log_interval):
# self._ConnectionError_last_log_time = time.time()
# logger = self.logger.info
# else:
if self.logger:
logger = self.logger.debug
logger("{}".format(err_msg))
raise self.ConnectionError(err_msg)
def discover_master(self):
"""
Return the master instance (host, ip)
"""
if self.sentinel and self.master:
attempts = 0
while attempts < ATTEMPT_LIMIT:
try:
attempts += 1
master = self.sentinel.discover_master(self.master)
break
except redis.sentinel.MasterNotFoundError as e:
# Pause for specified time
print "try %d" % attempts
time.sleep(ATTEMPT_PAUSE)
else:
self._raise_ConnectionError(e)
return master
else:
self._raise_ConnectionError("Sentinels not properly defined")
def discover_slaves(self):
"""
Return the slave instances [(host, ip),]
"""
if self.sentinel and self.master:
attempts = 0
while attempts < ATTEMPT_LIMIT:
try:
attempts += 1
master = self.sentinel.discover_slaves(self.master)
break
except redis.sentinel.SlaveNotFoundError as e:
# Pause for specified time
print "try %d" % attempts
time.sleep(ATTEMPT_PAUSE)
else:
self._raise_ConnectionError(e)
return master
else:
self._raise_ConnectionError("Sentinels not properly defined")
#############
# PIPELINE Methods
#############
def pipeline(self):
"""return a pipeline instance"""
return self.redis.pipeline()
#############
# GET Methods
#############
def __getitem__(self, key, return_dict=False):
"""
Return the value of the specified key(s).
key can be either a str, or a tuple or list of strs. It is recommended
that key(s) be present in backend.redis_console_keys.keys.
`transform` indicates whether to apply transformations to the value
before returning it, as defined in the self._transform_value method.
`return_dict` indicates whether to return values in a dict. If True,
the keys in the returned dict match the specified key(s).
"""
# Get value(s)
if isinstance(key, str):
return self.get(key, return_dict)
elif isinstance(key, tuple) or isinstance(key, list):
return self.mget(key, return_dict)
get = __getitem__
@connectionErrorWrapper
def get(self, key, return_dict=False):
"""
Return the value of `key`.
`return_dict` indicates whether to return values in a dict. If True,
the keys in the returned dict match the specified key(s).
"""
# self.logger.debug("get key:{} return_dict:{}".format(key, return_dict))
# Retrieve value
value = self.redis.get(key)
# Wrap in dict if requested
return ({key: value} if return_dict else value)
# def get(self, key, return_dict=False):
# """
# Return the value of `key`.
# `return_dict` indicates whether to return values in a dict. If True,
# the keys in the returned dict match the specified key(s).
# """
# # self.logger.debug("get key:{} return_dict:{}".format(key, return_dict))
# # Retrieve value
# attempts = 0
# while attempts < ATTEMPT_LIMIT:
# try:
# attempts += 1
# value = self.redis.get(key)
# break
# except redis.exceptions.ConnectionError as e:
# # Pause for specified time
# print "try %d" % attempts
# time.sleep(ATTEMPT_PAUSE)
# else:
# self._raise_ConnectionError(e)
# return ({key: value} if return_dict else value)
@connectionErrorWrapper
def mget(self, keys, return_dict=False):
"""
Return the values of the sequence `keys`.
`transform` indicates whether to apply transformations to the value
before returning it, as defined in the self._transform_value method.
`return_dict` indicates whether to return values in a dict. If True,
the keys in the returned dict match the specified key(s).
"""
# Retrieve values
values = self.redis.mget(keys)
return (dict(zip(keys,values)) if return_dict else values)
# def mget(self, keys, return_dict=False):
# """
# Return the values of the sequence `keys`.
# `transform` indicates whether to apply transformations to the value
# before returning it, as defined in the self._transform_value method.
# `return_dict` indicates whether to return values in a dict. If True,
# the keys in the returned dict match the specified key(s).
# """
# # self.logger.log(5, "{} keys:{} transform:{} return_dict:{}".format(tools.cm_name(), keys, transform, return_dict))
# # Retrieve values
# try:
# connection = redis.Redis(connection_pool=self._pool)
# values = connection.mget(keys)
# except redis.exceptions.ConnectionError as e:
# self._raise_ConnectionError(e)
# # self.logger.log(5 if transform else logging.DEBUG, "{} keys:{} values:{}".format(tools.cm_name(), keys, repr(values)))
# # Transform and return values
# # if transform: values = self._transform_value(keys, values)
# return (dict(zip(keys,values)) if return_dict else values)
#############
# SET Methods
#############
@connectionErrorWrapper
def set(self, key, value):
"""
Set the indicated key value pair.
"""
if self.logger:
self.logger.debug("set key:{} value:{}".format(key, value))
self.redis.set(key, str(value))
# def set(self, key, value):
# """
# Set the indicated key value pair.
# """
# self.logger.debug("set key:{} value:{}".format(key, value))
# # Set
# attempts = 0
# while attempts < ATTEMPT_LIMIT:
# try:
# attempts += 1
# self.redis.set(key, str(value))
# break
# except redis.exceptions.ConnectionError as error:
# # Pause for specified time
# print "try %d" % attempts
# time.sleep(ATTEMPT_PAUSE)
# else:
# self._raise_ConnectionError(error)
__setitem__ = set
@connectionErrorWrapper
def mset(self, mapping):
"""
Set the key value pairs which are in the dict `mapping`.
"""
# self.logger.debug("{} mapping:{}".format(tools.cm_name(), mapping))
# Get the mapping ready
mapping = {k: str(v) for k, v in mapping.items()}
# Set in redis
self.redis.mset(mapping)
# def mset(self, mapping):
# """
# Set the key value pairs which are in the dict `mapping`.
# """
# # self.logger.debug("{} mapping:{}".format(tools.cm_name(), mapping))
# mapping = {k: str(v) for k, v in mapping.items()}
# # Set
# attempts = 0
# while attempts < ATTEMPT_LIMIT:
# try:
# attempts += 1
# self.redis.mset(mapping)
# break
# except redis.exceptions.ConnectionError as error:
# # Pause for specified time
# # print "try %d" % attempts
# time.sleep(ATTEMPT_PAUSE)
# else:
# self._raise_ConnectionError(error)
@connectionErrorWrapper
def setex(self, key, expire_time, value):
"""
Set the indicated key value pair with expire time.
"""
#if self.logger:
# self.logger.debug("set key:{} value:{}".format(key, value))
self.redis.setex(key, int(expire_time), value)
#############
# KEY Methods
#############
@connectionErrorWrapper
def keys(self, template):
"""Searches for keys fitting template."""
#self.logger.debug("set key:{} value:{}".format(key, value))
return self.redis.keys(template)
@connectionErrorWrapper
def delete(self, key):
"""Delete a key"""
#self.logger.debug("set key:{} value:{}".format(key, value))
return self.redis.delete(key)
##############
# LIST Methods
##############
@connectionErrorWrapper
def llen(self, key):
"""
LLEN get length of list
"""
value = self.redis.llen(key)
return value
@connectionErrorWrapper
def lpop(self, key):
"""
LPOP a value off a given list
"""
value = self.redis.lpop(key)
return value
@connectionErrorWrapper
def lpush(self, key, value):
"""
LPUSH a values onto a given list
"""
self.redis.lpush(key, value)
@connectionErrorWrapper
def brpop(self, key):
"""
RPOP a value off a given list
"""
value = self.redis.brpop(key)
return value
@connectionErrorWrapper
def rpop(self, key):
"""
RPOP a value off a given list
"""
value = self.redis.rpop(key)
return value
@connectionErrorWrapper
def rpoplpush(self, list1, list2):
"""
RPOPLPUSH pop a value off a given list and push on another list
"""
value = self.redis.rpoplpush(list1, list2)
#return value
##############
# HASH Methods
##############
@connectionErrorWrapper
def hget(self, key, field):
"""
HGET field on a key
"""
return self.redis.hget(key, field)
@connectionErrorWrapper
def hgetall(self, key):
"""
HGETALL a mapping on a key
"""
return self.redis.hgetall(key)
@connectionErrorWrapper
def hmset(self, key, mapping):
"""
HMSET a key with the attached mapping
"""
self.redis.hmset(key, mapping)
@connectionErrorWrapper
def hset(self, key, field, value):
"""
Sets field in the hash stored at key to value.
"""
self.redis.hset(key, field, value)
################
# PUBSUB Methods
################
@connectionErrorWrapper
def publish(self, key, value):
"""
Publish a value on a given key
"""
# print "publish {} {}".format(key, value)
self.redis.publish(key, value)
####################
# EXPIRATION Methods
####################
@connectionErrorWrapper
def expire(self, key, seconds):
"""
Set a timeout on key
"""
self.redis.expire(key, seconds)
# def publish(self, key, value):
# """
# Publish a value on a given key
# """
# # print "publish {} {}".format(key, value)
# attempts = 0
# while attempts < ATTEMPT_LIMIT:
# try:
# attempts += 1
# self.redis.publish(key, value)
# break
# except redis.exceptions.ConnectionError as error:
# # Pause for specified time
# print "try %d" % attempts
# time.sleep(ATTEMPT_PAUSE)
# else:
# self._raise_ConnectionError(error)
@connectionErrorWrapper
def get_message(self, id):
"""
Get message on pubsub connection
"""
message = self.pubsubs[id].get_message()
return message
# def get_message(self, id):
# """
# Get message on pubsub connection
# """
# attempts = 0
# while attempts < ATTEMPT_LIMIT:
# try:
# attempts += 1
# message = self.pubsubs[id].get_message()
# break
# except redis.exceptions.ConnectionError as error:
# # Pause for specified time
# print "try %d" % attempts
# time.sleep(ATTEMPT_PAUSE)
# else:
# self._raise_ConnectionError(error)
# return message
def get_pubsub(self):
"""
Returns a pubsub connection - no error is triggered if no connection
"""
# Create the pubsub object
ps = self.redis.pubsub()
# Give it an id
id = len(self.pubsubs) + 1
ps._id = id
# Store a reference
self.pubsubs[id] = ps
# Return the id
return id
@connectionErrorWrapper
def psubscribe(self, id=False, pattern=False):
"""
Add pattern subscription to a pubsub object and return id of pubsub object
"""
if pattern:
# No id passed in - create pubsub object
if not id:
id = self.get_pubsub()
self.pubsubs[id].psubscribe(pattern)
return id
else:
raise TypeError("pattern must be specified")
# def psubscribe(self, id=False, pattern=False):
# """
# Add pattern subscription to a pubsub object and return id of pubsub object
# """
# # No id passed in - create pubsub object
# if not id:
# id = self.get_pubsub()
# attempts = 0
# while attempts < ATTEMPT_LIMIT:
# try:
# attempts += 1
# self.pubsubs[id].psubscribe(pattern)
# break
# except redis.exceptions.ConnectionError as error:
# # Pause for specified time
# # print "try %d" % attempts
# time.sleep(ATTEMPT_PAUSE)
# else:
# self._raise_ConnectionError(error)
# return id
@connectionErrorWrapper
def subscribe(self, id=None, channel=None):
"""
Add subscription to a pubsub object and return id of pubsub object
"""
if channel:
# No id passed in - create pubsub object
if not id:
id = self.get_pubsub()
self.pubsubs[id].subscribe(channel)
return id
else:
raise TypeError("channel must be specified")
class SentinelConnectionFactory(ConnectionFactory):
# Creates Sentinel Client from connection params
# It does not cache anything
def __init__(self, options):
super().__init__(options)
pool_cls_path = options.get(
'CONNECTION_POOL_CLASS',
'redis.sentinel.SentinelConnectionPool',
)
self.pool_cls = djredis_util.load_class(pool_cls_path)
self.pool_cls_kwargs = options.get('CONNECTION_POOL_KWARGS', {})
redis_client_cls_path = options.get(
'REDIS_CLIENT_CLASS',
'redis.client.StrictRedis',
)
self.redis_client_cls = djredis_util.load_class(redis_client_cls_path)
self.redis_client_cls_kwargs = options.get('REDIS_CLIENT_KWARGS', {})
self.service_name = options.get('SENTINEL_SERVICE_NAME', None)
if not self.service_name:
raise ImproperlyConfigured(
'SentinelClient requires SENTINEL_SERVICE_NAME in OPTIONS')
# Get sentinels servers from options (even though it's not an option...)
self.sentinels = options.get('SENTINELS', [])
self.options = options
# Sentinel Connection Pool is not cached, not indexed by URL,
# so params are constant for each connection
self.sentinel_conn_pool_cls_kwargs = {
'parser_class': self.get_parser_cls()
}
password = self.options.get('PASSWORD', None)
if password:
self.sentinel_conn_pool_cls_kwargs['password'] = password
socket_timeout = self.options.get('SOCKET_TIMEOUT', None)
if socket_timeout:
assert isinstance(
socket_timeout,
(int, float)), 'Socket timeout should be float or integer'
self.sentinel_conn_pool_cls_kwargs[
'socket_timeout'] = socket_timeout
socket_connect_timeout = self.options.get('SOCKET_CONNECT_TIMEOUT',
None)
if socket_connect_timeout:
assert isinstance(
socket_connect_timeout,
(int,
float)), 'Socket connect timeout should be float or integer'
self.sentinel_conn_pool_cls_kwargs[
'socket_connect_timeout'] = socket_connect_timeout
# Actual Sentinel client, it is responsible of creating the StrictRedis clients
self._sentinel = Sentinel(self.sentinels,
**self.sentinel_conn_pool_cls_kwargs)
# Returns a list of current slaves
self._has_slaves = len(
self._sentinel.discover_slaves(self.service_name))
def has_slaves(self):
return self._has_slaves
def connect_master(self):
"""
Given a basic connection parameters and sentinel client,
return a new master connection.
:raises MasterNotFoundError: if no master available
then raises this
"""
return self._sentinel.master_for(self.service_name,
**self.redis_client_cls_kwargs)
def connect_slave(self, force_slave=False):
"""
Given a basic connection parameters and sentinel client,
return a new slave connection if available, master's if not
:raises SlaveNotFoundError: it automatically fallback to master if not slaves available,
if nobody available then raises this
"""
if self.has_slaves() or force_slave:
return self._sentinel.slave_for(self.service_name,
**self.redis_client_cls_kwargs)
else:
# If the cluster had no slaves when creating the pool
# then no need for callbacks and unnecessary discoveries, fall back
# to master directly
return self.connect_master()
import hashlib
from rediscluster import StrictRedisCluster
from SIT_tool.pyflask.lib.readconfig import getredis
sentinel = Sentinel([('gis-ass-oms-gk1.cachesit.sfdc.com.cn', 8001),
('gis-ass-oms-gk2.cachesit.sfdc.com.cn', 8001),
('gis-ass-oms-gk3.cachesit.sfdc.com.cn', 8001)],
socket_timeout=1)
# 获取主服务器地址
master = sentinel.discover_master('GIS_ASS_OMS_REDIS_GK_C01')
print(master)
# 输出：('192.168.196.132', 6379)
# 获取从服务器地址
slave = sentinel.discover_slaves('GIS_ASS_OMS_REDIS_GK_C01')
print(slave)
# 输出：[('192.168.196.129', 6379)]
# 获取主服务器进行写入
master = sentinel.master_for('GIS_ASS_OMS_REDIS_GK_C01',
socket_timeout=0.5,
password='******',
db=2)
w_ret = master.set('foo', 'bar')
# 输出：True
# 获取从服务器进行读取（默认是round-roubin,随机从多个slave服务中读取数据）
slave = sentinel.slave_for('GIS_ASS_OMS_REDIS_GK_C01',
socket_timeout=0.5,
password='******',
import time
from redis.sentinel import Sentinel
sentinel = Sentinel([('192.168.11.35', 26379),('192.168.11.37', 26379),('192.168.11.37', 26379)], socket_timeout=0.1)
for i in range(1,1000):
print "ciclo " + str(i)
try:
print "master: " + str(sentinel.discover_master('mymaster'))
print "slave/s: " + str(sentinel.discover_slaves('mymaster'))
master = sentinel.master_for('mymaster', socket_timeout=0.1)
slave = sentinel.slave_for('mymaster', socket_timeout=0.1)
except:
print "sentinel eligiendo a un master"
clave = "clave" + str(i)
valor = "valor" + str(i)
try:
print "master.set " + clave + " " + valor
master.set(clave, valor)
print "master.get " + clave + " = " + str(master.get(clave))
print "master.set " + clave + " = " + str(slave.get(clave))
print "master.delete " + clave
master.delete(clave)
except:
print "cluster en modo solo lectura"
from redis.sentinel import Sentinel
sentinel = Sentinel([("127.0.0.1", 26379), ("127.0.0.1", 26380),
("127.0.0.1", 26381)],
socket_timeout=0.1)
print(sentinel.discover_master("mymaster"))
print(sentinel.discover_slaves("mymaster"))
sentinel = Sentinel([('localhost', 55511)], socket_timeout=0.1)
if sentinel.discover_master(NAME)[1] != 33311:
print 'unexpected situation: expecting port 33311 to be the master'
sys.exit(99)
print """>> Getting an High Availability connection through the sentinel.
"""
mha = sentinel.master_for(NAME)
if int(mha.config_get()['port']) != 33311:
print 'unexpected master. expecting master to be on port 33311.'
sys.exit(99)
assert mha.ping() and a.ping()
print """>> Currently: (MASTER:{})<==(SLAVES:{})""".format(sentinel.discover_master(NAME),
sentinel.discover_slaves(NAME))
print """>> Stopping A instance (current master)
"""
# err = subprocess.call('./stop-master.sh')
# if err:
# print 'error stopping master instance.'
# sys.exit(99)
raw_input('./stop-master.sh')
print ">> waiting for A to stop."
try:
while a.ping():
pass
except ConnectionError:
print '>> instance A stopped.'
# https://www.throwable.club/2019/10/07/redis-server-sentinel-install-guide/
import pprint
from redis.sentinel import Sentinel
sentinel = Sentinel([('127.0.0.1', '26379')])
master = sentinel.discover_master("doge-master")
slaver = sentinel.discover_slaves("doge-master")
pprint.pprint(master)
pprint.pprint(slaver)
master = sentinel.master_for("doge-master")
slaver = sentinel.slave_for("doge-master")
pprint.pprint(master.set("a", 1))
pprint.pprint(slaver.info("server").get("tcp_port"))
import redis
import time
from redis.sentinel import Sentinel
sentinel = Sentinel([('192.168.55.31', 26379),('192.168.55.32', 26379),('192.168.55.33', 26379)],socket_timeout=0.5)
master = sentinel.discover_master('sdemo')
print(master)
slave = sentinel.discover_slaves('sdemo')
print(slave)
for i in range(1, 1000):
key = "k%d" % i
value = "v%d" % i
try:
master = sentinel.master_for('sdemo', socket_timeout=0.5, password='******', db=0)
slave = sentinel.slave_for('sdemo', socket_timeout=0.5, password='******', db=0)
master.set(key, value)
ret = slave.get(key)
print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime((time.time()))), ret)
except Exception as e:
print(str(e))
time.sleep(1)
from redis.sentinel import Sentinel, MasterNotFoundError
while True:
import time
try:
sentinel = Sentinel([('redis-sentinel', 26379)], socket_timeout=0.1)
dm = sentinel.discover_master('dpf')
ds = sentinel.discover_slaves('dpf')
print('master:', dm, '====>', 'slaves', ds)
master = sentinel.master_for('dpf', socket_timeout=0.1)
master.set('foo', time.asctime())
slave = sentinel.slave_for('dpf', socket_timeout=0.1)
print('get from redis:', slave.get('foo'))
except MasterNotFoundError as e:
print('redis master is GONE, waiting for failover...', e)
except Exception as e:
print('exception happened, connection lost?', e)
time.sleep(1.0)
if __name__ == "__main__":
# change logging config
logging.basicConfig(level=logging.DEBUG,
format='[%(asctime)s.%(msecs)03d][%(filename)s:%(lineno)d][%(levelname)s]%(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
logger = logging.getLogger(__file__)
# redis
sentinels = [
('redis-sentinel-1', 26379),
('redis-sentinel-2', 26379),
('redis-sentinel-3', 26379)
]
while True:
try:
sentinel = Sentinel(sentinels, socket_timeout=0.1)
logger.debug('Cluster Master: %s', str(sentinel.discover_master('mymaster')))
logger.debug('Cluster Slave: %s', str(sentinel.discover_slaves('mymaster')))
master = sentinel.master_for('mymaster', socket_timeout=0.1, password='******')
slave = sentinel.slave_for('mymaster', socket_timeout=0.1, password='******')
master.set('timestamp', datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
logger.debug('Reading key(timestamp): %s', slave.get('timestamp'))
except Exception as ex:
logger.error(str(ex))
finally:
time.sleep(1)
class Database:
"""
Provide an interface using which to communicate with a Redis server.
A new connection to the server is created by each class instance.
Custom exception hierarchy:
Error
+-- ConnectionError
+-- WaitTimeoutError
"""
socket_timeout = 1
pubsubs = {}
sentinel = None
class Error(Exception):
pass
# print "Error"
class ConnectionError(Error):
pass
# print "ConnectionError"
class WaitTimeoutError(Error):
def __str__(self):
return self.msg
def __init__(self, msg, key, value, timeout):
msg = str(msg) # in the event msg was an Exception, for example
self.msg = msg
self.key = key
self.value = value
self.timeout = timeout
super(RedisClient.WaitTimeoutError,
self).__init__(self, msg) # this raises the exception
# Note: In Python 2.x, self.__class__ should not be used in place of base class name.
def state_connection(self):
"""
Return a dict containing keys and values providing information about
the state of the connection to the Redis server.
"""
if self.logger:
self.logger.debug("state_connection")
# conn_kwargs = self._pool.connection_kwargs
# server_addr = (conn_kwargs["host"], conn_kwargs["port"])
# state = {"Server address": "{}:{}".format(*server_addr),}
state = {}
# Do ping test
try:
# connection = redis.Redis(connection_pool=self._pool)
state["Server pingable"] = self.redis.ping() # returns True
except redis.exceptions.ConnectionError: # as e:
#self.logger.error("{} Console Redis ping failure: {}".format(tools.cm_name(), e))
# intentionally not self.logger.exception
state["Server pingable"] = False
# # Test availability of a key
# try:
# self["MONO_SV"]
# except self.ConnectionError: # as e:
# #self.logger.error("{} Console Redis test failure: {}".format(tools.cm_name(), e))
# # intentionally not self.logger.exception
# state["Test result"] = "Failed"
# else:
# state["Test result"] = "Passed"
return state
def state_server(self):
"""Return a dict containing keys and values providing some properties
about the the Redis server."""
if self.logger:
self.logger.debug("state_server")
try:
# connection = redis.Redis(connection_pool=self._pool)
info = self.redis.info()
except redis.exceptions.ConnectionError as e:
if self.logger:
self.logger.error(
"Redis info acquisition failure: {}".format(e))
# intentionally not self.logger.exception
state = {}
else:
state = {
"Number of connected clients":
info["connected_clients"],
"Redis version":
info["redis_version"],
"Uptime":
datetime.datetime.utcnow() -
datetime.datetime.utcfromtimestamp(time.time() -
info["uptime_in_seconds"]),
}
return state
def __init__(self,
host="localhost",
port=6379,
db=0,
password=None,
sentinels=None,
master=None,
settings=None,
logger=False):
"""Initialize the client."""
# Save passed-in variables
self.host = host
self.port = port
self.db = db
self.password = password
self.sentinels = sentinels
self.master = master
self.logger = logger
# Use the RAPD CONTROL_DATABASE_SETTINGS object
if settings:
# Sentinel
if settings["REDIS_CONNECTION"] == "sentinel":
self.host = None
self.port = None
self.db = None
self.password = settings.get("REDIS_PASSWORD", None)
self.sentinels = settings["REDIS_SENTINEL_HOSTS"]
self.master = settings["REDIS_MASTER_NAME"]
# Standard
else:
self.host = settings["REDIS_HOST"]
self.port = settings["REDIS_PORT"]
self.db = settings["REDIS_DB"]
self.password = settings.get("REDIS_PASSWORD", None)
self.sentinels = None
self.master = None
self._ConnectionError_last_log_time = float("-inf")
# Connect to server
self._connect_lock = threading.Lock()
self._connect(
) # "with self._connect_lock:" is not necessary in __init__
atexit.register(self._disconnect)
###############
# ADMIN Methods
###############
def _connect(self):
"""Connect to server."""
# self.logger.debug(tools.cm_name())
socket_timeout = self.socket_timeout
# Sentinel connection
if self.sentinels:
self.sentinel = Sentinel(self.sentinels)
# Assign master connection to self.redis
self.redis = self.sentinel.master_for(self.master)
# Standard connection
else:
attempt_count = 0
while attempt_count < CONNECTION_ATTEMPT_LIMIT:
try:
# Connect
self.redis = redis.Redis(host=self.host,
port=self.port,
db=self.db,
password=self.password)
# self._pool = redis.ConnectionPool(host=self.host, port=self.port, db=0)
# Make sure redis server is up
_ = self.redis.ping()
break
# self._connection = redis.Redis(host, socket_timeout=socket_timeout)
# redis module reconnects automatically as needed.
# socket_timeout is not set to a higher value because it blocks.
except redis.exceptions.ConnectionError as e:
print "except"
if self.logger:
self.logger.debug("Connection error {}".format(e))
time.sleep(1)
attempt_count += 1
# self._raise_ConnectionError(e)
# # Make sure redis server is up
# try:
# _ = self.redis.ping()
# except redis.sentinel.MasterNotFoundError as error:
# self._raise_ConnectionError(error)
def _disconnect(self):
"""Disconnect from server."""
# self.logger.debug(tools.cm_name())
try:
pool = self._pool
except AttributeError:
pass
else:
pool.disconnect()
#self.logger.info("{} Disconnected Redis client from Redis server {host}:{port}.".format(tools.cm_name(), **connection.connection_pool.connection_kwargs))
def _raise_ConnectionError(self, exc):
"""
Raise ConnectionError exception with an error message corresponding
to the provided exception.
"""
# TODO: Implement functionality of _raise_ConnectionError directly into ConnectionError exception instead. Delete method definition.
err_msg = str(exc)
if err_msg: err_msg = ": {}".format(err_msg)
err_msg = "Redis connection error{}".format(err_msg)
# Conditionally log error
# log_interval = backend.settings.redis["Console"]["ConnectionError log interval"]
# if time.time() > (self._ConnectionError_last_log_time + log_interval):
# self._ConnectionError_last_log_time = time.time()
# logger = self.logger.info
# else:
if self.logger:
logger = self.logger.debug
logger("{}".format(err_msg))
raise self.ConnectionError(err_msg)
def discover_master(self):
"""
Return the master instance (host, ip)
"""
if self.sentinel and self.master:
attempts = 0
while attempts < ATTEMPT_LIMIT:
try:
attempts += 1
master = self.sentinel.discover_master(self.master)
break
except redis.sentinel.MasterNotFoundError as e:
# Pause for specified time
print "try %d" % attempts
time.sleep(ATTEMPT_PAUSE)
else:
self._raise_ConnectionError(e)
return master
else:
self._raise_ConnectionError("Sentinels not properly defined")
def discover_slaves(self):
"""
Return the slave instances [(host, ip),]
"""
if self.sentinel and self.master:
attempts = 0
while attempts < ATTEMPT_LIMIT:
try:
attempts += 1
master = self.sentinel.discover_slaves(self.master)
break
except redis.sentinel.SlaveNotFoundError as e:
# Pause for specified time
print "try %d" % attempts
time.sleep(ATTEMPT_PAUSE)
else:
self._raise_ConnectionError(e)
return master
else:
self._raise_ConnectionError("Sentinels not properly defined")
#############
# PIPELINE Methods
#############
def pipeline(self):
"""return a pipeline instance"""
return self.redis.pipeline()
#############
# GET Methods
#############
def __getitem__(self, key, return_dict=False):
"""
Return the value of the specified key(s).
key can be either a str, or a tuple or list of strs. It is recommended
that key(s) be present in backend.redis_console_keys.keys.
`transform` indicates whether to apply transformations to the value
before returning it, as defined in the self._transform_value method.
`return_dict` indicates whether to return values in a dict. If True,
the keys in the returned dict match the specified key(s).
"""
# Get value(s)
if isinstance(key, str):
return self.get(key, return_dict)
elif isinstance(key, tuple) or isinstance(key, list):
return self.mget(key, return_dict)
get = __getitem__
@connectionErrorWrapper
def get(self, key, return_dict=False):
"""
Return the value of `key`.
`return_dict` indicates whether to return values in a dict. If True,
the keys in the returned dict match the specified key(s).
"""
# self.logger.debug("get key:{} return_dict:{}".format(key, return_dict))
# Retrieve value
value = self.redis.get(key)
# Wrap in dict if requested
return ({key: value} if return_dict else value)
# def get(self, key, return_dict=False):
# """
# Return the value of `key`.
# `return_dict` indicates whether to return values in a dict. If True,
# the keys in the returned dict match the specified key(s).
# """
# # self.logger.debug("get key:{} return_dict:{}".format(key, return_dict))
# # Retrieve value
# attempts = 0
# while attempts < ATTEMPT_LIMIT:
# try:
# attempts += 1
# value = self.redis.get(key)
# break
# except redis.exceptions.ConnectionError as e:
# # Pause for specified time
# print "try %d" % attempts
# time.sleep(ATTEMPT_PAUSE)
# else:
# self._raise_ConnectionError(e)
# return ({key: value} if return_dict else value)
@connectionErrorWrapper
def mget(self, keys, return_dict=False):
"""
Return the values of the sequence `keys`.
`transform` indicates whether to apply transformations to the value
before returning it, as defined in the self._transform_value method.
`return_dict` indicates whether to return values in a dict. If True,
the keys in the returned dict match the specified key(s).
"""
# Retrieve values
values = self.redis.mget(keys)
return (dict(zip(keys, values)) if return_dict else values)
# def mget(self, keys, return_dict=False):
# """
# Return the values of the sequence `keys`.
# `transform` indicates whether to apply transformations to the value
# before returning it, as defined in the self._transform_value method.
# `return_dict` indicates whether to return values in a dict. If True,
# the keys in the returned dict match the specified key(s).
# """
# # self.logger.log(5, "{} keys:{} transform:{} return_dict:{}".format(tools.cm_name(), keys, transform, return_dict))
# # Retrieve values
# try:
# connection = redis.Redis(connection_pool=self._pool)
# values = connection.mget(keys)
# except redis.exceptions.ConnectionError as e:
# self._raise_ConnectionError(e)
# # self.logger.log(5 if transform else logging.DEBUG, "{} keys:{} values:{}".format(tools.cm_name(), keys, repr(values)))
# # Transform and return values
# # if transform: values = self._transform_value(keys, values)
# return (dict(zip(keys,values)) if return_dict else values)
#############
# SET Methods
#############
@connectionErrorWrapper
def set(self, key, value):
"""
Set the indicated key value pair.
"""
if self.logger:
self.logger.debug("set key:{} value:{}".format(key, value))
self.redis.set(key, str(value))
# def set(self, key, value):
# """
# Set the indicated key value pair.
# """
# self.logger.debug("set key:{} value:{}".format(key, value))
# # Set
# attempts = 0
# while attempts < ATTEMPT_LIMIT:
# try:
# attempts += 1
# self.redis.set(key, str(value))
# break
# except redis.exceptions.ConnectionError as error:
# # Pause for specified time
# print "try %d" % attempts
# time.sleep(ATTEMPT_PAUSE)
# else:
# self._raise_ConnectionError(error)
__setitem__ = set
@connectionErrorWrapper
def mset(self, mapping):
"""
Set the key value pairs which are in the dict `mapping`.
"""
# self.logger.debug("{} mapping:{}".format(tools.cm_name(), mapping))
# Get the mapping ready
mapping = {k: str(v) for k, v in mapping.items()}
# Set in redis
self.redis.mset(mapping)
# def mset(self, mapping):
# """
# Set the key value pairs which are in the dict `mapping`.
# """
# # self.logger.debug("{} mapping:{}".format(tools.cm_name(), mapping))
# mapping = {k: str(v) for k, v in mapping.items()}
# # Set
# attempts = 0
# while attempts < ATTEMPT_LIMIT:
# try:
# attempts += 1
# self.redis.mset(mapping)
# break
# except redis.exceptions.ConnectionError as error:
# # Pause for specified time
# # print "try %d" % attempts
# time.sleep(ATTEMPT_PAUSE)
# else:
# self._raise_ConnectionError(error)
@connectionErrorWrapper
def setex(self, key, expire_time, value):
"""
Set the indicated key value pair with expire time.
"""
#if self.logger:
# self.logger.debug("set key:{} value:{}".format(key, value))
self.redis.setex(key, int(expire_time), value)
#############
# KEY Methods
#############
@connectionErrorWrapper
def keys(self, template):
"""Searches for keys fitting template."""
#self.logger.debug("set key:{} value:{}".format(key, value))
return self.redis.keys(template)
@connectionErrorWrapper
def delete(self, key):
"""Delete a key"""
#self.logger.debug("set key:{} value:{}".format(key, value))
return self.redis.delete(key)
##############
# LIST Methods
##############
@connectionErrorWrapper
def llen(self, key):
"""
LLEN get length of list
"""
value = self.redis.llen(key)
return value
@connectionErrorWrapper
def lpop(self, key):
"""
LPOP a value off a given list
"""
value = self.redis.lpop(key)
return value
@connectionErrorWrapper
def lpush(self, key, value):
"""
LPUSH a values onto a given list
"""
self.redis.lpush(key, value)
@connectionErrorWrapper
def brpop(self, key):
"""
RPOP a value off a given list
"""
value = self.redis.brpop(key)
return value
@connectionErrorWrapper
def rpop(self, key):
"""
RPOP a value off a given list
"""
value = self.redis.rpop(key)
return value
@connectionErrorWrapper
def rpoplpush(self, list1, list2):
"""
RPOPLPUSH pop a value off a given list and push on another list
"""
value = self.redis.rpoplpush(list1, list2)
#return value
##############
# HASH Methods
##############
@connectionErrorWrapper
def hget(self, key, field):
"""
HGET field on a key
"""
return self.redis.hget(key, field)
@connectionErrorWrapper
def hgetall(self, key):
"""
HGETALL a mapping on a key
"""
return self.redis.hgetall(key)
@connectionErrorWrapper
def hmset(self, key, mapping):
"""
HMSET a key with the attached mapping
"""
self.redis.hmset(key, mapping)
@connectionErrorWrapper
def hset(self, key, field, value):
"""
Sets field in the hash stored at key to value.
"""
self.redis.hset(key, field, value)
################
# PUBSUB Methods
################
@connectionErrorWrapper
def publish(self, key, value):
"""
Publish a value on a given key
"""
# print "publish {} {}".format(key, value)
self.redis.publish(key, value)
####################
# EXPIRATION Methods
####################
@connectionErrorWrapper
def expire(self, key, seconds):
"""
Set a timeout on key
"""
self.redis.expire(key, seconds)
# def publish(self, key, value):
# """
# Publish a value on a given key
# """
# # print "publish {} {}".format(key, value)
# attempts = 0
# while attempts < ATTEMPT_LIMIT:
# try:
# attempts += 1
# self.redis.publish(key, value)
# break
# except redis.exceptions.ConnectionError as error:
# # Pause for specified time
# print "try %d" % attempts
# time.sleep(ATTEMPT_PAUSE)
# else:
# self._raise_ConnectionError(error)
@connectionErrorWrapper
def get_message(self, id):
"""
Get message on pubsub connection
"""
message = self.pubsubs[id].get_message()
return message
# def get_message(self, id):
# """
# Get message on pubsub connection
# """
# attempts = 0
# while attempts < ATTEMPT_LIMIT:
# try:
# attempts += 1
# message = self.pubsubs[id].get_message()
# break
# except redis.exceptions.ConnectionError as error:
# # Pause for specified time
# print "try %d" % attempts
# time.sleep(ATTEMPT_PAUSE)
# else:
# self._raise_ConnectionError(error)
# return message
def get_pubsub(self):
"""
Returns a pubsub connection - no error is triggered if no connection
"""
# Create the pubsub object
ps = self.redis.pubsub()
# Give it an id
id = len(self.pubsubs) + 1
ps._id = id
# Store a reference
self.pubsubs[id] = ps
# Return the id
return id
@connectionErrorWrapper
def psubscribe(self, id=False, pattern=False):
"""
Add pattern subscription to a pubsub object and return id of pubsub object
"""
if pattern:
# No id passed in - create pubsub object
if not id:
id = self.get_pubsub()
self.pubsubs[id].psubscribe(pattern)
return id
else:
raise TypeError("pattern must be specified")
# def psubscribe(self, id=False, pattern=False):
# """
# Add pattern subscription to a pubsub object and return id of pubsub object
# """
# # No id passed in - create pubsub object
# if not id:
# id = self.get_pubsub()
# attempts = 0
# while attempts < ATTEMPT_LIMIT:
# try:
# attempts += 1
# self.pubsubs[id].psubscribe(pattern)
# break
# except redis.exceptions.ConnectionError as error:
# # Pause for specified time
# # print "try %d" % attempts
# time.sleep(ATTEMPT_PAUSE)
# else:
# self._raise_ConnectionError(error)
# return id
@connectionErrorWrapper
def subscribe(self, id=None, channel=None):
"""
Add subscription to a pubsub object and return id of pubsub object
"""
if channel:
# No id passed in - create pubsub object
if not id:
id = self.get_pubsub()
self.pubsubs[id].subscribe(channel)
return id
else:
raise TypeError("channel must be specified")
# 3.配置文件 bind 不能为127.0.0.1 应该是 0.0.0.0
from redis.sentinel import Sentinel
# 连接哨兵服务器(主机名也可以用域名)
sentinel = Sentinel([('192.168.56.101', 26379)], socket_timeout=0.5)
# 获取主服务器地址
master = sentinel.discover_master('mymaster')
print(master)
# 输出：('172.31.0.2', 5001)
# 获取从服务器地址
slave = sentinel.discover_slaves('mymaster')
print(slave)
# 输出：[('172.31.3', 5001), ('172.31.0.4', 5001), ('172.31.0.5', 5001)]
# 获取主服务器进行写入
master = sentinel.master_for('mymaster', socket_timeout=0.5, db=15)
w_ret = master.set('foo', 'bar')
# 输出：True
# # 获取从服务器进行读取（默认是round-roubin）
slave = sentinel.slave_for('mymaster', socket_timeout=0.5, db=15)
r_ret = slave.get('foo')
print(r_ret)
# -*- coding: utf-8 -*-
from redis.sentinel import Sentinel
REDIS_SENTINEL_SERVICE_NAME = 'mymaster'
# 指定sentinel的地址和端口号
sentinel = Sentinel([('192.168.1.24', 26382), ('192.168.1.24', 26383),
('192.168.1.24', 26384), ('192.168.1.24', 26385)],
socket_timeout=0.01)
# 测试，获取以下主库和从库的信息
print sentinel.discover_master(REDIS_SENTINEL_SERVICE_NAME)
print sentinel.discover_slaves('mymaster')
##配置读写分离
#写节点
master = sentinel.master_for(REDIS_SENTINEL_SERVICE_NAME, socket_timeout=0.01)
#读节点
slave = sentinel.slave_for(REDIS_SENTINEL_SERVICE_NAME, socket_timeout=0.01)
###读写分离测试 key
print master.set('oldboy', '123')
print slave.get('oldboy')
# -*- coding:utf-8 -*-
from redis.sentinel import Sentinel
# 连接哨兵服务器(主机名也可以用域名)
sentinel = Sentinel([('172.31.0.2', 5001), ('172.31.0.3', 5001),
('172.31.0.4', 5001), ('172.31.0.5', 5001)],
socket_timeout=10000)
# 获取主服务器地址
master = sentinel.discover_master('mymaster')
print(master)
# 输出：('172.31.0.2', 5001)
# 获取从服务器地址
slave = sentinel.discover_slaves('mymaster')
print(slave)
# 输出：[('172.31.3', 5001), ('172.31.0.4', 5001), ('172.31.0.5', 5001)]
# 获取主服务器进行写入
master = sentinel.master_for('mymaster',
socket_timeout=0.5,
password='******',
db=15)
w_ret = master.set('foo', 'bar')
# 输出：True
# # 获取从服务器进行读取（默认是round-roubin）
slave = sentinel.slave_for('mymaster',
socket_timeout=0.5,
password='******',
#!/usr/bin/python
from redis.sentinel import Sentinel
sentinel = Sentinel([('localhost', 19001)], socket_timeout=0.1)
master = sentinel.discover_master('redis-cluster')
slaves = sentinel.discover_slaves('redis-cluster')
print ""
print "Master(" + str(master) + ")"
print ""
print "Slaves(" + str(len(slaves)) + ") Nodes(" + str(slaves) + ")"
print ""
from redis.sentinel import Sentinel
from config import config
sentinel = Sentinel([
('192.168.39.57',26379),
('192.168.39.110',26379),
],
socket_timeout=0.5,
)
# 获取主服务器地址
master = sentinel.discover_master(config.REDIS['sentinel']['master'])
print(master)
# 获取从服务器地址
slave = sentinel.discover_slaves(config.REDIS['sentinel']['master'])
print(slave)
# 获取主服务器进行写入mq
def write_mq(theme, msg):
master_write = sentinel.master_for(config.REDIS['sentinel']['master'],
socket_timeout=0.5,
password=config.REDIS['password'],
db=config.REDIS['database']
)
return master_write.set(theme, msg)
# 获取从服务器进行读取
def read_mq(theme):
from redis.sentinel import Sentinel
from redisgraph import Node, Edge, Graph
sentinel = Sentinel([('localhost', 26379)])
#sentinel = Sentinel([('io-madstat-prod-redis-redis-ha.redis', 26379)], socket_timeout=0.1)
print "MASTERS > {0}".format(sentinel.discover_master('mymaster'))
print "SLAVES > {0}".format(sentinel.discover_slaves('mymaster'))
slave = sentinel.slave_for('mymaster', socket_timeout=0.3)
redis_graph = Graph('bulk', slave)
query = "MATCH (t:Tag {name: 'odin'}) RETURN t"
result = redis_graph.query(query)
result.pretty_print()
import redis # pip install redis
from redis.sentinel import Sentinel
import random
bind_ip = "localhost"
master_port = 7000
slave_ports = [7001, 7002]
master_name = "mymaster"
sentinel_hosts = [(bind_ip, 5000), (bind_ip, 5001), (bind_ip, 5002)]
# TODO: intend to split reads and writes
master = redis.Redis(host=bind_ip, port=master_port, db=0)
clients = [redis.Redis(host=bind_ip, port=port) for port in slave_ports]
master.set("foo", "bar")
assert "bar" == master.get("foo"), "get foo not return bar"
client = random.choice(clients)
assert "bar" == client.get("foo"), "get foo not return bar"
master.delete("foo")
assert None == client.get("foo"), "get foo not return None"
print("test set get delete ok")
sentinel = Sentinel(sentinel_hosts, socket_timeout=0.1)
print(sentinel.discover_master(master_name))
print(sentinel.discover_slaves(master_name))
master = sentinel.master_for(master_name, socket_timeout=0.1)
slave = sentinel.slave_for(master_name, socket_timeout=0.1)
master.set('foo1', 'bar1')
print(slave.get('foo1'))
def add_master_slave_stats(self):
master_normal = 1
switch_over = 0
slaves_list = []
master_host = ''
sentinel_list = [(node.get('host', 'localhost'),
node.get('sentinel_port', 26379))
for node in self.node_list]
sentinel = Sentinel(sentinel_list, socket_timeout=5)
try:
master_host, master_port = sentinel.discover_master(
self.master_name)
last_master = ''
with open(self.tmp_master_file, 'r') as f:
last_master = f.readline().strip()
if last_master != master_host:
switch_over = 1
with open(self.tmp_master_file, 'w') as f:
f.write(master_host)
except MasterNotFoundError:
master_normal = 0
try:
master_conn = sentinel.master_for(self.master_name,
db=self.db,
password=self.password)
set_data = random.randint(0, 10)
if master_conn.set('newrelic_redis_cluster_agent', set_data):
if master_conn.get('newrelic_redis_cluster_agent') != str(
set_data):
master_normal = 0
else:
master_normal = 0
except MasterNotFoundError:
master_normal = 0
slaves_list = sentinel.discover_slaves(self.master_name)
slaves_hosts = [slave[0] for slave in slaves_list]
for node in self.node_list:
host = node.get('host', 'localhost')
status = 0
if host == master_host:
status = 2
if host in slaves_hosts:
status = 1
self.add_gauge_value('Redis_Cluster/ClusterStatus/%s' % host,
None,
status,
count=1)
self.add_gauge_value('Redis_Cluster/SlavesNum',
None,
len(slaves_list),
count=1)
self.add_gauge_value('Redis_Cluster/MasterStatus',
None,
master_normal,
count=1)
self.add_gauge_value('Redis_Cluster/SwitchOver',
None,
switch_over,
count=1)
