def init_app(self, app):
socket_timeout = app.config.get('REDIS_SOCKET_TIMEOUT', None)
retry_on_timeout = app.config.get('REDIS_RETRY_ON_TIMEOUT', True)
if (app.config.get('REDIS_MODE') == 'master'):
self.master = StrictRedis(
db=0,
host=app.config.get('REDIS_MK_MASTER'),
port=app.config.get('REDIS_PORT'),
password=app.config.get('REDIS_PASSWORD'),
ssl=app.config.get('REDIS_SSL'),
socket_timeout=socket_timeout,
retry_on_timeout=retry_on_timeout)
self.slave = StrictRedis(db=0,
host=app.config.get('REDIS_MK_SLAVE'),
port=app.config.get('REDIS_PORT'),
password=app.config.get('REDIS_PASSWORD'),
ssl=app.config.get('REDIS_SSL'),
socket_timeout=socket_timeout,
retry_on_timeout=retry_on_timeout)
else:
self.connection = sentinel.Sentinel(
app.config['REDIS_SENTINEL'],
socket_timeout=socket_timeout,
retry_on_timeout=retry_on_timeout)
redis_db = app.config.get('REDIS_DB') or 0
redis_master = app.config.get('REDIS_MASTER') or 'mymaster'
self.master = self.connection.master_for(redis_master, db=redis_db)
self.slave = self.connection.slave_for(redis_master, db=redis_db)
def _sentinel_managed_pool(self, asynchronous=False):
connparams = self._connparams(asynchronous)
additional_params = connparams.copy()
additional_params.pop('host', None)
additional_params.pop('port', None)
sentinels = []
for url in self.connection.client.alt:
url = _parse_url(url)
if url.scheme == 'sentinel':
port = url.port or self.connection.default_port
sentinels.append((url.hostname, port))
# Fallback for when only one sentinel is provided.
if not sentinels:
sentinels.append((connparams['host'], connparams['port']))
sentinel_inst = sentinel.Sentinel(
sentinels,
min_other_sentinels=getattr(self, 'min_other_sentinels', 0),
sentinel_kwargs=getattr(self, 'sentinel_kwargs', None),
**additional_params)
master_name = getattr(self, 'master_name', None)
if master_name is None:
raise ValueError(
"'master_name' transport option must be specified.")
return sentinel_inst.master_for(
master_name,
self.Client,
).connection_pool
def init_app(self, app):
self.connection = sentinel.Sentinel(app.config['REDIS_SENTINEL'],
socket_timeout=0.1)
redis_db = app.config.get('REDIS_DB') or 0
redis_master = app.config.get('REDIS_MASTER') or 'mymaster'
self.master = self.connection.master_for(redis_master, db=redis_db)
self.slave = self.connection.slave_for(redis_master, db=redis_db)
def _sentinel_managed_pool(self, asynchronous=False):
connparams = self._connparams(asynchronous)
additional_params = connparams.copy()
additional_params.pop("host", None)
additional_params.pop("port", None)
sentinels = []
for url in self.connection.client.alt:
url = _parse_url(url)
if url.scheme == "sentinel":
sentinels.append((url.hostname, url.port))
# Fallback for when only one sentinel is provided.
if not sentinels:
sentinels.append((connparams["host"], connparams["port"]))
sentinel_inst = sentinel.Sentinel(
sentinels,
min_other_sentinels=getattr(self, "min_other_sentinels", 0),
sentinel_kwargs=getattr(self, "sentinel_kwargs", None),
**additional_params)
master_name = getattr(self, "master_name", None)
return sentinel_inst.master_for(
master_name,
self.Client,
).connection_pool
def get_client(conf, scripts=None):
if redis is None:
raise RuntimeError("Redis Python module is unavailable")
parsed_url = parse.urlparse(conf.redis_url)
options = parse.parse_qs(parsed_url.query)
kwargs = {}
if parsed_url.hostname:
kwargs['host'] = parsed_url.hostname
if parsed_url.port:
kwargs['port'] = parsed_url.port
else:
if not parsed_url.path:
raise ValueError("Expected socket path in parsed urls path")
kwargs['unix_socket_path'] = parsed_url.path
if parsed_url.password:
kwargs['password'] = parsed_url.password
for a in CLIENT_ARGS:
if a not in options:
continue
if a in CLIENT_BOOL_ARGS:
v = utils.strtobool(options[a][-1])
elif a in CLIENT_LIST_ARGS:
v = options[a]
elif a in CLIENT_INT_ARGS:
v = int(options[a][-1])
else:
v = options[a][-1]
kwargs[a] = v
# Ask the sentinel for the current master if there is a
# sentinel arg.
if 'sentinel' in kwargs:
sentinel_hosts = [
tuple(fallback.split(':'))
for fallback in kwargs.get('sentinel_fallback', [])
]
sentinel_hosts.insert(0, (kwargs['host'], kwargs['port']))
sentinel_server = sentinel.Sentinel(
sentinel_hosts, socket_timeout=kwargs.get('socket_timeout'))
sentinel_name = kwargs['sentinel']
del kwargs['sentinel']
if 'sentinel_fallback' in kwargs:
del kwargs['sentinel_fallback']
master_client = sentinel_server.master_for(sentinel_name, **kwargs)
# The master_client is a redis.StrictRedis using a
# Sentinel managed connection pool.
return master_client
client = redis.StrictRedis(**kwargs)
if scripts is not None:
scripts = {
name: client.register_script(code)
for name, code in six.iteritems(scripts)
}
return client, scripts
def get_sentinel():
global _sentinel
if not _sentinel:
options = settings.SENTINEL_OPTIONS
options.setdefault('socket_timeout', settings.SOCKET_TIMEOUT)
_sentinel = sentinel.Sentinel(settings.SENTINELS, **options)
return _sentinel
def init_app(self, app):
socket_timeout = app.config.get('REDIS_SOCKET_TIMEOUT', None)
retry_on_timeout = app.config.get('REDIS_RETRY_ON_TIMEOUT', True)
self.connection = sentinel.Sentinel(app.config['REDIS_SENTINEL'],
socket_timeout=socket_timeout,
retry_on_timeout=retry_on_timeout)
redis_db = app.config.get('REDIS_DB') or 0
redis_master = app.config.get('REDIS_MASTER') or 'redis-master'
self.master = self.connection.master_for(redis_master, db=redis_db)
self.slave = self.connection.slave_for(redis_master, db=redis_db)
def __connect(self, redis_config):
try:
self.connection = sentinel.Sentinel(
[(redis_config["master_host"], redis_config["master_port"]),
(redis_config["slave_1_host"], redis_config["slave_1_port"]),
(redis_config["slave_2_host"], redis_config["slave_2_port"]),
(redis_config["slave_3_host"], redis_config["slave_3_port"])],
min_other_sentinels=2,
encoding="utf-8",
decode_responses=True)
except RedisError as err:
error_str = "Error while connecting to redis : " + str(err)
sys.exit(error_str)
def create_redis_client(db=None):
c = conf.memdb
if db is None:
db = c.db_main_index
if conf.test:
db += c.db_index_test_offset
if c.sentinel:
sent = sentinel.Sentinel(c.hosts,
socket_timeout=c.sentinel_timeout,
socket_keepalive=True)
return sent.master_for(c.sentinel, socket_timeout=c.timeout, db=db)
else:
host, port = c.hosts[0]
return StrictRedis(host=host, port=port, db=db, socket_keepalive=True)
def get_redis_pool(redis_conf, redis_sentinel_conf):
"""
@param redis_conf: 针对整个redis配置都更改的情况
@return: redis连接池
"""
if redis_sentinel_conf['use_sentinel']:
redis_sentinel = sentinel.Sentinel(
[(redis_conf['host'], redis_conf['port'])], socket_timeout=5)
return sentinel.SentinelConnectionPool(
redis_sentinel_conf['master_name'],
redis_sentinel,
db=redis_conf['db'],
password=redis_conf['password'],
max_connections=redis_conf['max_connections'])
else:
return redis.BlockingConnectionPool(**redis_conf)
class SentinelChannel(Channel):
"""Channel with explicit Redis Sentinel knowledge.
Broker url is supposed to look like:
sentinel://0.0.0.0:26379;sentinel://0.0.0.0:26380/...
where each sentinel is separated by a `;`. Multiple sentinels are handled
by :class:`kombu.Connection` constructor, and placed in the alternative
list of servers to connect to in case of connection failure.
Other arguments for the sentinel should come from the transport options
(see :method:`Celery.connection` which is in charge of creating the
`Connection` object).
You must provide at least one option in Transport options:
* `service_name` - name of the redis group to poll
"""
from_transport_options = Channel.from_transport_options + (
'master_name',
'min_other_sentinels',
'sentinel_kwargs')
connection_class = sentinel.SentinelManagedConnection if sentinel else None
def _sentinel_managed_pool(self, async=False):
connparams = self._connparams(async)
additional_params = connparams.copy()
del additional_params['host']
del additional_params['port']
sentinel_inst = sentinel.Sentinel(
[(connparams['host'], connparams['port'])],
min_other_sentinels=getattr(self, 'min_other_sentinels', 0),
sentinel_kwargs=getattr(self, 'sentinel_kwargs', {}),
**additional_params
)
master_name = getattr(self, 'master_name', None)
return sentinel_inst.master_for(
master_name,
self.Client,
).connection_pool
def _make_client(cls, parsed_url, options, default_socket_timeout):
kwargs = {}
if parsed_url.hostname:
kwargs['host'] = parsed_url.hostname
if parsed_url.port:
kwargs['port'] = parsed_url.port
else:
if not parsed_url.path:
raise ValueError("Expected socket path in parsed urls path")
kwargs['unix_socket_path'] = parsed_url.path
if parsed_url.password:
kwargs['password'] = parsed_url.password
for a in cls.CLIENT_ARGS:
if a not in options:
continue
if a in cls.CLIENT_BOOL_ARGS:
v = strutils.bool_from_string(options[a])
elif a in cls.CLIENT_LIST_ARGS:
v = options[a]
elif a in cls.CLIENT_INT_ARGS:
v = int(options[a])
else:
v = options[a]
kwargs[a] = v
if 'socket_timeout' not in kwargs:
kwargs['socket_timeout'] = default_socket_timeout
# Ask the sentinel for the current master if there is a
# sentinel arg.
if 'sentinel' in kwargs:
sentinel_hosts = [
tuple(fallback.split(':'))
for fallback in kwargs.get('sentinel_fallback', [])
]
sentinel_hosts.insert(0, (kwargs['host'], kwargs['port']))
sentinel_server = sentinel.Sentinel(
sentinel_hosts,
socket_timeout=kwargs['socket_timeout'])
sentinel_name = kwargs['sentinel']
del kwargs['sentinel']
if 'sentinel_fallback' in kwargs:
del kwargs['sentinel_fallback']
master_client = sentinel_server.master_for(sentinel_name, **kwargs)
# The master_client is a redis.StrictRedis using a
# Sentinel managed connection pool.
return master_client
return redis.StrictRedis(**kwargs)
def _sentinel_managed_pool(self, asynchronous=False):
connparams = self._connparams(asynchronous)
additional_params = connparams.copy()
additional_params.pop('host', None)
additional_params.pop('port', None)
sentinel_inst = sentinel.Sentinel(
[(connparams['host'], connparams['port'])],
min_other_sentinels=getattr(self, 'min_other_sentinels', 0),
sentinel_kwargs=getattr(self, 'sentinel_kwargs', {}),
**additional_params)
master_name = getattr(self, 'master_name', None)
return sentinel_inst.master_for(
master_name,
self.Client,
).connection_pool
def init_app(self, app):
socket_timeout = app.config.get('REDIS_SOCKET_TIMEOUT', None)
retry_on_timeout = app.config.get('REDIS_RETRY_ON_TIMEOUT', True)
redis_db = app.config.get('REDIS_DB', 0)
redis_master = app.config.get('REDIS_MASTER', 'mymaster')
if app.config.get('REDIS_SENTINEL'):
self.connection = sentinel.Sentinel(
app.config['REDIS_SENTINEL'],
socket_timeout=socket_timeout,
retry_on_timeout=retry_on_timeout)
self.master = self.connection.master_for(redis_master, db=redis_db)
self.slave = self.connection.slave_for(redis_master, db=redis_db)
else:
self.connection = None
self.master = StrictRedis(
host=app.config.get('REDIS_HOST', 'localhost'),
port=app.config.get('REDIS_PORT', 6379),
db=redis_db,
password=app.config.get('REDIS_PASSWORD'))
self.slave = self.master
def init_app(self, app):
conn_kwargs = {
'db': app.config.get('REDIS_DB') or 0,
'password': app.config.get('REDIS_PWD')
}
if app.config.get('REDIS_MASTER_DNS') and \
app.config.get('REDIS_SLAVE_DNS') and \
app.config.get('REDIS_PORT'):
self.master = StrictRedis(host=app.config['REDIS_MASTER_DNS'],
port=app.config['REDIS_PORT'],
**conn_kwargs)
self.slave = StrictRedis(host=app.config['REDIS_SLAVE_DNS'],
port=app.config['REDIS_PORT'],
**conn_kwargs)
else:
self.connection = sentinel.Sentinel(app.config['REDIS_SENTINEL'],
socket_timeout=0.1)
redis_master = app.config.get('REDIS_MASTER') or 'mymaster'
self.master = self.connection.master_for(redis_master,
**conn_kwargs)
self.slave = self.connection.slave_for(redis_master, **conn_kwargs)
def _sentinel_managed_pool(self, asynchronous=False):
connparams = self._connparams(asynchronous)
additional_params = connparams.copy()
additional_params.pop('host', None)
additional_params.pop('port', None)
connection_list = []
for url in self.connection.client.alt:
if url and 'sentinel://' in url:
connection_list.append(url.split('/')[2].split(':'))
sentinel_inst = sentinel.Sentinel(
connection_list,
min_other_sentinels=getattr(self, 'min_other_sentinels', 0),
sentinel_kwargs=getattr(self, 'sentinel_kwargs', None),
**additional_params)
master_name = getattr(self, 'master_name', None)
return sentinel_inst.master_for(
master_name,
self.Client,
).connection_pool
def _generate_sentinel(self, redis_kwargs):
# pass redis_kwargs through to the sentinel object to be used for each connection to the redis servers.
return sentinel.Sentinel(self.hosts, **redis_kwargs)
from redis import sentinel
import traceback
from urllib.parse import parse_qs
from conf.constants import *
from utils.logger_conf import configure_logger
from utils.get_conf import *
log_path = get_logger_file()
log_path = log_path + 'author.log'
author_log = configure_logger('author', log_path)
try:
if if_redis_sentinel:
_hosts = [hp.split(':') for hp in pre_rs_hosts]
hosts = [(hp[0].strip(), int(hp[1].strip())) for hp in _hosts]
r_sentinel = sentinel.Sentinel(hosts, socket_timeout=r_timeout)
redis_cli = r_sentinel.master_for(r_master,
socket_timeout=r_timeout,
password=r_password,
db=r_database,
decode_responses=True)
else:
redis_cli = redis.StrictRedis(host=r_host,
port=r_port,
db=r_database,
password=r_password,
decode_responses=True)
except (KeyError, ValueError, IndexError) as e:
author_log.exception('wrong configure pattern')
exit(0)
import redis
import logging.config
from redis import sentinel
logging.config.fileConfig('cnf/logging.conf')
# 连接哨兵
stinel = sentinel.Sentinel(sentinels=[('192.168.161.10', 26379)])
if stinel.discover_master('mymaster') != ('192.168.161.10', 6381):
logging.getLogger('rotate')
logging.warning('master change to {}'.format(
stinel.discover_master('mymaster')))
from redis import sentinel
import logging.config
import redis
# 指定日志格式的配置文件位置
logging.config.fileConfig('./cnf/logging.conf')
# 连接哨兵
stinel = sentinel.Sentinel([('192.168.43.100', 26379)])
# ------------------1.测试方式一----------------------------------------------------
if stinel.discover_master('mymaster') != ('192.168.43.100', 6380):
logging.getLogger('rotate')
logging.warning('！！！master is on {}'.format(stinel.discover_master('mymaster')))
else:
print('now master is on {}'.format(stinel.discover_master('mymaster')))
# -------------------2.测试方式二-----------------------------------------------------
# 捕捉异常，当主机刚宕机，从机正在选票时会发送错误：找不到主机
try:
master = stinel.discover_master('mymaster')
print(master)
except redis.sentinel.MasterNotFoundError as error:
logging.getLogger('rotate')
logging.error(error)
# 采用钉钉报警的方式通知运维人员dingtalk(error)
finally:
master = stinel.discover_master('mymaster')
def init_app(self, app):
self.connection = sentinel.Sentinel(app.config['REDIS_SENTINEL'],
socket_timeout=0.1)
self.master = self.connection.master_for('mymaster')
self.slave = self.connection.slave_for('mymaster')
"""
TODO: 完成redis sentinel
redis master/slave
redis_sentinel conf
logfile /va/log/redis-sentinel.log
daemonize yes
port num
bind ""
sentinel monitor mymaster ip port 2
sentinel
"""
REDIS_SENTINELS = [
('127.0.0.1', '26379'),
('127.0.0.1', '26378'),
('127.0.0.1', '26377'),
]
REDIS_SENTINEL_SERVICE_NAME = 'mymaster'
_sentinel = sentinel.Sentinel(REDIS_SENTINELS)
redis_master = _sentinel.master_for(REDIS_SENTINEL_SERVICE_NAME) # rw
redis_slave = _sentinel.slave_for(REDIS_SENTINEL_SERVICE_NAME) # r
import redis
import logging.config
from redis import sentinel
logging.config.fileConfig('./logconfig.conf')
stinel = sentinel.Sentinel([('192.168.40.142',26379)])
try:
master = stinel.discover_master('mymaster')
print(master)
except redis.sentinel.MasterNotFoundError as error:
logging.getLogger('rotate')
logging.error(error)
finally:
master = stinel.discover_master('mymaster')
if master != ('192.168.40.142',6380):
logging.warning('master on {}'.format(master))
def record(log_file, log_level, redis_password, redis_port, sentinel_port, sentinel_name, commands):
logger=log.Logger(log_file, log_level)
db=database.db()
logger.logger.debug("记录Redis资源")
record_time=datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
port=int(redis_port)
pid=tools.find_pid(port)
normal=1
if pid==0:
logger.logger.error(f"Redis({port})未运行")
sql="insert into redis_constant(record_time, pid, port, boot_time) values(?, ?, ?, ?)"
db.update_one(sql, (record_time, pid, port, "0"))
db.update_one("update redis_role set record_time=?, role=?", ("0", "master"))
else:
try:
conn=Redis(host="127.0.0.1", port=redis_port, password=redis_password)
conn.ping()
except Exception as e:
normal=0
msg=e
if normal==1:
redis_info=conn.info()
redis_psutil_info=psutil.Process(pid).as_dict()
boot_time=datetime.datetime.fromtimestamp(redis_psutil_info["create_time"]).strftime("%Y-%m-%d %H:%M:%S")
sql="insert into redis_constant(record_time, pid, port, boot_time) values(?, ?, ?, ?)"
db.update_one(sql, (record_time, pid, redis_port, boot_time))
redis_memory_percent=redis_psutil_info['memory_percent']
redis_memory=psutil.virtual_memory()[0] * redis_memory_percent / 100
redis_connections=redis_info['connected_clients']
redis_num_threads=redis_psutil_info['num_threads']
sql="insert into redis_variable values(?, ?, ?, ?, ?, ?)"
db.update_one(sql, (record_time, pid, redis_memory, redis_memory_percent, redis_connections, redis_num_threads))
logger.logger.debug("记录Redis集群资源")
role=redis_info['role']
db.update_one("update redis_role set record_time=?, role=?", (record_time, role))
if role=="master":
connected_slaves=redis_info['connected_slaves']
sql="replace into redis_master values(?, ?, ?, ?)"
db.update_one(sql, (record_time, pid, role, connected_slaves))
slaves_list=[]
if connected_slaves!=0:
for i in range(connected_slaves):
slave=f"slave{i}"
slaves_list.append((record_time, redis_info[slave]['ip'], redis_info[slave]['port'], redis_info[slave]['state']))
sql="replace into redis_slaves_info values(?, ?, ?, ?)"
db.update_all(sql, slaves_list)
elif role=="slave":
sql="replace into redis_slave values(?, ?, ?, ?, ?, ?)"
db.update_one(sql, (record_time, pid, role, redis_info['master_host'], redis_info['master_port'], redis_info['master_link_status']))
"""显示自定义命令
if commands is not None:
printf("-"*40)
printf("自定义命令查询:")
commands_list=commands.split(",")
for command in commands_list:
command=command.strip()
result=conn.execute_command(command)
printf(f"{command} 结果: {result}")
"""
conn.close()
elif normal==0:
logger.logger.error(f"无法连接redis: {msg}")
sql="insert into error values(?, ?, ?, ?, ?)"
db.update_one(sql, (record_time, "Redis", "connection", str(msg), 0))
# sentinel信息
if sentinel_port is not None:
logger.logger.debug(f"记录Redis Sentinel信息...")
conn=sentinel.Sentinel(
[('127.0.0.1', sentinel_port)],
socket_timeout=1
)
try:
sentinel_info=[]
master=conn.discover_master(sentinel_name)
sentinel_info.append((record_time, 'master', master[0], master[1]))
slaves=conn.discover_slaves(sentinel_name)
for i in slaves:
sentinel_info.append((record_time, 'slave', i[0], i[1]))
sql="replace into redis_sentinel values(?, ?, ?, ?)"
db.update_all(sql, sentinel_info)
except Exception as e:
logger.logger.error(f"Redis Sentinel无法连接...")
sql="insert into error values(?, ?, ?, ?, ?)"
db.update_one(sql, (record_time, 'Sentinel', "connection", str(e), 0))
from redis import sentinel
import redis
import logging.config
logging.config.fileConfig('./logging_conf')
stinel = sentinel.Sentinel([('192.168.52.134', 26379)])
try:
master = stinel.discover_master('mymaster')
print(master)
except redis.sentinel.MasterNotFoundError as error:
logging.getLogger('rotate')
logging.error(error)
finally:
master = stinel.discover_master('mymaster')
if master != ('192.168.52.134', 6380):
logging.warning('master on {}'.format(master))