def get_db():
redis_store = getattr(g, '_database', None)
if redis_store is None:
if 'FAKE_DB' in app.config and app.config['FAKE_DB']:
from mockredis import mock_redis_client
redis_store = g._database = mock_redis_client(
encoding='utf-8', decode_responses=True)
else:
if app.config['REDIS_URL'].startswith('redis://'):
redis_store = g._database = Redis.from_url(
app.config['REDIS_URL'],
encoding='utf-8',
decode_responses=True)
elif app.config['REDIS_URL'].startswith('sentinel://'):
parsed_url = urlparse(app.config['REDIS_URL'])
service = parsed_url.path.lstrip('/')
port = 26379
if ':' in parsed_url.netloc:
host, str_port = parsed_url.netloc.split(':')
port = int(str_port)
else:
host = parsed_url.netloc
sentinel = Sentinel([(host, port)], socket_timeout=0.1)
redis_store = sentinel.master_for(service,
redis_class=Redis,
socket_timeout=0.1)
return redis_store
def __init__(self, name, namespace="queue", **redis_kwargs):
import redis # pylint: disable=import-error
from redis.sentinel import Sentinel # pylint: disable=import-error
redis_kwargs["decode_responses"] = True
if redis_kwargs.get("sentinel") and redis_kwargs.get(
"sentinel_service"):
sentinels = [
tuple(l.split(":"))
for l in redis_kwargs.pop("sentinel").split(",")
]
sentinel_service = redis_kwargs.pop("sentinel_service")
kwargs = {
k: v
for k, v in redis_kwargs.items() if k in
["decode_responses", "password", "db", "socket_timeout"]
}
self._redis = Sentinel(sentinels,
**kwargs).master_for(sentinel_service)
else:
kwargs = {
k: v
for k, v in redis_kwargs.items() if "sentinel" not in k
}
self._redis = redis.Redis(**kwargs)
self._redis.ping()
self.key = "{}:{}".format(namespace, name)
logger.debug("Created redis queue with socket_timeout of {}s".format(
redis_kwargs["socket_timeout"]))
# clean up from previous implementations
if self._redis.type(self.key) != "zset":
self._redis.delete(self.key)
def __init__(self,
servers,
hash_method='crc32',
sentinel=None,
strict_redis=False):
self.nodes = []
self.connections = {}
self.pool = None
servers = format_servers(servers)
if sentinel:
sentinel = Sentinel(sentinel['hosts'],
socket_timeout=sentinel.get(
'socket_timeout', 1))
for server_config in servers:
name = server_config.pop('name')
server_config["max_connections"] = int(
server_config.get("max_connections", 100))
if name in self.connections:
raise ValueError("server's name config must be unique")
if sentinel:
self.connections[name] = SentinelRedis(sentinel, name)
elif strict_redis:
self.connections[name] = redis.StrictRedis(**server_config)
else:
self.connections[name] = redis.Redis(**server_config)
server_config['name'] = name
self.nodes.append(name)
self.ring = HashRing(self.nodes, hash_method=hash_method)
class RedisCluster(object):
def __init__(self, sentinel_hosts, service_name, **connection_kwargs):
self.service_name = service_name
self.sentinel = Sentinel(sentinel_hosts, **connection_kwargs)
self.master = None
self.master_host_port = None
self.reconfigure()
self.sentinel_threads = {}
self.listen()
def reconfigure(self):
try:
self.master_host_port = self.sentinel.discover_master(self.service_name)
self.master = self.sentinel.master_for(self.service_name)
log.info(f"Reconfigured master to {self.master_host_port}")
except Exception as ex:
log.error(f"Error while reconfiguring. {ex.args[0]}")
def listen(self):
def on_new_master(workerThread):
self.reconfigure()
for sentinel in self.sentinel.sentinels:
sentinel_host = sentinel.connection_pool.connection_kwargs["host"]
self.sentinel_threads[sentinel_host] = MyPubSubWorkerThread(
sentinel, on_new_master, msg_sleep_time=0.001, daemon=True
)
self.sentinel_threads[sentinel_host].start()
def __init__(self, namespace="lock", **redis_kwargs):
import redis # pylint: disable=import-error
from redis.sentinel import Sentinel # pylint: disable=import-error
redis_kwargs["decode_responses"] = True
if redis_kwargs.get("sentinel") and redis_kwargs.get(
"sentinel_service"):
sentinels = [
tuple(l.split(":"))
for l in redis_kwargs.pop("sentinel").split(",")
]
sentinel_service = redis_kwargs.pop("sentinel_service")
kwargs = {
k: v
for k, v in redis_kwargs.items() if k in
["decode_responses", "password", "db", "socket_timeout"]
}
self._redis = Sentinel(sentinels,
**kwargs).master_for(sentinel_service)
else:
kwargs = {
k: v
for k, v in redis_kwargs.items() if "sentinel" not in k
}
self._redis = redis.Redis(**kwargs)
self._redis.ping()
self._namespace = namespace
logger.debug(
"Created redis lock manager with socket_timeout of {}s".format(
redis_kwargs["socket_timeout"]))
def get_connection(self, is_read_only=False) -> redis.StrictRedis:
"""
Gets a StrictRedis connection for normal redis or for redis sentinel based upon redis mode in configuration.
:type is_read_only: bool
:param is_read_only: In case of redis sentinel, it returns connection to slave
:return: Returns a StrictRedis connection
"""
if self.connection is not None:
return self.connection
if self.is_sentinel:
kwargs = dict()
if self.password:
kwargs["password"] = self.password
sentinel = Sentinel([(self.host, self.port)], **kwargs)
if is_read_only:
connection = sentinel.slave_for(self.sentinel_service, decode_responses=True)
else:
connection = sentinel.master_for(self.sentinel_service, decode_responses=True)
else:
connection = redis.StrictRedis(host=self.host, port=self.port, decode_responses=True,
password=self.password)
self.connection = connection
return connection
def get_master(self):
print(self.redis_hosts)
sentinel_hosts = []
for redis_host in self.redis_hosts:
redis_address_array = redis_host.split(':')
redis_ip = redis_address_array[0]
redis_port = redis_address_array[1]
address = (redis_ip, redis_port)
sentinel_hosts.append(address)
print(sentinel_hosts)
sentinel = Sentinel([('192.168.206.41', 26379),
('192.168.206.42', 26379),
('192.168.206.43', 26379)],
socket_timeout=0.1)
print(sentinel.discover_master('mymaster'))
master = sentinel.discover_master('mymaster')
self.master_ip = master[0]
self.master_port = master[1]
print(str(master[0]) + ' is master')
self.ip_master = {
"addresses": [{
"ip": self.master_ip
}],
"ports": [{
"port": int(self.master_port),
"protocol": "TCP"
}]
}
return True
def _get_sentinel_connection(self, uri, kw):
"""
get sentinel connection details from _uri
"""
try:
from redis.sentinel import Sentinel
except ImportError:
raise AnsibleError(
"The 'redis' python module (version 2.9.0 or newer) is required to use redis sentinel."
)
if ';' not in uri:
raise AnsibleError('_uri does not have sentinel syntax.')
# format: "localhost:26379;localhost2:26379;0:changeme"
connections = uri.split(';')
connection_args = connections.pop(-1)
if len(connection_args) > 0: # hanle if no db nr is given
connection_args = connection_args.split(':')
kw['db'] = connection_args.pop(0)
try:
kw['password'] = connection_args.pop(0)
except IndexError:
pass # password is optional
sentinels = [tuple(shost.split(':')) for shost in connections]
display.vv('\nUsing redis sentinels: %s' % sentinels)
scon = Sentinel(sentinels, **kw)
try:
return scon.master_for(self._sentinel_service_name,
socket_timeout=0.2)
except Exception as exc:
raise AnsibleError('Could not connect to redis sentinel: %s' %
to_native(exc))
def setup_rq_connection():
from redis.sentinel import Sentinel
redis_url = current_app.config.get('RQ_SCHEDULER_DASHBOARD_REDIS_URL')
redis_sentinels = current_app.config.get(
'RQ_SCHEDULER_DASHBOARD_REDIS_SENTINELS')
if redis_url:
current_app.redis_conn = from_url(redis_url)
elif redis_sentinels:
redis_master = current_app.config.get(
'RQ_SCHEDULER_DASHBOARD_REDIS_MASTER_NAME')
password = current_app.config.get(
'RQ_SCHEDULER_DASHBOARD_REDIS_PASSWORD')
db = current_app.config.get('RQ_SCHEDULER_DASHBOARD_REDIS_DB')
sentinel_hosts = [
tuple(sentinel.split(':', 1))
for sentinel in redis_sentinels.split(',')
]
sentinel = Sentinel(sentinel_hosts, db=db, password=password)
current_app.redis_conn = sentinel.master_for(redis_master)
else:
current_app.redis_conn = Redis(
host=current_app.config.get('REDIS_HOST'),
port=current_app.config.get('REDIS_PORT'),
password=current_app.config.get('REDIS_PASSWORD'),
db=current_app.config.get('REDIS_DB'))
def __init__(self,
connection_str,
db=0,
project=None,
service=None,
host=None,
conf=cfg.CONF,
**kwargs):
"""Redis driver for OSProfiler."""
super(RedisSentinel, self).__init__(connection_str,
project=project,
service=service,
host=host,
conf=conf,
**kwargs)
try:
from redis.sentinel import Sentinel
except ImportError:
raise exc.CommandError("To use this command, you should install "
"'redis' manually. Use command:\n "
"'pip install redis'.")
self.conf = conf
socket_timeout = self.conf.profiler.socket_timeout
parsed_url = parser.urlparse(self.connection_str)
sentinel = Sentinel([(parsed_url.hostname, int(parsed_url.port))],
password=parsed_url.password,
socket_timeout=socket_timeout)
self.db = sentinel.master_for(self.conf.profiler.sentinel_service_name,
socket_timeout=socket_timeout)
def get_redis_connection(config, use_strict_redis=False):
"""
Returns a redis connection from a connection config
"""
redis_cls = redis.StrictRedis if use_strict_redis else redis.Redis
if 'URL' in config:
return redis_cls.from_url(config['URL'], db=config.get('DB'))
if 'USE_REDIS_CACHE' in config.keys():
try:
from django.core.cache import caches
cache = caches[config['USE_REDIS_CACHE']]
except ImportError:
from django.core.cache import get_cache
cache = get_cache(config['USE_REDIS_CACHE'])
if hasattr(cache, 'client'):
# We're using django-redis. The cache's `client` attribute
# is a pluggable backend that return its Redis connection as
# its `client`
try:
# To get Redis connection on django-redis >= 3.4.0
# we need to use cache.client.get_client() instead of
# cache.client.client used in older versions
try:
return cache.client.get_client()
except AttributeError:
return cache.client.client
except NotImplementedError:
pass
else:
# We're using django-redis-cache
try:
return cache._client
except AttributeError:
# For django-redis-cache > 0.13.1
return cache.get_master_client()
if 'UNIX_SOCKET_PATH' in config:
return redis_cls(unix_socket_path=config['UNIX_SOCKET_PATH'],
db=config['DB'])
if 'SENTINELS' in config:
sentinel_kwargs = {
'db': config.get('DB'),
'password': config.get('PASSWORD'),
'socket_timeout': config.get('SOCKET_TIMEOUT'),
}
sentinel_kwargs.update(config.get('CONNECTION_KWARGS', {}))
sentinel = Sentinel(config['SENTINELS'], **sentinel_kwargs)
return sentinel.master_for(
service_name=config['MASTER_NAME'],
redis_class=redis_cls,
)
return redis_cls(host=config['HOST'],
port=config['PORT'],
db=config['DB'],
password=config.get('PASSWORD'))
def post(self, location):
if location not in self.locations:
return 'Unknown Location'
if app.config['DEVEL']:
pool = redis.ConnectionPool(host='localhost', port=6379)
r = redis.Redis(connection_pool=pool)
else:
sentinel = Sentinel([('localhost', 26379)], socket_timeout=1)
r = sentinel.master_for('mymaster', socket_timeout=1)
beers = {key: r.hgetall(key) for key in r.keys('beer_{0}_*'.format(location))}
for beername, beer in beers.items():
r.hset(beername, 'active', 'True' if beername in request.form.getlist('checks') else 'False')
for beer in request.form.getlist('delete'):
r.delete(beer)
r.save()
beers = []
for key in r.keys('beer_{0}_*'.format(location)):
beer = convert(r.hgetall(key))
beer['beername'] = key
beers.append(beer)
beers.sort(key=operator.itemgetter('brewery', 'name', 'type'))
return redirect(location)
def get(self, location):
if location not in self.locations:
return 'Unknown Location'
colors = get_colors(location, self.config)
priceinfo = get_priceinfo(location, self.config)
if app.config['DEVEL']:
pool = redis.ConnectionPool(host='localhost', port=6379)
r = redis.Redis(connection_pool=pool)
else:
sentinel = Sentinel([('localhost', 26379)], socket_timeout=1)
r = sentinel.master_for('mymaster', socket_timeout=1)
beers = []
for key in r.keys('beer_{0}_*'.format(location)):
beer = convert(r.hgetall(key))
beer['beername'] = key
beers.append(beer)
if 'items' in priceinfo:
for beer in beers:
beer['price'] = '$ {0}'.format(' /'.join([beer[p].replace('.0', '') for p in priceinfo['items'] if p in beer]))
beers.sort(key=operator.itemgetter('brewery', 'name', 'type'))
return render_template(
'edit.html',
title='Beer List',
beers=beers,
location=location,
colors=colors,
roles=self.groups,
locations=self.locations,
priceinfo=priceinfo,
scroll=False
)
class TestApplication(Application):
def __init__(self):
# Call to the superclass to bootstrap.
super(TestApplication, self).__init__(name='krux-redis-sentinel')
self.logger.debug('Parsing sentinels from args: %r', self.args.sentinel)
sentinels = hostports_from_args(self.args)
self.logger.debug('Parsed sentinel host, port pairs: %r', sentinels)
self.logger.debug('Initializing Sentinel instance...')
self.sentinel = Sentinel(sentinels, socket_timeout=SOCKET_TIMEOUT)
self.logger.debug('Initialized Sentinel instance: %r', self.sentinel)
def add_cli_arguments(self, parser):
add_sentinel_cli_arguments(parser)
def run(self):
master = self.sentinel.master_for(self.name)
slave = self.sentinel.slave_for(self.name)
try:
self.logger.info('Ping master: %s', master.ping())
except ConnectionError as err:
self.logger.warning('Could not connect to master!')
self.logger.error(err)
try:
self.logger.info('Ping slave: %s', slave.ping())
except ConnectionError as err:
self.logger.warning('Could not connect to slave!')
self.logger.error(err)
def redis_client():
if settings.CACHEOPS_REDIS and settings.CACHEOPS_SENTINEL:
raise ImproperlyConfigured("CACHEOPS_REDIS and CACHEOPS_SENTINEL are mutually exclusive")
client_class = CacheopsRedis
if settings.CACHEOPS_CLIENT_CLASS:
client_class = import_string(settings.CACHEOPS_CLIENT_CLASS)
if settings.CACHEOPS_SENTINEL:
if not {'locations', 'service_name'} <= set(settings.CACHEOPS_SENTINEL):
raise ImproperlyConfigured("Specify locations and service_name for CACHEOPS_SENTINEL")
sentinel = Sentinel(
settings.CACHEOPS_SENTINEL['locations'],
**omit(settings.CACHEOPS_SENTINEL, ('locations', 'service_name', 'db')))
return sentinel.master_for(
settings.CACHEOPS_SENTINEL['service_name'],
redis_class=client_class,
db=settings.CACHEOPS_SENTINEL.get('db', 0)
)
# Allow client connection settings to be specified by a URL.
if isinstance(settings.CACHEOPS_REDIS, six.string_types):
return client_class.from_url(settings.CACHEOPS_REDIS)
else:
return client_class(**settings.CACHEOPS_REDIS)
def setup_rq_connection():
# It's the only place where we can safely define default value for web background
# since It is used in template
current_app.config.setdefault('RQ_DASHBOARD_WEB_BACKGROUND', 'black')
# we need to do It here instead of cli, since It may be embeded
upgrade_config(current_app)
# Getting Redis connection parameters for RQ
redis_url = current_app.config.get('RQ_DASHBOARD_REDIS_URL')
redis_sentinels = current_app.config.get('RQ_DASHBOARD_REDIS_SENTINELS')
if isinstance(redis_url, list):
current_app.redis_conn = from_url(redis_url[0])
elif isinstance(redis_url, string_types):
current_app.redis_conn = from_url(redis_url)
elif redis_sentinels:
redis_master = current_app.config.get('RQ_DASHBOARD_REDIS_MASTER_NAME')
password = current_app.config.get('RQ_DASHBOARD_REDIS_PASSWORD')
db = current_app.config.get('RQ_DASHBOARD_REDIS_DB')
sentinel_hosts = [tuple(sentinel.split(':', 1))
for sentinel in redis_sentinels.split(',')]
sentinel = Sentinel(sentinel_hosts, db=db, password=password)
current_app.redis_conn = sentinel.master_for(redis_master)
else:
current_app.redis_conn = Redis(
host=current_app.config.get('RQ_DASHBOARD_REDIS_HOST', 'localhost'),
port=current_app.config.get('RQ_DASHBOARD_REDIS_PORT', 6379),
password=current_app.config.get('RQ_DASHBOARD_REDIS_PASSWORD'),
db=current_app.config.get('RQ_DASHBOARD_REDIS_DB', 0),
)
def _init(self, server, params):
super(CacheClass, self).__init__(params)
self._server = server
self._params = params
self.service_name = self.options.get("SERVICE_NAME", 'mymaster')
self.slave_read = self.options.get("READ_SLAVE", False)
if self.server:
def _get_hosttupe(conn):
host_lst = conn.rsplit(':', 1)
return host_lst[0], int(host_lst[1])
conn_list = [_get_hosttupe(conn) for conn in self.server]
else:
raise ImproperlyConfigured("sentinel server config error.")
kwargs = self.connection_pool_class_kwargs
max_connections = kwargs.pop('max_connections')
sentinel_manager = Sentinel(conn_list, sentinel_kwargs=kwargs)
kwargs.update({
'db': self.db,
'password': self.password,
'max_connections': max_connections
})
self._client = sentinel_manager.master_for(self.service_name,
connection_pool_class=self.connection_pool_class,
**kwargs)
if self.slave_read:
self._slave_client = sentinel_manager.slave_for(self.service_name,
connection_pool_class=self.connection_pool_class,
**kwargs)
def get_redis(app=None):
app = app_or_default(app)
conf = ensure_conf(app)
if not hasattr(app, 'redbeat_redis') or app.redbeat_redis is None:
redis_options = conf.app.conf.get(
'REDBEAT_REDIS_OPTIONS',
conf.app.conf.get('BROKER_TRANSPORT_OPTIONS', {}))
retry_period = redis_options.get('retry_period')
if conf.redis_url.startswith(
'redis-sentinel') and 'sentinels' in redis_options:
from redis.sentinel import Sentinel
sentinel = Sentinel(
redis_options['sentinels'],
socket_timeout=redis_options.get('socket_timeout'),
password=redis_options.get('password'),
decode_responses=True)
connection = sentinel.master_for(
redis_options.get('service_name', 'master'))
else:
connection = StrictRedis.from_url(conf.redis_url,
decode_responses=True)
if retry_period is None:
app.redbeat_redis = connection
else:
app.redbeat_redis = RetryingConnection(retry_period, connection)
return app.redbeat_redis
def create_app():
'''工厂方法'''
# 创建app对象
app = Flask(__name__)
# 加载配置
config = config_dict.get(app.config.get('ENV'))
app.config.from_object(config)
# 加载数据库配置
from common.models import db
db.init_app(app)
# 记录日志信息
log_file(config.LOG_LV)
# redis集群加载
from rediscluster import RedisCluster
app.redis_cluster = RedisCluster(
startup_nodes=app.config.get('REDIS_CLUSTER'))
# redis哨兵,主从配置
from redis.sentinel import Sentinel
sentinel = Sentinel(app.config.get('REDIS_SENTINELS'))
app.redis_master = sentinel.master_for(
app.config.get('REDIS_SENTINEL_SERVICES_NAME'))
app.redis_slave = sentinel.slave_for(
app.config.get('REDIS_SENTINEL_SERVICES_NAME'))
# 注册用户蓝图到app中
from .resources.users import user_blue
app.register_blueprint(user_blue)
return app
def __get_master(self, update=False):
master_ip = self.node.get_ip()
master_port = self.node.get_port()
master_node_id = self.node.get_node_id()
hosts = self.__get_hosts(refresh=update)
if len(hosts) > 0:
master_ip = hosts[0]['ip']
master_node_id = hosts[0]['node_id']
if os.path.isfile(Constants.MASTER_FILE):
fd = open(Constants.MASTER_FILE, "r")
temp_ip, temp_port = fd.readline().split(' ')
fd.close()
if temp_ip in hosts:
master_ip, master_port = temp_ip, temp_port
else:
os.remove(Constants.MASTER_FILE)
for host in hosts:
sentinel = Sentinel([(host["ip"], Constants.SENTINEL_PORT)], socket_timeout=0.1)
try:
ip, port = sentinel.discover_master(Constants.REDIS_SENTINEL_NAME)
for ht in hosts:
if ip == ht['ip']:
master_ip, master_port = ip, port
break
except:
pass
return master_ip, master_port
def __init__(self, name, namespace='queue', **redis_kwargs):
import redis
from redis.sentinel import Sentinel
redis_kwargs['decode_responses'] = True
if redis_kwargs.get('sentinel') and redis_kwargs.get(
'sentinel_service'):
sentinels = [
tuple(l.split(':'))
for l in redis_kwargs.pop('sentinel').split(',')
]
sentinel_service = redis_kwargs.pop('sentinel_service')
kwargs = {
k: v
for k, v in redis_kwargs.items() if k in
["decode_responses", "password", "db", "socket_timeout"]
}
self._redis = Sentinel(sentinels,
**kwargs).master_for(sentinel_service)
else:
kwargs = {
k: v
for k, v in redis_kwargs.items() if "sentinel" not in k
}
self._redis = redis.Redis(**kwargs)
self._redis.ping()
self.key = "{}:{}".format(namespace, name)
info("Created redis queue with socket_timeout of {}s".format(
redis_kwargs['socket_timeout']))
# clean up from previous implementations
if self._redis.type(self.key) != 'zset':
self._redis.delete(self.key)
def redis_populate():
"""Function to populate keys in Redis Server"""
# client = redis.StrictRedis(host=configs["redis_host"], port=configs["redis_port"])
# sentinel = Sentinel(
# [
# ("redis-cloudflare-node-0.redis-cloudflare-headless.dns-proxy.svc.cluster.local", 26379),
# ("redis-cloudflare-node-1.redis-cloudflare-headless.dns-proxy.svc.cluster.local", 26379),
# ("redis-cloudflare-node-2.redis-cloudflare-headless.dns-proxy.svc.cluster.local", 26379)
# ],
# socket_timeout=0.1)
from redis.sentinel import Sentinel
sentinel = Sentinel([
('redis-cloudflare', 26379),
],
sentinel_kwargs={'password': '******'},
password='******')
sentinel.discover_master('mymaster')
client = sentinel.master_for('mymaster', socket_timeout=0.5)
i = 93218
key = 'key' + str(i)
value = 'value' + str(i)
result = client.get(key)
print(result)
def redis_populate():
"""Function to populate keys in Redis Server"""
# client = redis.StrictRedis(host=configs["redis_host"], port=configs["redis_port"])
# sentinel = Sentinel(
# [
# ("redis-cloudflare-node-0.redis-cloudflare-headless.dns-proxy.svc.cluster.local", 26379),
# ("redis-cloudflare-node-1.redis-cloudflare-headless.dns-proxy.svc.cluster.local", 26379),
# ("redis-cloudflare-node-2.redis-cloudflare-headless.dns-proxy.svc.cluster.local", 26379)
# ],
# socket_timeout=0.1)
from redis.sentinel import Sentinel
sentinel = Sentinel([
('redis-cloudflare', 26379),
],
sentinel_kwargs={'password': '******'},
password='******')
sentinel.discover_master('mymaster')
client = sentinel.master_for('mymaster', socket_timeout=0.5)
for i in range(1000000):
key = 'key' + str(i)
# value='value'+str(i)
value = str(
i
) + '::: https://www.google.co.uk/#sclient=psy-ab&hl=en&source=hp&q=ASUSTeK+Computer+INC.+Model+M4A78LT-M+manual&pbx=1&oq=ASUSTeK+Computer+INC.+Model+M4A78LT-M+manual&aq=f&aqi=&aql=&gs_sm=3&gs_upl=52765l57528l0l57848l8l8l0l0l0l0l2413l3989l8-1.1l2l0&bav=on.2,or.r_gc.r_pw.,cf.osb&fp=3d6c1d1d0a5ea45f&biw=1262&bih=879'
client.set(key, value)
print(key, value)
def get_redis_connection(config, is_master=True):
"""
Returns redis connection basing on configuration dict
Args:
config: Dict with configuration, if key `SENTINELS` is present
sentinel connection pool will be used.
is_master: Whether connection should be master or slave,
applies only if sentinels are used
Returns: StrictRedis connection
"""
if 'SENTINELS' in config:
sentinel = Sentinel(
config['SENTINELS'],
db=config.get('DB'),
password=config.get('PASSWORD'),
socket_timeout=config.get('TIMEOUT'),
socket_connect_timeout=config.get('CONNECT_TIMEOUT'),
)
if is_master:
return sentinel.master_for(settings.REDIS_CLUSTER_NAME)
else:
return sentinel.slave_for(settings.REDIS_CLUSTER_NAME)
else:
return StrictRedis(
host=config['HOST'],
port=config['PORT'],
db=config.get('DB'),
password=config.get('PASSWORD'),
socket_timeout=config.get('TIMEOUT'),
socket_connect_timeout=config.get('CONNECT_TIMEOUT'),
)
def sentinel_connect(self, master_name):
url = urlparse.urlparse(self.schedule_url)
def parse_host(s):
if ':' in s:
host, port = s.split(':', 1)
port = int(port)
else:
host = s
port = 26379
return host, port
if '@' in url.netloc:
auth, hostspec = url.netloc.split('@', 1)
else:
auth = None
hostspec = url.netloc
if auth and ':' in auth:
_, password = auth.split(':', 1)
else:
password = None
path = url.path
if path.startswith('/'):
path = path[1:]
hosts = [parse_host(s) for s in hostspec.split(',')]
sentinel = Sentinel(hosts, password=password, db=path)
master = sentinel.master_for(master_name)
return master
def connect(self, index=0, write=True):
"""
Creates a redis connection with connection pool.
"""
master_name, sentinel_hosts, db = self.parse_connection_string(self._connection_string)
sentinel_timeout = self._options.get('SENTINEL_TIMEOUT', 1)
sentinel = Sentinel(sentinel_hosts, socket_timeout=sentinel_timeout)
if write:
host, port = sentinel.discover_master(master_name)
else:
host, port = random.choice([sentinel.discover_master(master_name)] + sentinel.discover_slaves(master_name))
kwargs = {
"db": db,
"parser_class": self.parser_class,
"password": self._options.get('PASSWORD', None),
}
kwargs.update({'host': host, 'port': port, 'connection_class': Connection})
if 'SOCKET_TIMEOUT' in self._options:
kwargs.update({'socket_timeout': int(self._options['SOCKET_TIMEOUT'])})
kwargs.update(self._pool_cls_kwargs)
connection_pool = get_or_create_connection_pool(self._pool_cls, **kwargs)
connection = Redis(connection_pool=connection_pool)
return connection
def __redis_client__(self, instance):
try:
LOG.debug('Connecting to redis databaseinfra %s', self.databaseinfra)
# redis uses timeout in seconds
connection_timeout_in_seconds = Configuration.get_by_name_as_int('redis_connect_timeout', default=REDIS_CONNECTION_DEFAULT_TIMEOUT)
if (instance and instance.instance_type == Instance.REDIS) or (not self.databaseinfra.plan.is_ha and not instance):
connection_address, connection_port = self.__get_admin_single_connection(instance)
client = redis.Redis(host = connection_address,
port = int(connection_port),
password = self.databaseinfra.password,
socket_connect_timeout = connection_timeout_in_seconds)
else:
sentinels = self.__get_admin_sentinel_connection(instance)
sentinel = Sentinel(sentinels, socket_timeout=connection_timeout_in_seconds)
client = sentinel.master_for(self.databaseinfra.name,
socket_timeout=connection_timeout_in_seconds,
password = self.databaseinfra.password)
LOG.debug('Successfully connected to redis databaseinfra %s' % (self.databaseinfra))
return client
except Exception, e:
raise e
def _connection(self):
sentinel = Sentinel(self._sentinels,
socket_timeout=self._socket_timeout)
slave = sentinel.slave_for(self._sentinel_master,
socket_timeout=self._socket_timeout,
db=self._db)
return slave
def connect(self, write=True):
"""
Create a redis connection with connection pool.
"""
self.log.debug("connect called: write=%s", write)
sentinel_timeout = self._options.get("SENTINEL_TIMEOUT", 1)
password = self._options.get("PASSWORD", None)
sentinel = Sentinel(
self._sentinel_hosts, socket_timeout=sentinel_timeout, password=password
)
if write:
host, port = sentinel.discover_master(self._master_name)
else:
try:
host, port = random.choice( # nosec
sentinel.discover_slaves(self._master_name)
)
except IndexError:
self.log.debug("no slaves are available. using master for read.")
host, port = sentinel.discover_master(self._master_name)
if password:
connection_url = f"redis://:{password}@{host}:{port}/{self._database_name}"
else:
connection_url = f"redis://{host}:{port}/{self._database_name}"
self.log.debug("Connecting to: %s", connection_url)
return self.connection_factory.connect(connection_url)
def get_redis_master(redis_sentinel_nodes):
nodes_list = redis_sentinel_nodes.split(',')
sentinel = Sentinel([tuple(nodes_list[0].split(':'))], socket_timeout=0.1)
master = sentinel.discover_master('mymaster')
if not master:
sys.exit(1)
return list(master)
def __init__(self, namespace='lock', **redis_kwargs):
import redis
from redis.sentinel import Sentinel
redis_kwargs['decode_responses'] = True
if redis_kwargs.get('sentinel') and redis_kwargs.get(
'sentinel_service'):
sentinels = [
tuple(l.split(':'))
for l in redis_kwargs.pop('sentinel').split(',')
]
sentinel_service = redis_kwargs.pop('sentinel_service')
kwargs = {
k: v
for k, v in redis_kwargs.items() if k in
["decode_responses", "password", "db", "socket_timeout"]
}
self._redis = Sentinel(sentinels,
**kwargs).master_for(sentinel_service)
else:
kwargs = {
k: v
for k, v in redis_kwargs.items() if "sentinel" not in k
}
self._redis = redis.Redis(**kwargs)
self._redis.ping()
self._namespace = namespace
info("Created redis lock manager with socket_timeout of {}s".format(
redis_kwargs['socket_timeout']))
class RedisLock(Lock):
def __init__(self, namespace='lock', **redis_kwargs):
import redis
from redis.sentinel import Sentinel
redis_kwargs['decode_responses'] = True
if redis_kwargs.get('sentinel') and redis_kwargs.get('sentinel_service'):
sentinels = [tuple(l.split(':')) for l in redis_kwargs.pop('sentinel').split(',')]
sentinel_service = redis_kwargs.pop('sentinel_service')
kwargs = {k: v for k, v in redis_kwargs.items() if k in ["decode_responses", "password", "db", "socket_timeout"]}
self._redis = Sentinel(sentinels, **kwargs).master_for(sentinel_service)
else:
kwargs = {k: v for k, v in redis_kwargs.items() if "sentinel" not in k}
self._redis = redis.Redis(**kwargs)
self._redis.ping()
self._namespace = namespace
info("Created redis lock manager with socket_timeout of {}s".format(redis_kwargs['socket_timeout']))
def __key(self, name):
return "{}:{}".format(self._namespace, name)
def lock(self, name, owner, expiration=1, allow_owner_relock=False):
"""
Obtain the named lock.
:param allow_owner_relock: bool
:param name: str the name of the lock
:param owner: str a unique name for the locking node
:param expiration: int in seconds, 0 expiration means forever
"""
import redis
try:
if int(expiration) < 1:
expiration = 1
key = self.__key(name)
non_existing = not (allow_owner_relock and self._redis.get(key) == owner)
return self._redis.set(key, owner, ex=int(expiration), nx=non_existing)
except redis.exceptions.ResponseError as e:
exception("Unable to obtain lock, local state: name: %s, owner: %s, expiration: %s, allow_owner_relock: %s",
name, owner, expiration, allow_owner_relock)
def unlock(self, name, owner):
"""
Release the named lock.
:param name: str the name of the lock
:param owner: str a unique name for the locking node
"""
key = self.__key(name)
if self._redis.get(key) == owner:
self._redis.delete(key)
return True
return False
def check_lock(self, name):
return self._redis.get(self.__key(name)) is not None
def print_locks(self):
keys = self._redis.keys(self.__key('*'))
for key in keys:
print("{} locked by {}, expires in {} seconds".format(key, self._redis.get(key), self._redis.ttl(key)))
def get_redis_from_config(settings, connection_class=Redis):
"""Returns a StrictRedis instance from a dictionary of settings.
To use redis sentinel, you must specify a dictionary in the configuration file.
Example of a dictionary with keys without values:
SENTINEL: {'INSTANCES':, 'SOCKET_TIMEOUT':, 'PASSWORD':,'DB':, 'MASTER_NAME':}
"""
if settings.get('REDIS_URL') is not None:
return connection_class.from_url(settings['REDIS_URL'])
elif settings.get('SENTINEL') is not None:
instances = settings['SENTINEL'].get('INSTANCES', [('localhost', 26379)])
socket_timeout = settings['SENTINEL'].get('SOCKET_TIMEOUT', None)
password = settings['SENTINEL'].get('PASSWORD', None)
db = settings['SENTINEL'].get('DB', 0)
master_name = settings['SENTINEL'].get('MASTER_NAME', 'mymaster')
sn = Sentinel(instances, socket_timeout=socket_timeout, password=password, db=db)
return sn.master_for(master_name)
kwargs = {
'host': settings.get('REDIS_HOST', 'localhost'),
'port': settings.get('REDIS_PORT', 6379),
'db': settings.get('REDIS_DB', 0),
'password': settings.get('REDIS_PASSWORD', None),
'ssl': settings.get('REDIS_SSL', False),
}
return connection_class(**kwargs)
def get_connection(self, is_read_only=False) -> redis.Redis:
"""
Gets a Redis connection for normal redis or for redis sentinel based upon redis mode in configuration.
:type is_read_only: bool
:param is_read_only: In case of redis sentinel, it returns connection to slave
:return: Returns a Redis connection
"""
if self.connection is not None:
return self.connection
if self.is_sentinel:
kwargs = dict()
if self.password:
kwargs["password"] = self.password
sentinel = Sentinel([(self.host, self.port)], **kwargs)
if is_read_only:
connection = sentinel.slave_for(self.sentinel_service,
decode_responses=True)
else:
connection = sentinel.master_for(self.sentinel_service,
decode_responses=True)
else:
connection = redis.Redis(host=self.host,
port=self.port,
decode_responses=True,
password=self.password)
self.connection = connection
return connection
def __init__(self, **config):
try:
redisIpConf = config['_channel_redis_sentinel']
if not redisIpConf:
log.error('Can not find valid ip config.')
else:
redisConfigure = []
redisIpSplit = redisIpConf.split(',')
for redisInfo in redisIpSplit:
redisInfoSplit = redisInfo.split(':')
redisConfigure.append(
(redisInfoSplit[0], redisInfoSplit[1]))
self.sentinel = Sentinel(redisConfigure)
self.redisInstance = self.sentinel.master_for(
'redis-master', password=config['_channel_redis_password'])
##self._pool = redis.ConnectionPool(host=config['_channel_redis_host'], port=config['_channel_redis_port'], db=1, password=config['_channel_redis_password'])
##self.redisInstance = redis.StrictRedis(connection_pool=self._pool)
self._master_pub_topic = config['_master_pub_topic']
self._none_match_ip = 'no_ip_matched'
self._sub_node = config['_sub_node']
except:
#log.error("redis init failed.")
log.error(traceback.format_exc())
def redis_client():
if settings.CACHEOPS_REDIS and settings.CACHEOPS_SENTINEL:
raise ImproperlyConfigured("CACHEOPS_REDIS and CACHEOPS_SENTINEL are mutually exclusive")
client_class = CacheopsRedis
if settings.CACHEOPS_CLIENT_CLASS:
client_class = import_string(settings.CACHEOPS_CLIENT_CLASS)
if settings.CACHEOPS_SENTINEL:
if not {'locations', 'service_name'} <= set(settings.CACHEOPS_SENTINEL):
raise ImproperlyConfigured("Specify locations and service_name for CACHEOPS_SENTINEL")
sentinel = Sentinel(
settings.CACHEOPS_SENTINEL['locations'],
**omit(settings.CACHEOPS_SENTINEL, ('locations', 'service_name', 'db')))
return sentinel.master_for(
settings.CACHEOPS_SENTINEL['service_name'],
redis_class=client_class,
db=settings.CACHEOPS_SENTINEL.get('db', 0)
)
# Allow client connection settings to be specified by a URL.
if isinstance(settings.CACHEOPS_REDIS, str):
return client_class.from_url(settings.CACHEOPS_REDIS)
else:
return client_class(**settings.CACHEOPS_REDIS)
def connect2redis(host, port=6379, db=0, connectionPool=True, logging=None, sentinel_service=prms.REDIS_SENTINEL_SERVICE_NAME, master=True):
if sentinel_service:
retry_count = 0
while retry_count < 10:
sentinel = Sentinel([(host, 26379)])
if master:
client = sentinel.master_for(sentinel_service)
else:
client = sentinel.slave_for(sentinel_service)
if is_available(client, logging=logging):
logging.info("Connected to Redis succesfully.")
return client
else:
print(f"Waiting for {pow(2, retry_count)} secs.")
time.sleep(pow(2, retry_count))
retry_count += 1
else:
retry_count = 0
while retry_count < 10:
if connectionPool:
pool = redis.ConnectionPool(host=host, port=port, db=db)
client = redis.StrictRedis(connection_pool=pool)
else:
client = redis.StrictRedis(host=host, port=port, db=db)
if is_available(client, logging=logging):
logging.info("Connected to Redis succesfully.")
return client
else:
print(f"Waiting for {pow(2, retry_count)} secs.")
time.sleep(pow(2, retry_count))
retry_count += 1
logging.error("Exiting...")
exit(3)
def get_redis_from_config(settings, connection_class=Redis):
"""Returns a StrictRedis instance from a dictionary of settings.
To use redis sentinel, you must specify a dictionary in the configuration file.
Example of a dictionary with keys without values:
SENTINEL: {'INSTANCES':, 'SOCKET_TIMEOUT':, 'PASSWORD':,'DB':, 'MASTER_NAME':}
"""
if settings.get('REDIS_URL') is not None:
return connection_class.from_url(settings['REDIS_URL'])
elif settings.get('SENTINEL') is not None:
instances = settings['SENTINEL'].get('INSTANCES', [('localhost', 26379)])
socket_timeout = settings['SENTINEL'].get('SOCKET_TIMEOUT', None)
password = settings['SENTINEL'].get('PASSWORD', None)
db = settings['SENTINEL'].get('DB', 0)
master_name = settings['SENTINEL'].get('MASTER_NAME', 'mymaster')
sn = Sentinel(instances, socket_timeout=socket_timeout, password=password, db=db)
return sn.master_for(master_name)
kwargs = {
'host': settings.get('REDIS_HOST', 'localhost'),
'port': settings.get('REDIS_PORT', 6379),
'db': settings.get('REDIS_DB', 0),
'password': settings.get('REDIS_PASSWORD', None),
'ssl': settings.get('REDIS_SSL', False),
}
return connection_class(**kwargs)
def run():
try:
logging.info('Calculating training booking reminders')
if SENTINEL_HOST is None:
client = redis.StrictRedis(host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD)
else:
sentinel = Sentinel([(SENTINEL_HOST, SENTINEL_PORT)])
client = sentinel.master_for(SENTINEL_MASTER)
# When was the last time we notified for? Default to now
last_not_str = client.get(WEEK_LAST_CHECK)
now = datetime.datetime.now()
last_notified = now
if last_not_str is not None:
last_notified_date = dateutil.parser.parse(last_not_str)
# Check that this is after the current date
if last_notified_date > now:
last_notified = last_notified_date
# Now work out the date that is midnight one week and a day from now
one_week = datetime.datetime.combine(datetime.date.today() + datetime.timedelta(8), datetime.time())
logging.info('Getting sessions from %s until %s' % (str(last_notified), str(one_week)))
if one_week <= last_notified:
return
# Find all the sessions that fall bewtween the last notified date and a weeks time
sessions = TrainingSession.query({
'find': {
'start': {
'$gte': last_notified,
'$lt': one_week
}
}
})
# Notify that these need reminding
for session in sessions:
data = {
'name': 'reminder',
'data': {
'type': 'trainingSession',
'data': session._id
}
}
client.rpush(EVENT_Q, json.dumps(data))
if (len(sessions) > 0):
client.publish(EVENT_CHAN, 'event')
logging.info('Published training booking events')
# Store that we've notified
client.set(WEEK_LAST_CHECK, one_week.isoformat())
logging.info('Finished training booking')
except Exception as ex:
logging.exception(ex)
def check_sso_auth(user_id, brand_id, trans_type ):
sentinel = Sentinel([('120.76.154.208', 26379)], socket_timeout=0.5)
log.d(sentinel.discover_master('mymaster'))
redis_obj = sentinel.master_for('mymaster', socket_timeout=0.1)
db_obj = PostgreSQLConnector()
sso_cls = SSOAuth(redis_obj,db_obj)
return sso_cls.check_auth(user_id, brand_id, trans_type)
def sso_log(user_id, brand_id, trans_type, data_type,data_content,log_time):
sentinel = Sentinel([('112.74.198.224', 26379)], socket_timeout=0.5)
log.d(sentinel.discover_master('mymaster'))
redis_obj = sentinel.master_for('mymaster', socket_timeout=0.1)
db_obj = PostgreSQLConnector()
sso_cls = SSOAuth(redis_obj,db_obj)
return sso_cls.send_log(user_id, brand_id, trans_type, data_type,data_content,log_time)
def redis_conn(readonly=True):
global sentinel, master_client, slave_client
if not sentinel:
sentinel = Sentinel(settings.REDIS_SENTINEL, socket_timeout=1)
if not master_client:
master_client = sentinel.master_for(settings.REDIS_CLUSTER)
if not slave_client:
slave_client = sentinel.slave_for(settings.REDIS_CLUSTER)
return slave_client if readonly else master_client
def setUp(self):
sentinel = Sentinel(settings_test.REDIS_SENTINEL)
db = getattr(settings_test, 'REDIS_DB', 0)
self.connection = sentinel.master_for('mymaster', db=db)
self.connection.flushall()
self.guard = ContributionsGuard(self.connection)
self.anon_user = {'user_id': None, 'user_ip': '127.0.0.1'}
self.auth_user = {'user_id': 33, 'user_ip': None}
self.task = Task(id=22)
def client(self):
sentinel = Sentinel(
self.sentinel_conf.get('sentinels'),
min_other_sentinels=self.sentinel_conf.get("min_other_sentinels", 0),
password=self.sentinel_conf.get("password"),
sentinel_kwargs={"socket_timeout": self.sentinel_conf.get("sentinel_timeout")},
)
return sentinel.master_for(self.sentinel_conf.get("service_name"), Redis,
socket_timeout=self.sentinel_conf.get("socket_timeout"))
def __init__(self, **redis_conf):
self.prefix = 'q'
sentinels = redis_conf.pop('sentinels', '()')
sentinels_service_name = redis_conf.pop('sentinels_service_name',
'mymaster')
min_other_sentinels = redis_conf.pop('min_other_sentinels', 0)
s = Sentinel(sentinels, **redis_conf)
self.db = s.master_for(sentinels_service_name)
#self.db = StrictRedis(**redis_conf)
self.key = '%s:%s' % (self.prefix, '%s')
def _sentinel_managed_pool(self):
sentinel = Sentinel(
self.sentinels,
min_other_sentinels=getattr(self, "min_other_sentinels", 0),
password=getattr(self, "password", None),
sentinel_kwargs={"socket_timeout": getattr(self, "sentinel_timeout", None)},
)
return sentinel.master_for(self.service_name, self.Client,
socket_timeout=self.socket_timeout).connection_pool
def get_redis_url():
if hasattr(config, 'ETCD_HOST'):
return config.HERA_REDIS_SENTINEL
sentinel_list = [tuple(i.split(':')) for i in get_etcd_setting('HERA_REDIS_SENTINEL').split(',')]
redis_db = get_etcd_setting('HERA_REDIS_DB')
sentinel = Sentinel(sentinel_list, socket_timeout=0.1)
master_name = get_etcd_setting('HERA_REDIS_MASTERNAME')
return 'redis://{}/{}'.format(
':'.join(map(str, sentinel.discover_master(master_name))),
redis_db
)
def new_new(_, cls, broker_url, *args, **kwargs):
scheme = urlparse(broker_url).scheme
if scheme == "redis-sentinel":
from rpaas.tasks import app
opts = app.conf.BROKER_TRANSPORT_OPTIONS
s = Sentinel(opts["sentinels"], password=opts["password"])
host, port = s.discover_master(opts["service_name"])
return RedisBroker("redis://:{}@{}:{}".format(opts["password"], host, port))
else:
old_new(cls, broker_url, *args, **kwargs)
def _redis(self):
if getattr(settings, 'EXPERIMENTS_REDIS_SENTINELS', None):
sentinel = Sentinel(settings.EXPERIMENTS_REDIS_SENTINELS, socket_timeout=settings.EXPERIMENTS_REDIS_SENTINELS_TIMEOUT)
host, port = sentinel.discover_master(settings.EXPERIMENTS_REDIS_MASTER_NAME)
else:
host = getattr(settings, 'EXPERIMENTS_REDIS_HOST', 'localhost')
port = getattr(settings, 'EXPERIMENTS_REDIS_PORT', 6379)
password = getattr(settings, 'EXPERIMENTS_REDIS_PASSWORD', None)
db = getattr(settings, 'EXPERIMENTS_REDIS_DB', 0)
return redis.Redis(host=host, port=port, password=password, db=db)
def init_shards_from_sentinel(self):
sentinel_client = Sentinel(
[(h, 8422) for h in self._sentinels], socket_timeout=self._timeout)
# Connect to all the shards with the names specified per
# shard_name_format. The names are important since it's getting
# the instances from Redis Sentinel.
for shard_num in range(0, self.num_shards()):
shard_name = self.get_shard_name(shard_num)
self._shards[shard_num] = sentinel_client.master_for(
shard_name, socket_timeout=self._timeout)
# Just in case we need it later.
self._sentinel_client = sentinel_client
def client(self):
if not self._redis_connection:
sentinel = Sentinel(
self.sentinel_conf.get("sentinels"),
min_other_sentinels=self.sentinel_conf.get("min_other_sentinels", 0),
password=self.sentinel_conf.get("password"),
socket_timeout=self.sentinel_conf.get("sentinel_timeout", None),
)
redis_connection = sentinel.master_for(
self.sentinel_conf.get("service_name"), Redis, socket_timeout=self.sentinel_conf.get("socket_timeout")
)
self._redis_connection.append(redis_connection)
return self._redis_connection[0]
def init(self):
"""
Called by Scheduler to say 'let's prepare yourself guy'
"""
logger.info("[RedisRetention] Initialization of the redis module")
#self.return_queue = self.properties['from_queue']
if self.sentinel_servers:
logger.info("[RedisRetention] sentinel_servers: %s" % str(self.sentinel_servers))
logger.info("[RedisRetention] redis_instance: %s" % str(self.redis_instance))
sentinel = Sentinel(self.sentinel_servers)
self.mc = sentinel.master_for(self.redis_instance, socket_timeout=0.1)
else:
self.mc = redis.Redis(self.server)
def get(self):
if self._config.sentinel_id:
sentinel = Sentinel([(self._config.host,
self._config.port)],
socket_timeout=self._config.timeout)
return sentinel.master_for(self._config.sentinel_id,
socket_timeout=self._config.timeout,
db=self._config.db,
retry_on_timeout=True)
else:
conn = StrictRedis(host=self._config.host, port=self._config.port,
db=self._config.db, retry_on_timeout=True,
socket_timeout=self._config.timeout)
return conn
def get_redis_connection(config, use_strict_redis=False):
"""
Returns a redis connection from a connection config
"""
redis_cls = redis.StrictRedis if use_strict_redis else redis.Redis
if 'URL' in config:
return redis_cls.from_url(config['URL'], db=config.get('DB'))
if 'USE_REDIS_CACHE' in config.keys():
try:
from django.core.cache import caches
cache = caches[config['USE_REDIS_CACHE']]
except ImportError:
from django.core.cache import get_cache
cache = get_cache(config['USE_REDIS_CACHE'])
if hasattr(cache, 'client'):
# We're using django-redis. The cache's `client` attribute
# is a pluggable backend that return its Redis connection as
# its `client`
try:
# To get Redis connection on django-redis >= 3.4.0
# we need to use cache.client.get_client() instead of
# cache.client.client used in older versions
try:
return cache.client.get_client()
except AttributeError:
return cache.client.client
except NotImplementedError:
pass
else:
# We're using django-redis-cache
try:
return cache._client
except AttributeError:
# For django-redis-cache > 0.13.1
return cache.get_master_client()
if 'UNIX_SOCKET_PATH' in config:
return redis_cls(unix_socket_path=config['UNIX_SOCKET_PATH'], db=config['DB'])
if 'SENTINELS' in config:
sentinel = Sentinel(config['SENTINELS'])
return sentinel.master_for(
service_name=config['MASTER_NAME'], redis_class=redis_cls, db=config.get('DB'),
password=config.get('PASSWORD'), socket_timeout=config.get('SOCKET_TIMEOUT')
)
return redis_cls(host=config['HOST'], port=config['PORT'], db=config['DB'], password=config.get('PASSWORD'))
def __init__(self, sentinels, service, **kwargs):
sentinel = Sentinel(sentinels, **kwargs)
try:
master = sentinel.discover_master(service)
except MasterNotFoundError:
raise
try:
slaves = sentinel.discover_slaves(service)
except SlaveNotFoundError:
self._slaves = None
if version_info[0] < 3:
super(AutoRedisSentinel, self).__init__(master, slaves=slaves, **kwargs)
else:
super().__init__(master, slaves=slaves, **kwargs)
def client(self):
kwargs = {
'retry_on_timeout': False,
'socket_keepalive': None,
'socket_connect_timeout': 1,
'socket_timeout': self.socket_timeout or 1,
# redis连接池要小于进程允许的最大连接数
'max_connections': 10240,
}
sentinel = Sentinel(self.sentinels, min_other_sentinels=self.min_other_sentinels, **kwargs)
kwargs.update({
'db': self.db,
})
return sentinel.master_for(self.service_name, redis_class=Redis, **kwargs)
def run(self, terms, inject=None, **kwargs):
values = []
terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject)
for term in terms:
params = self.parse_params(term)
try:
conn = Sentinel(params['server'], socket_timeout=1)
res = conn.discover_master(params['name'])
if res is None:
res = ""
values.append(":".join((str(v) for v in res)))
except Exception, e:
print(e)
def connect(self, index=0, write=True):
"""
Creates a redis connection with connection pool.
"""
master_name, sentinel_hosts, db = self.parse_connection_string(self._connection_string)
sentinel_timeout = self._options.get("SENTINEL_TIMEOUT", 1)
sentinel = Sentinel(sentinel_hosts, socket_timeout=sentinel_timeout)
if write:
host, port = sentinel.discover_master(master_name)
else:
host, port = random.choice([sentinel.discover_master(master_name)] + sentinel.discover_slaves(master_name))
return self.connection_factory.connect(host, port, db)
