class RedisDriver:
def __init__(self, config):
self.config = config
sentinels = [(c['ip'], c['port']) for c in self.config['sentinels']]
self.sentinel = Sentinel(sentinels,
socket_timeout=0.1,
db=self.config['db'],
decode_responses=True)
self.__master = None
self.__slave = None
@property
def master(self):
if False:
if not self.__master:
self.__master = self.sentinel.master_for(
self.config['name'], password=self.config.get('auth', ''))
return self.__master
else:
return self.sentinel.master_for(self.config['name'],
password=self.config.get(
'auth', ''))
@property
def slave(self):
if False:
if not self.__slave:
self.__slave = self.sentinel.slave_for(
self.config['name'], password=self.config.get('auth', ''))
return self.__slave
else:
return self.sentinel.slave_for(self.config['name'],
password=self.config.get(
'auth', ''))
class redisSentinelHelper():
def __init__(self,sentinel_list,service_name,db):
self.sentinel = Sentinel(sentinel_list,socket_timeout=0.5)
self.service_name = service_name
self.db = db
def get_master_redis(self):
return self.sentinel.discover_master(self.service_name)
def get_slave_redis(self):
return self.sentinel.discover_slaves(self.service_name)
def set_key(self,key,value):
master = self.sentinel.master_for(
service_name=self.service_name,
socket_timeout=0.5,
db=self.db
)
return master.set(key,value)
def set_hash_key(self,hkey,key,value):
master = self.sentinel.master_for(
service_name=self.service_name,
socket_timeout=0.5,
db=self.db
)
return master.hset(hkey,key,value)
def get_key(self,key):
slave = self.sentinel.slave_for(
service_name=self.service_name,
socket_timeout=0.5,
db=self.db
)
return slave.get(key)
def get_hash_key(self,hkey,hey):
slave = self.sentinel.slave_for(
service_name=self.service_name,
socket_timeout=0.5,
db=self.db
)
return slave.hget(hkey,hey)
def delete(self,key):
master = self.sentinel.master_for(
service_name=self.service_name,
socket_timeout=0.5,
db=self.db
)
return master.delete(key)
class SentinelProxy:
"""Proxy for Redis sentinel."""
def __init__(self,
sentinel_host,
master_name,
socket_timeout=0.1,
**kwargs):
"""Initialize Redis sentinel connection.
:params: sentinel_host: (host, port)
"""
self.sentinel = Sentinel([sentinel_host],
socket_timeout=socket_timeout)
self.master = self.sentinel.master_for(master_name,
socket_timeout=socket_timeout,
**kwargs)
self.slave = self.sentinel.slave_for(master_name,
socket_timeout=socket_timeout,
**kwargs)
def __getattr__(self, name):
"""Get attribute from Redis master or slave."""
master_key = ('set', 'hset', 'hmset', 'lset', 'lpush', 'blpop',
'brpop', 'rpush', 'expire', 'delete', 'incr')
if name not in master_key:
target = self.slave
else:
target = self.master
return getattr(target, name)
def init_redis_client(cls):
# 防止多线程竞争，获取到锁后依然可能已经初始化了
if cls.redis_objs:
return
cls.instance_name = conf.get('sentinel', "")
cls.socket_timeout = conf.get("socket_timeout", 5)
cls.connect_timeout = conf.get("connect_timeout", 0.1)
if not cls.instance_name:
# 单例模式
host = conf["host"]
cls.redis_objs[cls.MODE_READ] = Redis(
host=host.split(':')[0],
port=host.split(':')[1],
socket_timeout=cls.socket_timeout,
socket_connect_timeout=cls.connect_timeout,
retry_on_timeout=1
)
cls.redis_objs[cls.MODE_WRITE] = cls.redis_objs[cls.MODE_READ]
else:
# 哨兵模式
sentinel = Sentinel(
cls.parse_config(),
socket_timeout=cls.socket_timeout,
socket_connect_timeout=cls.connect_timeout,
retry_on_timeout=1
)
cls.redis_objs[cls.MODE_READ] = sentinel.slave_for(cls.instance_name)
cls.redis_objs[cls.MODE_WRITE] = sentinel.master_for(cls.instance_name)
class RedisCluster(object):
def __init__(self, sentinel_hosts, service_name, **connection_kwargs):
self.service_name = service_name
self.sentinel = Sentinel(sentinel_hosts, **connection_kwargs)
self.master = None
self.master_host_port = None
self.reconfigure()
self.sentinel_threads = {}
self.listen()
def reconfigure(self):
try:
self.master_host_port = self.sentinel.discover_master(self.service_name)
self.master = self.sentinel.master_for(self.service_name)
log.info(f"Reconfigured master to {self.master_host_port}")
except Exception as ex:
log.error(f"Error while reconfiguring. {ex.args[0]}")
def listen(self):
def on_new_master(workerThread):
self.reconfigure()
for sentinel in self.sentinel.sentinels:
sentinel_host = sentinel.connection_pool.connection_kwargs["host"]
self.sentinel_threads[sentinel_host] = MyPubSubWorkerThread(
sentinel, on_new_master, msg_sleep_time=0.001, daemon=True
)
self.sentinel_threads[sentinel_host].start()
def setup_rq_connection():
from redis.sentinel import Sentinel
redis_url = current_app.config.get('RQ_SCHEDULER_DASHBOARD_REDIS_URL')
redis_sentinels = current_app.config.get(
'RQ_SCHEDULER_DASHBOARD_REDIS_SENTINELS')
if redis_url:
current_app.redis_conn = from_url(redis_url)
elif redis_sentinels:
redis_master = current_app.config.get(
'RQ_SCHEDULER_DASHBOARD_REDIS_MASTER_NAME')
password = current_app.config.get(
'RQ_SCHEDULER_DASHBOARD_REDIS_PASSWORD')
db = current_app.config.get('RQ_SCHEDULER_DASHBOARD_REDIS_DB')
sentinel_hosts = [
tuple(sentinel.split(':', 1))
for sentinel in redis_sentinels.split(',')
]
sentinel = Sentinel(sentinel_hosts, db=db, password=password)
current_app.redis_conn = sentinel.master_for(redis_master)
else:
current_app.redis_conn = Redis(
host=current_app.config.get('REDIS_HOST'),
port=current_app.config.get('REDIS_PORT'),
password=current_app.config.get('REDIS_PASSWORD'),
db=current_app.config.get('REDIS_DB'))
def redis_populate():
"""Function to populate keys in Redis Server"""
# client = redis.StrictRedis(host=configs["redis_host"], port=configs["redis_port"])
# sentinel = Sentinel(
# [
# ("redis-cloudflare-node-0.redis-cloudflare-headless.dns-proxy.svc.cluster.local", 26379),
# ("redis-cloudflare-node-1.redis-cloudflare-headless.dns-proxy.svc.cluster.local", 26379),
# ("redis-cloudflare-node-2.redis-cloudflare-headless.dns-proxy.svc.cluster.local", 26379)
# ],
# socket_timeout=0.1)
from redis.sentinel import Sentinel
sentinel = Sentinel([
('redis-cloudflare', 26379),
],
sentinel_kwargs={'password': '******'},
password='******')
sentinel.discover_master('mymaster')
client = sentinel.master_for('mymaster', socket_timeout=0.5)
i = 93218
key = 'key' + str(i)
value = 'value' + str(i)
result = client.get(key)
print(result)
def run(name):
sentinel = Sentinel([
('10.5.5.194', 26379),
('10.5.5.155', 26379),
('10.5.5.128', 26379),
],
socket_timeout=0.5)
master = sentinel.master_for('mymaster',
socket_timeout=0.5,
password='******',
db=0)
slave = sentinel.slave_for('mymaster',
socket_timeout=0.5,
password='******',
db=0)
# 判断是否所传参数对应的value值是否为1
while True:
time.sleep(0.05)
if slave.get(name) == "0" or slave.get(name) is None:
continue
else:
break
master.set(name, "0")
f_do = sys.argv[1] + '.do'
f_dat = sys.argv[1] + '.dat'
f = open(f_dat, 'w')
f.write('0' + '\n')
f.write('success' + '\n')
f.write('set ' + name + '为0 \n')
f.close()
f = open(f_do, 'w')
f.write('')
f.close()
def _get_sentinel_connection(self, uri, kw):
"""
get sentinel connection details from _uri
"""
try:
from redis.sentinel import Sentinel
except ImportError:
raise AnsibleError(
"The 'redis' python module (version 2.9.0 or newer) is required to use redis sentinel."
)
if ';' not in uri:
raise AnsibleError('_uri does not have sentinel syntax.')
# format: "localhost:26379;localhost2:26379;0:changeme"
connections = uri.split(';')
connection_args = connections.pop(-1)
if len(connection_args) > 0: # hanle if no db nr is given
connection_args = connection_args.split(':')
kw['db'] = connection_args.pop(0)
try:
kw['password'] = connection_args.pop(0)
except IndexError:
pass # password is optional
sentinels = [tuple(shost.split(':')) for shost in connections]
display.vv('\nUsing redis sentinels: %s' % sentinels)
scon = Sentinel(sentinels, **kw)
try:
return scon.master_for(self._sentinel_service_name,
socket_timeout=0.2)
except Exception as exc:
raise AnsibleError('Could not connect to redis sentinel: %s' %
to_native(exc))
def post(self, location):
if location not in self.locations:
return 'Unknown Location'
if app.config['DEVEL']:
pool = redis.ConnectionPool(host='localhost', port=6379)
r = redis.Redis(connection_pool=pool)
else:
sentinel = Sentinel([('localhost', 26379)], socket_timeout=1)
r = sentinel.master_for('mymaster', socket_timeout=1)
beers = {key: r.hgetall(key) for key in r.keys('beer_{0}_*'.format(location))}
for beername, beer in beers.items():
r.hset(beername, 'active', 'True' if beername in request.form.getlist('checks') else 'False')
for beer in request.form.getlist('delete'):
r.delete(beer)
r.save()
beers = []
for key in r.keys('beer_{0}_*'.format(location)):
beer = convert(r.hgetall(key))
beer['beername'] = key
beers.append(beer)
beers.sort(key=operator.itemgetter('brewery', 'name', 'type'))
return redirect(location)
def get(self, location):
if location not in self.locations:
return 'Unknown Location'
colors = get_colors(location, self.config)
priceinfo = get_priceinfo(location, self.config)
if app.config['DEVEL']:
pool = redis.ConnectionPool(host='localhost', port=6379)
r = redis.Redis(connection_pool=pool)
else:
sentinel = Sentinel([('localhost', 26379)], socket_timeout=1)
r = sentinel.master_for('mymaster', socket_timeout=1)
beers = []
for key in r.keys('beer_{0}_*'.format(location)):
beer = convert(r.hgetall(key))
beer['beername'] = key
beers.append(beer)
if 'items' in priceinfo:
for beer in beers:
beer['price'] = '$ {0}'.format(' /'.join([beer[p].replace('.0', '') for p in priceinfo['items'] if p in beer]))
beers.sort(key=operator.itemgetter('brewery', 'name', 'type'))
return render_template(
'edit.html',
title='Beer List',
beers=beers,
location=location,
colors=colors,
roles=self.groups,
locations=self.locations,
priceinfo=priceinfo,
scroll=False
)
def add_queues_length_stats(self):
if len(self.queues) == 0:
return
sentinel_list = [(node.get('host', 'localhost'),
node.get('sentinel_port', 26379))
for node in self.node_list]
sentinel = Sentinel(sentinel_list, socket_timeout=5)
master_host = ''
queues_length_map = {}
for queue in self.queues:
queues_length_map[queue] = -1
try:
master_host, master_port = sentinel.discover_master(
self.master_name)
master_conn = sentinel.master_for(self.master_name,
db=self.db,
password=self.password)
for queue in self.queues:
try:
queues_length_map[queue] = master_conn.llen(queue)
except Exception:
continue
except Exception:
pass
for queue in self.queues:
self.add_gauge_value('Redis_Queues/%s' % queue,
None,
queues_length_map[queue],
count=1)
def get_redis_connection(config, use_strict_redis=False):
"""
Returns a redis connection from a connection config
"""
redis_cls = redis.StrictRedis if use_strict_redis else redis.Redis
if 'URL' in config:
return redis_cls.from_url(config['URL'], db=config.get('DB'))
if 'USE_REDIS_CACHE' in config.keys():
try:
from django.core.cache import caches
cache = caches[config['USE_REDIS_CACHE']]
except ImportError:
from django.core.cache import get_cache
cache = get_cache(config['USE_REDIS_CACHE'])
if hasattr(cache, 'client'):
# We're using django-redis. The cache's `client` attribute
# is a pluggable backend that return its Redis connection as
# its `client`
try:
# To get Redis connection on django-redis >= 3.4.0
# we need to use cache.client.get_client() instead of
# cache.client.client used in older versions
try:
return cache.client.get_client()
except AttributeError:
return cache.client.client
except NotImplementedError:
pass
else:
# We're using django-redis-cache
try:
return cache._client
except AttributeError:
# For django-redis-cache > 0.13.1
return cache.get_master_client()
if 'UNIX_SOCKET_PATH' in config:
return redis_cls(unix_socket_path=config['UNIX_SOCKET_PATH'],
db=config['DB'])
if 'SENTINELS' in config:
sentinel_kwargs = {
'db': config.get('DB'),
'password': config.get('PASSWORD'),
'socket_timeout': config.get('SOCKET_TIMEOUT'),
}
sentinel_kwargs.update(config.get('CONNECTION_KWARGS', {}))
sentinel = Sentinel(config['SENTINELS'], **sentinel_kwargs)
return sentinel.master_for(
service_name=config['MASTER_NAME'],
redis_class=redis_cls,
)
return redis_cls(host=config['HOST'],
port=config['PORT'],
db=config['DB'],
password=config.get('PASSWORD'))
class TestApplication(Application):
def __init__(self):
# Call to the superclass to bootstrap.
super(TestApplication, self).__init__(name='krux-redis-sentinel')
self.logger.debug('Parsing sentinels from args: %r', self.args.sentinel)
sentinels = hostports_from_args(self.args)
self.logger.debug('Parsed sentinel host, port pairs: %r', sentinels)
self.logger.debug('Initializing Sentinel instance...')
self.sentinel = Sentinel(sentinels, socket_timeout=SOCKET_TIMEOUT)
self.logger.debug('Initialized Sentinel instance: %r', self.sentinel)
def add_cli_arguments(self, parser):
add_sentinel_cli_arguments(parser)
def run(self):
master = self.sentinel.master_for(self.name)
slave = self.sentinel.slave_for(self.name)
try:
self.logger.info('Ping master: %s', master.ping())
except ConnectionError as err:
self.logger.warning('Could not connect to master!')
self.logger.error(err)
try:
self.logger.info('Ping slave: %s', slave.ping())
except ConnectionError as err:
self.logger.warning('Could not connect to slave!')
self.logger.error(err)
def __init__(self,
connection_str,
db=0,
project=None,
service=None,
host=None,
conf=cfg.CONF,
**kwargs):
"""Redis driver for OSProfiler."""
super(RedisSentinel, self).__init__(connection_str,
project=project,
service=service,
host=host,
conf=conf,
**kwargs)
try:
from redis.sentinel import Sentinel
except ImportError:
raise exc.CommandError("To use this command, you should install "
"'redis' manually. Use command:\n "
"'pip install redis'.")
self.conf = conf
socket_timeout = self.conf.profiler.socket_timeout
parsed_url = parser.urlparse(self.connection_str)
sentinel = Sentinel([(parsed_url.hostname, int(parsed_url.port))],
password=parsed_url.password,
socket_timeout=socket_timeout)
self.db = sentinel.master_for(self.conf.profiler.sentinel_service_name,
socket_timeout=socket_timeout)
def get_connection(self, is_read_only=False) -> redis.StrictRedis:
"""
Gets a StrictRedis connection for normal redis or for redis sentinel based upon redis mode in configuration.
:type is_read_only: bool
:param is_read_only: In case of redis sentinel, it returns connection to slave
:return: Returns a StrictRedis connection
"""
if self.connection is not None:
return self.connection
if self.is_sentinel:
kwargs = dict()
if self.password:
kwargs["password"] = self.password
sentinel = Sentinel([(self.host, self.port)], **kwargs)
if is_read_only:
connection = sentinel.slave_for(self.sentinel_service, decode_responses=True)
else:
connection = sentinel.master_for(self.sentinel_service, decode_responses=True)
else:
connection = redis.StrictRedis(host=self.host, port=self.port, decode_responses=True,
password=self.password)
self.connection = connection
return connection
def get_redis(**kwargs):
"""Returns a redis client instance.
Parameters
----------
redis_cls : class, optional
Defaults to ``redis.StrictRedis``.
url : str, optional
If given, ``redis_cls.from_url`` is used to instantiate the class.
**kwargs
Extra parameters to be passed to the ``redis_cls`` class.
Returns
-------
server
Redis client instance.
"""
sentiels = kwargs.pop('sentinels', None)
if sentiels:
from redis.sentinel import Sentinel
master_name = kwargs.pop('master_name')
sentiel = Sentinel(sentiels, **kwargs)
master = sentiel.master_for(master_name)
return master
else:
redis_cls = kwargs.pop('redis_cls', defaults.REDIS_CLS)
url = kwargs.pop('url', None)
if url:
return redis_cls.from_url(url, **kwargs)
else:
return redis_cls(**kwargs)
class RedisSentinelMessageBroker(RedisMessageBroker):
""" Redis Sentinel Message broker.
"""
def __init__(self, **kwargs):
self.init_sentinel(**kwargs)
RedisMessageBroker.__init__(self, **kwargs)
def init_sentinel(self, **kwargs):
if not kwargs.has_key('sentinel_servers'):
raise ConfigurationError('Kewyword sentinel_servers is missing.')
if not kwargs.has_key('sentinel_master'):
raise ConfigurationError('Kewyword sentinel_master is missing.')
# Enable sentinel support
self.__sentinel_servers = kwargs['sentinel_servers']
self.__sentinel_master = kwargs['sentinel_master']
del kwargs['sentinel_servers']
del kwargs['sentinel_master']
self.__sentinel = Sentinel(self.__sentinel_servers, **kwargs)
def configure_redis(self, **kwargs):
return self.__sentinel.master_for(self.__sentinel_master)
def get_connection_port(self):
return self.__sentinel.discover_master(self.__sentinel_master)[1]
def get_connection_host(self):
return self.__sentinel.discover_master(self.__sentinel_master)[0]
def __create_redis_client(self, cfg_params, addr):
new_sentinel = None
new_redis = None
if cfg_params.db_sentinel_port is None:
new_redis = Redis(host=addr,
port=cfg_params.db_port,
db=0,
max_connections=20)
else:
sentinel_node = (addr, cfg_params.db_sentinel_port)
master_name = cfg_params.db_sentinel_master_name
new_sentinel = Sentinel([sentinel_node])
new_redis = new_sentinel.master_for(master_name)
new_redis.set_response_callback(
'SETIE', lambda r: r and nativestr(r) == 'OK' or False)
new_redis.set_response_callback('DELIE',
lambda r: r and int(r) == 1 or False)
redis_pubsub = PubSub(self.event_separator,
new_redis.connection_pool,
ignore_subscribe_messages=True)
pubsub_thread = threading.Thread(target=None)
run_in_thread = False
return _RedisConn(new_redis, redis_pubsub, pubsub_thread,
run_in_thread)
def __init__(self):
self.kafka_consumer = Consumer({
'bootstrap.servers': KAFKA_SERVERS,
'group.id': KAFKA_MESSAGE_CONSUMER_GROUP,
'session.timeout.ms': 6000,
'auto.offset.reset': 'latest'
})
self.kafka_consumer.subscribe([KAFKA_MESSAGE_TOPIC])
LOG.info('Connected with Kafka Consumer')
self.kafka_producer = Producer({'bootstrap.servers': KAFKA_SERVERS})
LOG.info('Connected with Kafka Producer')
hosts_sentinel = []
endpoint = DBAAS_SENTINEL_ENDPOINT_SIMPLE.replace("sentinel://", "")
for host in endpoint.split(','):
hosts_sentinel.append(tuple(host.split(':')))
sentinel = Sentinel(hosts_sentinel,
socket_timeout=REDIS_SOCKET_TIMEOUT)
self.rediscon = sentinel.master_for(
DBAAS_SENTINEL_SERVICE_NAME,
socket_timeout=REDIS_SOCKET_TIMEOUT,
password=DBAAS_SENTINEL_PASSWORD)
LOG.info('Connected with Redis Database')
def sentinel_connect(self, master_name):
url = urlparse.urlparse(self.schedule_url)
def parse_host(s):
if ':' in s:
host, port = s.split(':', 1)
port = int(port)
else:
host = s
port = 26379
return host, port
if '@' in url.netloc:
auth, hostspec = url.netloc.split('@', 1)
else:
auth = None
hostspec = url.netloc
if auth and ':' in auth:
_, password = auth.split(':', 1)
else:
password = None
path = url.path
if path.startswith('/'):
path = path[1:]
hosts = [parse_host(s) for s in hostspec.split(',')]
sentinel = Sentinel(hosts, password=password, db=path)
master = sentinel.master_for(master_name)
return master
def get_redis_from_config(settings, connection_class=Redis):
"""Returns a StrictRedis instance from a dictionary of settings.
To use redis sentinel, you must specify a dictionary in the configuration file.
Example of a dictionary with keys without values:
SENTINEL: {'INSTANCES':, 'SOCKET_TIMEOUT':, 'PASSWORD':,'DB':, 'MASTER_NAME':}
"""
if settings.get('REDIS_URL') is not None:
return connection_class.from_url(settings['REDIS_URL'])
elif settings.get('SENTINEL') is not None:
instances = settings['SENTINEL'].get('INSTANCES', [('localhost', 26379)])
socket_timeout = settings['SENTINEL'].get('SOCKET_TIMEOUT', None)
password = settings['SENTINEL'].get('PASSWORD', None)
db = settings['SENTINEL'].get('DB', 0)
master_name = settings['SENTINEL'].get('MASTER_NAME', 'mymaster')
sn = Sentinel(instances, socket_timeout=socket_timeout, password=password, db=db)
return sn.master_for(master_name)
kwargs = {
'host': settings.get('REDIS_HOST', 'localhost'),
'port': settings.get('REDIS_PORT', 6379),
'db': settings.get('REDIS_DB', 0),
'password': settings.get('REDIS_PASSWORD', None),
'ssl': settings.get('REDIS_SSL', False),
}
return connection_class(**kwargs)
def redis_client():
if settings.CACHEOPS_REDIS and settings.CACHEOPS_SENTINEL:
raise ImproperlyConfigured("CACHEOPS_REDIS and CACHEOPS_SENTINEL are mutually exclusive")
client_class = CacheopsRedis
if settings.CACHEOPS_CLIENT_CLASS:
client_class = import_string(settings.CACHEOPS_CLIENT_CLASS)
if settings.CACHEOPS_SENTINEL:
if not {'locations', 'service_name'} <= set(settings.CACHEOPS_SENTINEL):
raise ImproperlyConfigured("Specify locations and service_name for CACHEOPS_SENTINEL")
sentinel = Sentinel(
settings.CACHEOPS_SENTINEL['locations'],
**omit(settings.CACHEOPS_SENTINEL, ('locations', 'service_name', 'db')))
return sentinel.master_for(
settings.CACHEOPS_SENTINEL['service_name'],
redis_class=client_class,
db=settings.CACHEOPS_SENTINEL.get('db', 0)
)
# Allow client connection settings to be specified by a URL.
if isinstance(settings.CACHEOPS_REDIS, str):
return client_class.from_url(settings.CACHEOPS_REDIS)
else:
return client_class(**settings.CACHEOPS_REDIS)
def get_redis(app=None):
app = app_or_default(app)
conf = ensure_conf(app)
if not hasattr(app, 'redbeat_redis') or app.redbeat_redis is None:
redis_options = conf.app.conf.get(
'REDBEAT_REDIS_OPTIONS',
conf.app.conf.get('BROKER_TRANSPORT_OPTIONS', {}))
retry_period = redis_options.get('retry_period')
if conf.redis_url.startswith(
'redis-sentinel') and 'sentinels' in redis_options:
from redis.sentinel import Sentinel
sentinel = Sentinel(
redis_options['sentinels'],
socket_timeout=redis_options.get('socket_timeout'),
password=redis_options.get('password'),
decode_responses=True)
connection = sentinel.master_for(
redis_options.get('service_name', 'master'))
else:
connection = StrictRedis.from_url(conf.redis_url,
decode_responses=True)
if retry_period is None:
app.redbeat_redis = connection
else:
app.redbeat_redis = RetryingConnection(retry_period, connection)
return app.redbeat_redis
def get_redis_connection(config, is_master=True):
"""
Returns redis connection basing on configuration dict
Args:
config: Dict with configuration, if key `SENTINELS` is present
sentinel connection pool will be used.
is_master: Whether connection should be master or slave,
applies only if sentinels are used
Returns: StrictRedis connection
"""
if 'SENTINELS' in config:
sentinel = Sentinel(
config['SENTINELS'],
db=config.get('DB'),
password=config.get('PASSWORD'),
socket_timeout=config.get('TIMEOUT'),
socket_connect_timeout=config.get('CONNECT_TIMEOUT'),
)
if is_master:
return sentinel.master_for(settings.REDIS_CLUSTER_NAME)
else:
return sentinel.slave_for(settings.REDIS_CLUSTER_NAME)
else:
return StrictRedis(
host=config['HOST'],
port=config['PORT'],
db=config.get('DB'),
password=config.get('PASSWORD'),
socket_timeout=config.get('TIMEOUT'),
socket_connect_timeout=config.get('CONNECT_TIMEOUT'),
)
def get_connection(self, is_read_only=False) -> redis.Redis:
"""
Gets a Redis connection for normal redis or for redis sentinel based upon redis mode in configuration.
:type is_read_only: bool
:param is_read_only: In case of redis sentinel, it returns connection to slave
:return: Returns a Redis connection
"""
if self.connection is not None:
return self.connection
if self.is_sentinel:
kwargs = dict()
if self.password:
kwargs["password"] = self.password
sentinel = Sentinel([(self.host, self.port)], **kwargs)
if is_read_only:
connection = sentinel.slave_for(self.sentinel_service,
decode_responses=True)
else:
connection = sentinel.master_for(self.sentinel_service,
decode_responses=True)
else:
connection = redis.Redis(host=self.host,
port=self.port,
decode_responses=True,
password=self.password)
self.connection = connection
return connection
def __redis_client__(self, instance):
try:
LOG.debug('Connecting to redis databaseinfra %s', self.databaseinfra)
# redis uses timeout in seconds
connection_timeout_in_seconds = Configuration.get_by_name_as_int('redis_connect_timeout', default=REDIS_CONNECTION_DEFAULT_TIMEOUT)
if (instance and instance.instance_type == Instance.REDIS) or (not self.databaseinfra.plan.is_ha and not instance):
connection_address, connection_port = self.__get_admin_single_connection(instance)
client = redis.Redis(host = connection_address,
port = int(connection_port),
password = self.databaseinfra.password,
socket_connect_timeout = connection_timeout_in_seconds)
else:
sentinels = self.__get_admin_sentinel_connection(instance)
sentinel = Sentinel(sentinels, socket_timeout=connection_timeout_in_seconds)
client = sentinel.master_for(self.databaseinfra.name,
socket_timeout=connection_timeout_in_seconds,
password = self.databaseinfra.password)
LOG.debug('Successfully connected to redis databaseinfra %s' % (self.databaseinfra))
return client
except Exception, e:
raise e
def __init__(self):
"""
init setting and create redis instance
"""
settings = Settings.get("redis")
self.prefix = settings.get("prefix")
self.queue_max_len = settings.get("queue_max_len")
self.sentinel = settings.get("sentinel")
if self.sentinel:
sentinel = Sentinel(sentinels=map(lambda x: x.split(":"),
settings.get("sentinel_hosts")))
kwargs = dict(
service_name=settings.get("sentinel_master"),
password=settings.get("password"),
decode_responses=True,
)
self.master = sentinel.master_for(**kwargs)
self.slave = sentinel.slave_for(**kwargs)
else:
pool = redis.ConnectionPool(
host=settings.get("host"),
port=settings.get("port"),
db=settings.get("db"),
password=settings.get("password"),
decode_responses=True,
)
self.master = self.slave = redis.StrictRedis(connection_pool=pool)
def redis_populate():
"""Function to populate keys in Redis Server"""
# client = redis.StrictRedis(host=configs["redis_host"], port=configs["redis_port"])
# sentinel = Sentinel(
# [
# ("redis-cloudflare-node-0.redis-cloudflare-headless.dns-proxy.svc.cluster.local", 26379),
# ("redis-cloudflare-node-1.redis-cloudflare-headless.dns-proxy.svc.cluster.local", 26379),
# ("redis-cloudflare-node-2.redis-cloudflare-headless.dns-proxy.svc.cluster.local", 26379)
# ],
# socket_timeout=0.1)
from redis.sentinel import Sentinel
sentinel = Sentinel([
('redis-cloudflare', 26379),
],
sentinel_kwargs={'password': '******'},
password='******')
sentinel.discover_master('mymaster')
client = sentinel.master_for('mymaster', socket_timeout=0.5)
for i in range(1000000):
key = 'key' + str(i)
# value='value'+str(i)
value = str(
i
) + '::: https://www.google.co.uk/#sclient=psy-ab&hl=en&source=hp&q=ASUSTeK+Computer+INC.+Model+M4A78LT-M+manual&pbx=1&oq=ASUSTeK+Computer+INC.+Model+M4A78LT-M+manual&aq=f&aqi=&aql=&gs_sm=3&gs_upl=52765l57528l0l57848l8l8l0l0l0l0l2413l3989l8-1.1l2l0&bav=on.2,or.r_gc.r_pw.,cf.osb&fp=3d6c1d1d0a5ea45f&biw=1262&bih=879'
client.set(key, value)
print(key, value)
def setup_rq_connection():
# It's the only place where we can safely define default value for web background
# since It is used in template
current_app.config.setdefault('RQ_DASHBOARD_WEB_BACKGROUND', 'black')
# we need to do It here instead of cli, since It may be embeded
upgrade_config(current_app)
# Getting Redis connection parameters for RQ
redis_url = current_app.config.get('RQ_DASHBOARD_REDIS_URL')
redis_sentinels = current_app.config.get('RQ_DASHBOARD_REDIS_SENTINELS')
if isinstance(redis_url, list):
current_app.redis_conn = from_url(redis_url[0])
elif isinstance(redis_url, string_types):
current_app.redis_conn = from_url(redis_url)
elif redis_sentinels:
redis_master = current_app.config.get('RQ_DASHBOARD_REDIS_MASTER_NAME')
password = current_app.config.get('RQ_DASHBOARD_REDIS_PASSWORD')
db = current_app.config.get('RQ_DASHBOARD_REDIS_DB')
sentinel_hosts = [tuple(sentinel.split(':', 1))
for sentinel in redis_sentinels.split(',')]
sentinel = Sentinel(sentinel_hosts, db=db, password=password)
current_app.redis_conn = sentinel.master_for(redis_master)
else:
current_app.redis_conn = Redis(
host=current_app.config.get('RQ_DASHBOARD_REDIS_HOST', 'localhost'),
port=current_app.config.get('RQ_DASHBOARD_REDIS_PORT', 6379),
password=current_app.config.get('RQ_DASHBOARD_REDIS_PASSWORD'),
db=current_app.config.get('RQ_DASHBOARD_REDIS_DB', 0),
)
def get_db():
redis_store = getattr(g, '_database', None)
if redis_store is None:
if 'FAKE_DB' in app.config and app.config['FAKE_DB']:
from mockredis import mock_redis_client
redis_store = g._database = mock_redis_client(
encoding='utf-8', decode_responses=True)
else:
if app.config['REDIS_URL'].startswith('redis://'):
redis_store = g._database = Redis.from_url(
app.config['REDIS_URL'],
encoding='utf-8',
decode_responses=True)
elif app.config['REDIS_URL'].startswith('sentinel://'):
parsed_url = urlparse(app.config['REDIS_URL'])
service = parsed_url.path.lstrip('/')
port = 26379
if ':' in parsed_url.netloc:
host, str_port = parsed_url.netloc.split(':')
port = int(str_port)
else:
host = parsed_url.netloc
sentinel = Sentinel([(host, port)], socket_timeout=0.1)
redis_store = sentinel.master_for(service,
redis_class=Redis,
socket_timeout=0.1)
return redis_store
def redis_client():
if settings.CACHEOPS_REDIS and settings.CACHEOPS_SENTINEL:
raise ImproperlyConfigured("CACHEOPS_REDIS and CACHEOPS_SENTINEL are mutually exclusive")
client_class = CacheopsRedis
if settings.CACHEOPS_CLIENT_CLASS:
client_class = import_string(settings.CACHEOPS_CLIENT_CLASS)
if settings.CACHEOPS_SENTINEL:
if not {'locations', 'service_name'} <= set(settings.CACHEOPS_SENTINEL):
raise ImproperlyConfigured("Specify locations and service_name for CACHEOPS_SENTINEL")
sentinel = Sentinel(
settings.CACHEOPS_SENTINEL['locations'],
**omit(settings.CACHEOPS_SENTINEL, ('locations', 'service_name', 'db')))
return sentinel.master_for(
settings.CACHEOPS_SENTINEL['service_name'],
redis_class=client_class,
db=settings.CACHEOPS_SENTINEL.get('db', 0)
)
# Allow client connection settings to be specified by a URL.
if isinstance(settings.CACHEOPS_REDIS, six.string_types):
return client_class.from_url(settings.CACHEOPS_REDIS)
else:
return client_class(**settings.CACHEOPS_REDIS)
def get_redis_from_config(settings, connection_class=Redis):
"""Returns a StrictRedis instance from a dictionary of settings.
To use redis sentinel, you must specify a dictionary in the configuration file.
Example of a dictionary with keys without values:
SENTINEL: {'INSTANCES':, 'SOCKET_TIMEOUT':, 'PASSWORD':,'DB':, 'MASTER_NAME':}
"""
if settings.get('REDIS_URL') is not None:
return connection_class.from_url(settings['REDIS_URL'])
elif settings.get('SENTINEL') is not None:
instances = settings['SENTINEL'].get('INSTANCES', [('localhost', 26379)])
socket_timeout = settings['SENTINEL'].get('SOCKET_TIMEOUT', None)
password = settings['SENTINEL'].get('PASSWORD', None)
db = settings['SENTINEL'].get('DB', 0)
master_name = settings['SENTINEL'].get('MASTER_NAME', 'mymaster')
sn = Sentinel(instances, socket_timeout=socket_timeout, password=password, db=db)
return sn.master_for(master_name)
kwargs = {
'host': settings.get('REDIS_HOST', 'localhost'),
'port': settings.get('REDIS_PORT', 6379),
'db': settings.get('REDIS_DB', 0),
'password': settings.get('REDIS_PASSWORD', None),
'ssl': settings.get('REDIS_SSL', False),
}
return connection_class(**kwargs)
def connect2redis(host, port=6379, db=0, connectionPool=True, logging=None, sentinel_service=prms.REDIS_SENTINEL_SERVICE_NAME, master=True):
if sentinel_service:
retry_count = 0
while retry_count < 10:
sentinel = Sentinel([(host, 26379)])
if master:
client = sentinel.master_for(sentinel_service)
else:
client = sentinel.slave_for(sentinel_service)
if is_available(client, logging=logging):
logging.info("Connected to Redis succesfully.")
return client
else:
print(f"Waiting for {pow(2, retry_count)} secs.")
time.sleep(pow(2, retry_count))
retry_count += 1
else:
retry_count = 0
while retry_count < 10:
if connectionPool:
pool = redis.ConnectionPool(host=host, port=port, db=db)
client = redis.StrictRedis(connection_pool=pool)
else:
client = redis.StrictRedis(host=host, port=port, db=db)
if is_available(client, logging=logging):
logging.info("Connected to Redis succesfully.")
return client
else:
print(f"Waiting for {pow(2, retry_count)} secs.")
time.sleep(pow(2, retry_count))
retry_count += 1
logging.error("Exiting...")
exit(3)
def run(skip=0, stop=9999999):
cfg = yaml.load(open('config.yaml', 'r'))
sentinels = [(c['ip'], c['port']) for c in cfg['cache']]
sentinel = Sentinel(sentinels,
socket_timeout=0.1,
db=1,
decode_responses=True)
master = sentinel.master_for('madeira')
stream = open('clean.csv', 'r')
n = 0
for l in stream:
if n < skip:
print('SKIP %d' % n)
n += 1
continue
if n >= stop:
print('STOP AT %d' % n)
break
# Q2014120613471910003652,13413352219,10,cmcc
order_id, mobile, price, routing = l.strip().split(',')
print(order_id, mobile, price, routing)
partner = cfg['upstream'][routing]
order_id = 'T' + order_id[1:]
call_cmcc(master, mobile, price, order_id, partner)
n += 1
def create_app():
'''工厂方法'''
# 创建app对象
app = Flask(__name__)
# 加载配置
config = config_dict.get(app.config.get('ENV'))
app.config.from_object(config)
# 加载数据库配置
from common.models import db
db.init_app(app)
# 记录日志信息
log_file(config.LOG_LV)
# redis集群加载
from rediscluster import RedisCluster
app.redis_cluster = RedisCluster(
startup_nodes=app.config.get('REDIS_CLUSTER'))
# redis哨兵,主从配置
from redis.sentinel import Sentinel
sentinel = Sentinel(app.config.get('REDIS_SENTINELS'))
app.redis_master = sentinel.master_for(
app.config.get('REDIS_SENTINEL_SERVICES_NAME'))
app.redis_slave = sentinel.slave_for(
app.config.get('REDIS_SENTINEL_SERVICES_NAME'))
# 注册用户蓝图到app中
from .resources.users import user_blue
app.register_blueprint(user_blue)
return app
def _init(self, server, params):
super(CacheClass, self).__init__(params)
self._server = server
self._params = params
self.service_name = self.options.get("SERVICE_NAME", 'mymaster')
self.slave_read = self.options.get("READ_SLAVE", False)
if self.server:
def _get_hosttupe(conn):
host_lst = conn.rsplit(':', 1)
return host_lst[0], int(host_lst[1])
conn_list = [_get_hosttupe(conn) for conn in self.server]
else:
raise ImproperlyConfigured("sentinel server config error.")
kwargs = self.connection_pool_class_kwargs
max_connections = kwargs.pop('max_connections')
sentinel_manager = Sentinel(conn_list, sentinel_kwargs=kwargs)
kwargs.update({
'db': self.db,
'password': self.password,
'max_connections': max_connections
})
self._client = sentinel_manager.master_for(self.service_name,
connection_pool_class=self.connection_pool_class,
**kwargs)
if self.slave_read:
self._slave_client = sentinel_manager.slave_for(self.service_name,
connection_pool_class=self.connection_pool_class,
**kwargs)
def run():
try:
logging.info('Calculating training booking reminders')
if SENTINEL_HOST is None:
client = redis.StrictRedis(host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD)
else:
sentinel = Sentinel([(SENTINEL_HOST, SENTINEL_PORT)])
client = sentinel.master_for(SENTINEL_MASTER)
# When was the last time we notified for? Default to now
last_not_str = client.get(WEEK_LAST_CHECK)
now = datetime.datetime.now()
last_notified = now
if last_not_str is not None:
last_notified_date = dateutil.parser.parse(last_not_str)
# Check that this is after the current date
if last_notified_date > now:
last_notified = last_notified_date
# Now work out the date that is midnight one week and a day from now
one_week = datetime.datetime.combine(datetime.date.today() + datetime.timedelta(8), datetime.time())
logging.info('Getting sessions from %s until %s' % (str(last_notified), str(one_week)))
if one_week <= last_notified:
return
# Find all the sessions that fall bewtween the last notified date and a weeks time
sessions = TrainingSession.query({
'find': {
'start': {
'$gte': last_notified,
'$lt': one_week
}
}
})
# Notify that these need reminding
for session in sessions:
data = {
'name': 'reminder',
'data': {
'type': 'trainingSession',
'data': session._id
}
}
client.rpush(EVENT_Q, json.dumps(data))
if (len(sessions) > 0):
client.publish(EVENT_CHAN, 'event')
logging.info('Published training booking events')
# Store that we've notified
client.set(WEEK_LAST_CHECK, one_week.isoformat())
logging.info('Finished training booking')
except Exception as ex:
logging.exception(ex)
def check_sso_auth(user_id, brand_id, trans_type ):
sentinel = Sentinel([('120.76.154.208', 26379)], socket_timeout=0.5)
log.d(sentinel.discover_master('mymaster'))
redis_obj = sentinel.master_for('mymaster', socket_timeout=0.1)
db_obj = PostgreSQLConnector()
sso_cls = SSOAuth(redis_obj,db_obj)
return sso_cls.check_auth(user_id, brand_id, trans_type)
def sso_log(user_id, brand_id, trans_type, data_type,data_content,log_time):
sentinel = Sentinel([('112.74.198.224', 26379)], socket_timeout=0.5)
log.d(sentinel.discover_master('mymaster'))
redis_obj = sentinel.master_for('mymaster', socket_timeout=0.1)
db_obj = PostgreSQLConnector()
sso_cls = SSOAuth(redis_obj,db_obj)
return sso_cls.send_log(user_id, brand_id, trans_type, data_type,data_content,log_time)
def redis_conn(readonly=True):
global sentinel, master_client, slave_client
if not sentinel:
sentinel = Sentinel(settings.REDIS_SENTINEL, socket_timeout=1)
if not master_client:
master_client = sentinel.master_for(settings.REDIS_CLUSTER)
if not slave_client:
slave_client = sentinel.slave_for(settings.REDIS_CLUSTER)
return slave_client if readonly else master_client
def setUp(self):
sentinel = Sentinel(settings_test.REDIS_SENTINEL)
db = getattr(settings_test, 'REDIS_DB', 0)
self.connection = sentinel.master_for('mymaster', db=db)
self.connection.flushall()
self.guard = ContributionsGuard(self.connection)
self.anon_user = {'user_id': None, 'user_ip': '127.0.0.1'}
self.auth_user = {'user_id': 33, 'user_ip': None}
self.task = Task(id=22)
def client(self):
sentinel = Sentinel(
self.sentinel_conf.get('sentinels'),
min_other_sentinels=self.sentinel_conf.get("min_other_sentinels", 0),
password=self.sentinel_conf.get("password"),
sentinel_kwargs={"socket_timeout": self.sentinel_conf.get("sentinel_timeout")},
)
return sentinel.master_for(self.sentinel_conf.get("service_name"), Redis,
socket_timeout=self.sentinel_conf.get("socket_timeout"))
def __init__(self, **redis_conf):
self.prefix = 'q'
sentinels = redis_conf.pop('sentinels', '()')
sentinels_service_name = redis_conf.pop('sentinels_service_name',
'mymaster')
min_other_sentinels = redis_conf.pop('min_other_sentinels', 0)
s = Sentinel(sentinels, **redis_conf)
self.db = s.master_for(sentinels_service_name)
#self.db = StrictRedis(**redis_conf)
self.key = '%s:%s' % (self.prefix, '%s')
def _sentinel_managed_pool(self):
sentinel = Sentinel(
self.sentinels,
min_other_sentinels=getattr(self, "min_other_sentinels", 0),
password=getattr(self, "password", None),
sentinel_kwargs={"socket_timeout": getattr(self, "sentinel_timeout", None)},
)
return sentinel.master_for(self.service_name, self.Client,
socket_timeout=self.socket_timeout).connection_pool
class SentinelBackedRingClient(RingClient):
def __init__(self, hosts, options):
sentinel_kwargs = self._get_sentinel_kwargs(options)
node_kwargs = self._get_node_kwargs(options)
masters = None
# Try to fetch a list of all masters from any sentinel.
hosts = list(hosts)
shuffle(hosts) # Randomly sort sentinels before trying to bootstrap.
for host, port in hosts:
client = StrictRedis(host=host, port=port, **sentinel_kwargs)
try:
masters = client.sentinel_masters().keys()
break
except RedisError:
pass
if masters is None:
# No Sentinel responded successfully?
raise errors.MastersListUnavailable
if not len(masters):
# The masters list was empty?
raise errors.NoMastersConfigured
sentinel_kwargs.update({
# Sentinels connected to fewer sentinels than `MIN_SENTINELS` will
# be ignored.
'min_other_sentinels': options.get('MIN_SENTINELS',
len(hosts) / 2),
})
self.sentinel = Sentinel(hosts, **sentinel_kwargs)
masters = [self.sentinel.master_for(name, **node_kwargs)
for name in masters]
super(SentinelBackedRingClient, self).__init__(masters, options)
def _get_sentinel_kwargs(self, options):
password = options.get('SENTINEL_PASSWORD')
try:
socket_timeout = float(options.get('SOCKET_TIMEOUT', 0.2))
except ValueError:
raise ImproperlyConfigured('`SOCKET_TIMEOUT` must be a valid number type.')
return {
'password': password,
'socket_timeout': socket_timeout
}
def disconnect(self):
for node in self.sentinel.sentinels:
try:
node.connection_pool.disconnect()
except AttributeError:
# TODO(usmanm): Figure this shit out.
pass
super(SentinelBackedRingClient, self).disconnect()
def init_shards_from_sentinel(self):
sentinel_client = Sentinel(
[(h, 8422) for h in self._sentinels], socket_timeout=self._timeout)
# Connect to all the shards with the names specified per
# shard_name_format. The names are important since it's getting
# the instances from Redis Sentinel.
for shard_num in range(0, self.num_shards()):
shard_name = self.get_shard_name(shard_num)
self._shards[shard_num] = sentinel_client.master_for(
shard_name, socket_timeout=self._timeout)
# Just in case we need it later.
self._sentinel_client = sentinel_client
def client(self):
if not self._redis_connection:
sentinel = Sentinel(
self.sentinel_conf.get("sentinels"),
min_other_sentinels=self.sentinel_conf.get("min_other_sentinels", 0),
password=self.sentinel_conf.get("password"),
socket_timeout=self.sentinel_conf.get("sentinel_timeout", None),
)
redis_connection = sentinel.master_for(
self.sentinel_conf.get("service_name"), Redis, socket_timeout=self.sentinel_conf.get("socket_timeout")
)
self._redis_connection.append(redis_connection)
return self._redis_connection[0]
def init(self):
"""
Called by Scheduler to say 'let's prepare yourself guy'
"""
logger.info("[RedisRetention] Initialization of the redis module")
#self.return_queue = self.properties['from_queue']
if self.sentinel_servers:
logger.info("[RedisRetention] sentinel_servers: %s" % str(self.sentinel_servers))
logger.info("[RedisRetention] redis_instance: %s" % str(self.redis_instance))
sentinel = Sentinel(self.sentinel_servers)
self.mc = sentinel.master_for(self.redis_instance, socket_timeout=0.1)
else:
self.mc = redis.Redis(self.server)
def client(self):
kwargs = {
'retry_on_timeout': False,
'socket_keepalive': None,
'socket_connect_timeout': 1,
'socket_timeout': self.socket_timeout or 1,
# redis连接池要小于进程允许的最大连接数
'max_connections': 10240,
}
sentinel = Sentinel(self.sentinels, min_other_sentinels=self.min_other_sentinels, **kwargs)
kwargs.update({
'db': self.db,
})
return sentinel.master_for(self.service_name, redis_class=Redis, **kwargs)
def get_redis_connection(config, use_strict_redis=False):
"""
Returns a redis connection from a connection config
"""
redis_cls = redis.StrictRedis if use_strict_redis else redis.Redis
if 'URL' in config:
return redis_cls.from_url(config['URL'], db=config.get('DB'))
if 'USE_REDIS_CACHE' in config.keys():
try:
from django.core.cache import caches
cache = caches[config['USE_REDIS_CACHE']]
except ImportError:
from django.core.cache import get_cache
cache = get_cache(config['USE_REDIS_CACHE'])
if hasattr(cache, 'client'):
# We're using django-redis. The cache's `client` attribute
# is a pluggable backend that return its Redis connection as
# its `client`
try:
# To get Redis connection on django-redis >= 3.4.0
# we need to use cache.client.get_client() instead of
# cache.client.client used in older versions
try:
return cache.client.get_client()
except AttributeError:
return cache.client.client
except NotImplementedError:
pass
else:
# We're using django-redis-cache
try:
return cache._client
except AttributeError:
# For django-redis-cache > 0.13.1
return cache.get_master_client()
if 'UNIX_SOCKET_PATH' in config:
return redis_cls(unix_socket_path=config['UNIX_SOCKET_PATH'], db=config['DB'])
if 'SENTINELS' in config:
sentinel = Sentinel(config['SENTINELS'])
return sentinel.master_for(
service_name=config['MASTER_NAME'], redis_class=redis_cls, db=config.get('DB'),
password=config.get('PASSWORD'), socket_timeout=config.get('SOCKET_TIMEOUT')
)
return redis_cls(host=config['HOST'], port=config['PORT'], db=config['DB'], password=config.get('PASSWORD'))
def get(self):
if self._config.sentinel_id:
sentinel = Sentinel([(self._config.host,
self._config.port)],
socket_timeout=self._config.timeout)
return sentinel.master_for(self._config.sentinel_id,
socket_timeout=self._config.timeout,
db=self._config.db,
retry_on_timeout=True)
else:
conn = StrictRedis(host=self._config.host, port=self._config.port,
db=self._config.db, retry_on_timeout=True,
socket_timeout=self._config.timeout)
return conn
def put(self, location):
if location not in self.locations:
return 'Unknown Location'
form = BeerForm()
beer = self._beer(form, location)
if app.config['DEVEL']:
pool = redis.ConnectionPool(host='localhost', port=6379)
r = redis.Redis(connection_pool=pool)
else:
sentinel = Sentinel([('localhost', 26379)], socket_timeout=1)
r = sentinel.master_for('mymaster', socket_timeout=1)
r.hmset(request.args.get('name'), beer)
r.save()
return redirect('/{0}/entry'.format(location))
def wrapper(*args, **kwargs):
if os.environ.get('PYBOSSA_REDIS_CACHE_DISABLED') is None:
key = "%s::%s" % (settings.REDIS_KEYPREFIX, key_prefix)
sentinel = Sentinel(settings.REDIS_SENTINEL, socket_timeout=0.1)
master = sentinel.master_for('mymaster')
slave = sentinel.slave_for('mymaster')
output = slave.get(key)
if output:
return pickle.loads(output)
else:
output = f(*args, **kwargs)
master.setex(key, timeout, pickle.dumps(output))
return output
else:
return f(*args, **kwargs)
def get_redis_cli():
global _redis_cli
if not _redis_cli and is_redis_sentinel():
sentinel = Sentinel([(get_redis_host(), get_redis_port())], socket_timeout=0.1)
_redis_cli = sentinel.master_for(get_redis_sentinel_master(), socket_timeout=0.1)
elif not _redis_cli:
_redis_cli = redis.StrictRedis(
host=get_redis_host(),
port=get_redis_port(),
db=get_redis_db(),
password=get_redis_password()
)
return _redis_cli
def delete_memoized(function, arg=None):
if os.environ.get('PYBOSSA_REDIS_CACHE_DISABLED') is None:
sentinel = Sentinel(settings.REDIS_SENTINEL, socket_timeout=0.1)
master = sentinel.master_for(settings.REDIS_MASTER)
keys = []
if arg:
key_to_hash = ":%s" % arg
key = "%s:%s_args::%s" % (settings.REDIS_KEYPREFIX, function.__name__,
hashlib.md5(key_to_hash).hexdigest())
keys.append(key)
else:
key = "%s:%s_args::*" % (settings.REDIS_KEYPREFIX, function.__name__)
keys = master.keys(key)
for k in keys:
master.delete(k)
return True
def get_atomic_connection(self) -> StrictPipeline:
"""
Gets a StrictPipeline for normal redis or for redis sentinel based upon redis mode in configuration
:return: Returns a StrictPipeline object
"""
if self.is_sentinel:
kwargs = dict()
if self.password:
kwargs["password"] = self.password
sentinel = Sentinel([(self.host, self.port)], **kwargs)
pipeline = sentinel.master_for(self.sentinel_service, decode_responses=True).pipeline(True)
else:
pipeline = redis.StrictRedis(host=self.host, port=self.port, decode_responses=True,
password=self.password).pipeline(True)
return pipeline
def main():
last_send = None
sentinel = Sentinel(REDIS_SENTINEL_LIST)
redis_master = sentinel.master_for("mymaster", socket_timeout=120)
while True:
r = redis_master.blpop('alert:message', 60)
if not r:
continue
current_timestamp = int(time.time())
if last_send and current_timestamp - last_send < 60:
continue
last_send = current_timestamp
message = r[1]
for email in ALERT_EMAILS:
send_email(email, "viabtc server error", message)
def create_redis_connection(self):
if self.mode == 'Redis':
self.redis = stoneredis.StoneRedis(options.redisIP, db=options.redisDB, port=options.redisPort, socket_connect_timeout=5, socket_timeout=5)
self.redis.connect()
else:
SENTINEL_POOL = Sentinel(
options.sentinelServers,
socket_timeout=0.1,
max_connections=1000,
)
self.redis = SENTINEL_POOL.master_for(
options.sentinelName,
redis_class=stoneredis.StoneRedis,
socket_timeout=5,
socket_connect_timeout=5,
)
