def get_master(self):
print(self.redis_hosts)
sentinel_hosts = []
for redis_host in self.redis_hosts:
redis_address_array = redis_host.split(':')
redis_ip = redis_address_array[0]
redis_port = redis_address_array[1]
address = (redis_ip, redis_port)
sentinel_hosts.append(address)
print(sentinel_hosts)
sentinel = Sentinel([('192.168.206.41', 26379),
('192.168.206.42', 26379),
('192.168.206.43', 26379)],
socket_timeout=0.1)
print(sentinel.discover_master('mymaster'))
master = sentinel.discover_master('mymaster')
self.master_ip = master[0]
self.master_port = master[1]
print(str(master[0]) + ' is master')
self.ip_master = {
"addresses": [{
"ip": self.master_ip
}],
"ports": [{
"port": int(self.master_port),
"protocol": "TCP"
}]
}
return True
def connect(self, write=True):
"""
Create a redis connection with connection pool.
"""
self.log.debug("connect called: write=%s", write)
sentinel_timeout = self._options.get("SENTINEL_TIMEOUT", 1)
password = self._options.get("PASSWORD", None)
sentinel = Sentinel(
self._sentinel_hosts, socket_timeout=sentinel_timeout, password=password
)
if write:
host, port = sentinel.discover_master(self._master_name)
else:
try:
host, port = random.choice( # nosec
sentinel.discover_slaves(self._master_name)
)
except IndexError:
self.log.debug("no slaves are available. using master for read.")
host, port = sentinel.discover_master(self._master_name)
if password:
connection_url = f"redis://:{password}@{host}:{port}/{self._database_name}"
else:
connection_url = f"redis://{host}:{port}/{self._database_name}"
self.log.debug("Connecting to: %s", connection_url)
return self.connection_factory.connect(connection_url)
class RedisSentinelMessageBroker(RedisMessageBroker):
""" Redis Sentinel Message broker.
"""
def __init__(self, **kwargs):
self.init_sentinel(**kwargs)
RedisMessageBroker.__init__(self, **kwargs)
def init_sentinel(self, **kwargs):
if not kwargs.has_key('sentinel_servers'):
raise ConfigurationError('Kewyword sentinel_servers is missing.')
if not kwargs.has_key('sentinel_master'):
raise ConfigurationError('Kewyword sentinel_master is missing.')
# Enable sentinel support
self.__sentinel_servers = kwargs['sentinel_servers']
self.__sentinel_master = kwargs['sentinel_master']
del kwargs['sentinel_servers']
del kwargs['sentinel_master']
self.__sentinel = Sentinel(self.__sentinel_servers, **kwargs)
def configure_redis(self, **kwargs):
return self.__sentinel.master_for(self.__sentinel_master)
def get_connection_port(self):
return self.__sentinel.discover_master(self.__sentinel_master)[1]
def get_connection_host(self):
return self.__sentinel.discover_master(self.__sentinel_master)[0]
def redis_populate():
"""Function to populate keys in Redis Server"""
# client = redis.StrictRedis(host=configs["redis_host"], port=configs["redis_port"])
# sentinel = Sentinel(
# [
# ("redis-cloudflare-node-0.redis-cloudflare-headless.dns-proxy.svc.cluster.local", 26379),
# ("redis-cloudflare-node-1.redis-cloudflare-headless.dns-proxy.svc.cluster.local", 26379),
# ("redis-cloudflare-node-2.redis-cloudflare-headless.dns-proxy.svc.cluster.local", 26379)
# ],
# socket_timeout=0.1)
from redis.sentinel import Sentinel
sentinel = Sentinel([
('redis-cloudflare', 26379),
],
sentinel_kwargs={'password': '******'},
password='******')
sentinel.discover_master('mymaster')
client = sentinel.master_for('mymaster', socket_timeout=0.5)
for i in range(1000000):
key = 'key' + str(i)
# value='value'+str(i)
value = str(
i
) + '::: https://www.google.co.uk/#sclient=psy-ab&hl=en&source=hp&q=ASUSTeK+Computer+INC.+Model+M4A78LT-M+manual&pbx=1&oq=ASUSTeK+Computer+INC.+Model+M4A78LT-M+manual&aq=f&aqi=&aql=&gs_sm=3&gs_upl=52765l57528l0l57848l8l8l0l0l0l0l2413l3989l8-1.1l2l0&bav=on.2,or.r_gc.r_pw.,cf.osb&fp=3d6c1d1d0a5ea45f&biw=1262&bih=879'
client.set(key, value)
print(key, value)
def connect(self, index=0, write=True):
"""
Creates a redis connection with connection pool.
"""
master_name, sentinel_hosts, db = self.parse_connection_string(self._connection_string)
sentinel_timeout = self._options.get('SENTINEL_TIMEOUT', 1)
sentinel = Sentinel(sentinel_hosts, socket_timeout=sentinel_timeout)
if write:
host, port = sentinel.discover_master(master_name)
else:
host, port = random.choice([sentinel.discover_master(master_name)] + sentinel.discover_slaves(master_name))
kwargs = {
"db": db,
"parser_class": self.parser_class,
"password": self._options.get('PASSWORD', None),
}
kwargs.update({'host': host, 'port': port, 'connection_class': Connection})
if 'SOCKET_TIMEOUT' in self._options:
kwargs.update({'socket_timeout': int(self._options['SOCKET_TIMEOUT'])})
kwargs.update(self._pool_cls_kwargs)
connection_pool = get_or_create_connection_pool(self._pool_cls, **kwargs)
connection = Redis(connection_pool=connection_pool)
return connection
def redis_populate():
"""Function to populate keys in Redis Server"""
# client = redis.StrictRedis(host=configs["redis_host"], port=configs["redis_port"])
# sentinel = Sentinel(
# [
# ("redis-cloudflare-node-0.redis-cloudflare-headless.dns-proxy.svc.cluster.local", 26379),
# ("redis-cloudflare-node-1.redis-cloudflare-headless.dns-proxy.svc.cluster.local", 26379),
# ("redis-cloudflare-node-2.redis-cloudflare-headless.dns-proxy.svc.cluster.local", 26379)
# ],
# socket_timeout=0.1)
from redis.sentinel import Sentinel
sentinel = Sentinel([
('redis-cloudflare', 26379),
],
sentinel_kwargs={'password': '******'},
password='******')
sentinel.discover_master('mymaster')
client = sentinel.master_for('mymaster', socket_timeout=0.5)
i = 93218
key = 'key' + str(i)
value = 'value' + str(i)
result = client.get(key)
print(result)
def healthz():
sentinel = Sentinel([('redis-0.redis', 26379)], socket_timeout=0.1)
try:
sentinel.discover_master("redis")
return "ok"
except MasterNotFoundError:
logger.info("Redis master not yet ready")
raise
def save_to_redis(self):
'''
Save JSON dump into Redis storage
as covid_data_ru and covid_data_en objects
'''
status = False
data = {}
for language in ('ru', 'en'):
data[language] = self.create_geojson(language)
try:
logger.info('Connecting to Redis Sentinel')
sentinel_client = Sentinel(
[('redis-sentinel', 26379)], socket_timeout=0.1)
logger.info('Discovering Redis Master')
sentinel_client.discover_master(REDIS_MASTER)
master = sentinel_client.master_for(
REDIS_MASTER, socket_timeout=0.1)
logger.info('Discovering Redis Slave')
slave = sentinel_client.slave_for(REDIS_MASTER, socket_timeout=0.1)
for language in ('ru', 'en'):
logger.info(f'Saving {language.upper()} covid_data into Slave')
status = slave.set('covid_data_' + language,
json.dumps(data[language], ensure_ascii=False))
if not status:
return False
return status
except:
logger.warning(
'Can not proceed with Redis. Saving JSON as a file.')
try:
for language in ('ru', 'en'):
with open(language + '.json', 'w') as outfile:
logger.info(f'Saving {language.upper()} as {language}.json')
json.dump(data[language], outfile, ensure_ascii=False)
except:
logger.warning('Can not save data as a file.')
return False
def connect(self, index=0, write=True):
"""
Creates a redis connection with connection pool.
"""
master_name, sentinel_hosts, db = self.parse_connection_string(self._connection_string)
sentinel_timeout = self._options.get("SENTINEL_TIMEOUT", 1)
sentinel = Sentinel(sentinel_hosts, socket_timeout=sentinel_timeout)
if write:
host, port = sentinel.discover_master(master_name)
else:
host, port = random.choice([sentinel.discover_master(master_name)] + sentinel.discover_slaves(master_name))
return self.connection_factory.connect(host, port, db)
def get_redis_master(redis_sentinel_nodes):
nodes_list = redis_sentinel_nodes.split(',')
sentinel = Sentinel([tuple(nodes_list[0].split(':'))], socket_timeout=0.1)
master = sentinel.discover_master('mymaster')
if not master:
sys.exit(1)
return list(master)
class RedisCluster(object):
def __init__(self, sentinel_hosts, service_name, **connection_kwargs):
self.service_name = service_name
self.sentinel = Sentinel(sentinel_hosts, **connection_kwargs)
self.master = None
self.master_host_port = None
self.reconfigure()
self.sentinel_threads = {}
self.listen()
def reconfigure(self):
try:
self.master_host_port = self.sentinel.discover_master(self.service_name)
self.master = self.sentinel.master_for(self.service_name)
log.info(f"Reconfigured master to {self.master_host_port}")
except Exception as ex:
log.error(f"Error while reconfiguring. {ex.args[0]}")
def listen(self):
def on_new_master(workerThread):
self.reconfigure()
for sentinel in self.sentinel.sentinels:
sentinel_host = sentinel.connection_pool.connection_kwargs["host"]
self.sentinel_threads[sentinel_host] = MyPubSubWorkerThread(
sentinel, on_new_master, msg_sleep_time=0.001, daemon=True
)
self.sentinel_threads[sentinel_host].start()
def add_queues_length_stats(self):
if len(self.queues) == 0:
return
sentinel_list = [(node.get('host', 'localhost'),
node.get('sentinel_port', 26379))
for node in self.node_list]
sentinel = Sentinel(sentinel_list, socket_timeout=5)
master_host = ''
queues_length_map = {}
for queue in self.queues:
queues_length_map[queue] = -1
try:
master_host, master_port = sentinel.discover_master(
self.master_name)
master_conn = sentinel.master_for(self.master_name,
db=self.db,
password=self.password)
for queue in self.queues:
try:
queues_length_map[queue] = master_conn.llen(queue)
except Exception:
continue
except Exception:
pass
for queue in self.queues:
self.add_gauge_value('Redis_Queues/%s' % queue,
None,
queues_length_map[queue],
count=1)
def __init__(self,
host=None,
port=6379,
db=0,
password=None,
sentinels=None,
service_name=None,
decode_responses=True,
socket_timeout=None,
log=None,
*args,
**kwargs):
if sentinels:
sentinel = Sentinel(sentinels=sentinels)
master = sentinel.discover_master(service_name=service_name)
host, port = master
self.server = {
'host': host,
'port': port,
'db': db,
'password': password,
'decode_responses': decode_responses,
'socket_timeout': socket_timeout
}
self.log = log if log else SimpleLog()
self.pool = ConnectionPool(*args, **self.server, **kwargs)
Redis.__init__(self, connection_pool=self.pool)
# Redis.__init__(self, *args, **self.server, **kwargs)
self.cache_result = {}
def getSentinelConnection(sentinelHosts):
sentinel = Sentinel(sentinelHosts, socket_timeout=0.1)
master = sentinel.discover_master('mymaster')
r = redis.StrictRedis(master[0],
port=int(master[1]),
db=int(config["redisDb"]))
return r
def __get_master(self, update=False):
master_ip = self.node.get_ip()
master_port = self.node.get_port()
master_node_id = self.node.get_node_id()
hosts = self.__get_hosts(refresh=update)
if len(hosts) > 0:
master_ip = hosts[0]['ip']
master_node_id = hosts[0]['node_id']
if os.path.isfile(Constants.MASTER_FILE):
fd = open(Constants.MASTER_FILE, "r")
temp_ip, temp_port = fd.readline().split(' ')
fd.close()
if temp_ip in hosts:
master_ip, master_port = temp_ip, temp_port
else:
os.remove(Constants.MASTER_FILE)
for host in hosts:
sentinel = Sentinel([(host["ip"], Constants.SENTINEL_PORT)], socket_timeout=0.1)
try:
ip, port = sentinel.discover_master(Constants.REDIS_SENTINEL_NAME)
for ht in hosts:
if ip == ht['ip']:
master_ip, master_port = ip, port
break
except:
pass
return master_ip, master_port
def check_sso_auth(user_id, brand_id, trans_type):
sentinel = Sentinel([('120.76.154.208', 26379)], socket_timeout=0.5)
log.d(sentinel.discover_master('mymaster'))
redis_obj = sentinel.master_for('mymaster', socket_timeout=0.1)
db_obj = PostgreSQLConnector()
sso_cls = SSOAuth(redis_obj, db_obj)
return sso_cls.check_auth(user_id, brand_id, trans_type)
def redis_connect(sentinel_addr, sentinel_port):
sentinel = Sentinel([(sentinel_addr, sentinel_port)], socket_timeout=0.1)
master_ip = sentinel.discover_master('mymaster')[0]
master_port = sentinel.discover_master('mymaster')[1]
print("Found master on {}:{} ".format(master_ip, master_port))
slaves = sentinel.discover_slaves('mymaster')
print("Found slaves on:")
for slave in slaves:
print(" {}:{}".format(slave[0], slave[1]))
conn = redis.StrictRedis(host=master_ip, port=master_port, db=0)
return conn
def getRedisClient():
ipaddrs = config['redis']['redis_ipaddrs']
arr = ipaddrs.split(",")
arr1 = []
for s in arr:
ipaddr = s.split(":")
t = (ipaddr[0], int(ipaddr[1]))
arr1.append(t)
sentinel = Sentinel(arr1,
socket_timeout=config['redis']['redis_socket_timeout'],
password=config['redis']['redis_password'])
master_name = config['redis']['redis_master_name']
sentinel.discover_master(master_name)
sentinel.discover_slaves(master_name)
master = sentinel.master_for(
master_name, socket_timeout=config['redis']['redis_socket_timeout'])
return master
def check_sso_auth(user_id, brand_id, trans_type ):
sentinel = Sentinel([('120.76.154.208', 26379)], socket_timeout=0.5)
log.d(sentinel.discover_master('mymaster'))
redis_obj = sentinel.master_for('mymaster', socket_timeout=0.1)
db_obj = PostgreSQLConnector()
sso_cls = SSOAuth(redis_obj,db_obj)
return sso_cls.check_auth(user_id, brand_id, trans_type)
def sso_log(user_id, brand_id, trans_type, data_type,data_content,log_time):
sentinel = Sentinel([('112.74.198.224', 26379)], socket_timeout=0.5)
log.d(sentinel.discover_master('mymaster'))
redis_obj = sentinel.master_for('mymaster', socket_timeout=0.1)
db_obj = PostgreSQLConnector()
sso_cls = SSOAuth(redis_obj,db_obj)
return sso_cls.send_log(user_id, brand_id, trans_type, data_type,data_content,log_time)
def discover_master(s: Sentinel):
master = None
try:
master = s.discover_master(redis_master_name)
except Exception as Err:
pass
return master or "No master found."
def sso_log(user_id, brand_id, trans_type, data_type, data_content, log_time):
sentinel = Sentinel([('112.74.198.224', 26379)], socket_timeout=0.5)
log.d(sentinel.discover_master('mymaster'))
redis_obj = sentinel.master_for('mymaster', socket_timeout=0.1)
db_obj = PostgreSQLConnector()
sso_cls = SSOAuth(redis_obj, db_obj)
return sso_cls.send_log(user_id, brand_id, trans_type, data_type,
data_content, log_time)
def connect(self, index=0, write=True):
"""
Creates a redis connection with connection pool.
"""
master_name, sentinel_hosts, db = self.parse_connection_string(
self._connection_string)
sentinel_timeout = self._options.get('SENTINEL_TIMEOUT', 1)
sentinel = Sentinel(sentinel_hosts, socket_timeout=sentinel_timeout)
if write:
host, port = sentinel.discover_master(master_name)
else:
host, port = random.choice(
[sentinel.discover_master(master_name)] +
sentinel.discover_slaves(master_name))
return self.connection_factory.connect(host, port, db)
def get_redis_url():
if hasattr(config, 'ETCD_HOST'):
return config.HERA_REDIS_SENTINEL
sentinel_list = [tuple(i.split(':')) for i in get_etcd_setting('HERA_REDIS_SENTINEL').split(',')]
redis_db = get_etcd_setting('HERA_REDIS_DB')
sentinel = Sentinel(sentinel_list, socket_timeout=0.1)
master_name = get_etcd_setting('HERA_REDIS_MASTERNAME')
return 'redis://{}/{}'.format(
':'.join(map(str, sentinel.discover_master(master_name))),
redis_db
)
def new_new(_, cls, broker_url, *args, **kwargs):
scheme = urlparse(broker_url).scheme
if scheme == "redis-sentinel":
from rpaas.tasks import app
opts = app.conf.BROKER_TRANSPORT_OPTIONS
s = Sentinel(opts["sentinels"], password=opts["password"])
host, port = s.discover_master(opts["service_name"])
return RedisBroker("redis://:{}@{}:{}".format(opts["password"], host, port))
else:
old_new(cls, broker_url, *args, **kwargs)
def _redis(self):
if getattr(settings, 'EXPERIMENTS_REDIS_SENTINELS', None):
sentinel = Sentinel(settings.EXPERIMENTS_REDIS_SENTINELS, socket_timeout=settings.EXPERIMENTS_REDIS_SENTINELS_TIMEOUT)
host, port = sentinel.discover_master(settings.EXPERIMENTS_REDIS_MASTER_NAME)
else:
host = getattr(settings, 'EXPERIMENTS_REDIS_HOST', 'localhost')
port = getattr(settings, 'EXPERIMENTS_REDIS_PORT', 6379)
password = getattr(settings, 'EXPERIMENTS_REDIS_PASSWORD', None)
db = getattr(settings, 'EXPERIMENTS_REDIS_DB', 0)
return redis.Redis(host=host, port=port, password=password, db=db)
def init_redis():
sentinel_list = [("127.0.0.1",26379),("127.0.0.1",26380),("127.0.0.1",23681)]
sentinel = Sentinel(sentinel_list,socket_timeout=0.5)
master = sentinel.discover_master("mymaster")
print(master)
slave = sentinel.discover_slaves("mymaster")
print(slave)
redis_conn = sentinel.master_for('mymaster', socket_timeout=0.5)
return redis_conn
def _redis(self):
if getattr(settings, 'EXPERIMENTS_REDIS_SENTINELS', None):
sentinel = Sentinel(settings.EXPERIMENTS_REDIS_SENTINELS, socket_timeout=settings.EXPERIMENTS_REDIS_SENTINELS_TIMEOUT)
host, port = sentinel.discover_master(settings.EXPERIMENTS_REDIS_MASTER_NAME)
else:
host = getattr(settings, 'EXPERIMENTS_REDIS_HOST', 'localhost')
port = getattr(settings, 'EXPERIMENTS_REDIS_PORT', 6379)
password = getattr(settings, 'EXPERIMENTS_REDIS_PASSWORD', None)
db = getattr(settings, 'EXPERIMENTS_REDIS_DB', 0)
return redis.Redis(host=host, port=port, password=password, db=db)
class redisSentinelHelper():
def __init__(self,sentinel_list,service_name,db):
self.sentinel = Sentinel(sentinel_list,socket_timeout=0.5)
self.service_name = service_name
self.db = db
def get_master_redis(self):
return self.sentinel.discover_master(self.service_name)
def get_slave_redis(self):
return self.sentinel.discover_slaves(self.service_name)
def set_key(self,key,value):
master = self.sentinel.master_for(
service_name=self.service_name,
socket_timeout=0.5,
db=self.db
)
return master.set(key,value)
def set_hash_key(self,hkey,key,value):
master = self.sentinel.master_for(
service_name=self.service_name,
socket_timeout=0.5,
db=self.db
)
return master.hset(hkey,key,value)
def get_key(self,key):
slave = self.sentinel.slave_for(
service_name=self.service_name,
socket_timeout=0.5,
db=self.db
)
return slave.get(key)
def get_hash_key(self,hkey,hey):
slave = self.sentinel.slave_for(
service_name=self.service_name,
socket_timeout=0.5,
db=self.db
)
return slave.hget(hkey,hey)
def delete(self,key):
master = self.sentinel.master_for(
service_name=self.service_name,
socket_timeout=0.5,
db=self.db
)
return master.delete(key)
def connect(self, index=0, write=True):
"""
Creates a redis connection with connection pool.
"""
master_name, sentinel_hosts, db = self.parse_connection_string(
self._connection_string)
sentinel_timeout = self._options.get('SENTINEL_TIMEOUT', 1)
sentinel = Sentinel(sentinel_hosts, socket_timeout=sentinel_timeout)
if write:
host, port = sentinel.discover_master(master_name)
else:
host, port = random.choice(
[sentinel.discover_master(master_name)] +
sentinel.discover_slaves(master_name))
kwargs = {
"db": db,
"parser_class": self.parser_class,
"password": self._options.get('PASSWORD', None),
}
kwargs.update({
'host': host,
'port': port,
'connection_class': Connection
})
if 'SOCKET_TIMEOUT' in self._options:
kwargs.update(
{'socket_timeout': int(self._options['SOCKET_TIMEOUT'])})
kwargs.update(self._pool_cls_kwargs)
connection_pool = get_or_create_connection_pool(
self._pool_cls, **kwargs)
connection = Redis(connection_pool=connection_pool)
return connection
def main(system, logger):
"""
Returns the IP address of the current Redis Master
"""
sentinel = Sentinel(
[(config.SENTINEL_IP, config.PORT)], socket_timeout=0.1)
master = sentinel.discover_master(config.REDIS_MASTER)
if master[0] == config.REDIS_MASTER_IP:
return int(1)
else:
return int(0)
def connect(self, index=0, write=True):
"""
Creates a redis connection with connection pool.
"""
master_name, sentinel_hosts, db = self.parse_connection_string(self._connection_string)
sentinel_timeout = self._options.get('SENTINEL_TIMEOUT', 1)
sentinel = Sentinel(sentinel_hosts, socket_timeout=sentinel_timeout)
if write:
host, port = sentinel.discover_master(master_name)
else:
host, port = random.choice([sentinel.discover_master(master_name)] + sentinel.discover_slaves(master_name))
sentinel_options = self._options.copy()
sentinel_options['CONNECTION_POOL_CLASS'] = 'redis.sentinel.SentinelConnectionPool'
sentinel_options['CONNECTION_POOL_KWARGS'] = {
'service_name': master_name,
'sentinel_manager': sentinel}
self.connection_factory = get_connection_factory(options=sentinel_options)
return self.connection_factory.connect(host, port, db)
def new_new(_, cls, broker_url, *args, **kwargs):
scheme = urlparse(broker_url).scheme
if scheme == 'redis-sentinel':
from rpaas.tasks import app
opts = app.conf.BROKER_TRANSPORT_OPTIONS
s = Sentinel(
opts['sentinels'],
password=opts['password'],
)
host, port = s.discover_master(opts['service_name'])
return RedisBroker('redis://:{}@{}:{}'.format(
opts['password'], host, port))
else:
old_new(cls, broker_url, *args, **kwargs)
def fetch_redis_info(conf):
info = {}
# 获取redis状态信息(0 dead, 1 master, -1 slave)
try:
r = redis.Redis(host=conf['host'],
port=conf['port'],
password=conf['password'],
socket_connect_timeout=5)
for k, v in r.info().items():
if k in conf['redis_info'].keys():
info[k] = v
elif k.startswith('db'):
for i in ['keys', 'expires']:
info[k + '_' + i] = v[i]
elif k == 'role':
if v == 'master':
info['redis_alive'] = 1
else:
info['redis_alive'] = -1
if info['maxmemory'] > 0:
info['mem_used_ratio'] = round(
float(info['used_memory']) / float(info['maxmemory']) * 100, 2)
else:
info['mem_used_ratio'] = 0
for k, v in r.info('commandstats').items():
if k + '_calls' in conf['redis_info'].keys():
info[k + '_calls'] = v['calls']
except redis.RedisError as e:
collectd.error('redis %s:%s connection error!' %
(conf['host'], conf['port']))
info['redis_alive'] = 0
# 获取sentinel状态信息 (0 dead, 1 leader, -1 leaf)
try:
s = Sentinel([(conf['host'], conf['sentinel_port'])],
socket_timeout=0.1)
if conf['host'] == s.discover_master(conf['sentinel_name'])[0]:
info['sentinel_alive'] = 1
else:
info['sentinel_alive'] = -1
except redis.RedisError as e:
collectd.error('sentinel %s:%s connection error!' %
(conf['host'], conf['sentinel_port']))
info['sentinel_alive'] = 0
return info
def __init__(self, sentinels, service, **kwargs):
sentinel = Sentinel(sentinels, **kwargs)
try:
master = sentinel.discover_master(service)
except MasterNotFoundError:
raise
try:
slaves = sentinel.discover_slaves(service)
except SlaveNotFoundError:
self._slaves = None
if version_info[0] < 3:
super(AutoRedisSentinel, self).__init__(master, slaves=slaves, **kwargs)
else:
super().__init__(master, slaves=slaves, **kwargs)
def __init__(self, sentinels, service, **kwargs):
sentinel = Sentinel(sentinels, **kwargs)
try:
master = sentinel.discover_master(service)
except MasterNotFoundError:
raise
try:
slaves = sentinel.discover_slaves(service)
except SlaveNotFoundError:
self._slaves = None
if version_info[0] < 3:
super(AutoRedisSentinel, self).__init__(master, slaves=slaves, **kwargs)
else:
super().__init__(master, slaves=slaves, **kwargs)
def run(self, terms, inject=None, **kwargs):
values = []
terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject)
for term in terms:
params = self.parse_params(term)
try:
conn = Sentinel(params['server'], socket_timeout=1)
res = conn.discover_master(params['name'])
if res is None:
res = ""
values.append(":".join((str(v) for v in res)))
except Exception, e:
print(e)
def connect():
server_1 = '192.168.11.101'
server_2 = '192.168.11.111'
server_3 = '192.168.11.112'
password = '******'
connection = Sentinel([(server_1, 26379), (server_2, 26379), (server_3, 26379)])
host, port = connection.discover_master('redis-cluster')
redis_client = StrictRedis(
host=host,
port=port,
password=password
)
print(host)
print(redis_client.set('city', 'feni'))
def get_redis_connection(sentinels=None, host=None, port=None, db_num=0):
""" Connect to redis using sentinels if defined or directly using given host and port.
:arg sentinels list of tuples (host, port)
:arg host redis host
:arg port redis port
:arg db_num redis db number
:return redis.Redis
:raise redis.RedisError"""
if sentinels is not None:
logging.debug('Asking sentinels for master address')
sentinel_conn = Sentinel(sentinels, socket_timeout=0.2)
host, port = sentinel_conn.discover_master(conf.sentinel_master_name)
logging.debug('Connecting to redis at ' + str(host) +
':' + str(port) + ' db=' + str(db_num))
return redis.Redis(host=host, port=port, db=db_num)
else:
return str_to_bool(os_var)
REDIS_MASTER_IP = None
REDIS_MASTER_PORT = None
REDIS_SENTINEL_ENABLED = bool_from_env('REDIS_SENTINEL_ENABLED', False)
if REDIS_SENTINEL_ENABLED:
from redis.sentinel import Sentinel
# REDIS_SENTINEL_HOSTS env variable format: host_1:port;host_2:port
REDIS_SENTINEL_HOSTS = os.environ['REDIS_SENTINEL_HOSTS'].split(';')
REDIS_CLUSTER_NAME = os.environ['REDIS_CLUSTER_NAME']
sentinel = Sentinel(
[tuple(s_host.split(':')) for s_host in REDIS_SENTINEL_HOSTS],
socket_timeout=float(
os.environ.get('REDIS_SENTINEL_SOCKET_TIMEOUT', 0.2)
)
)
REDIS_MASTER_IP, REDIS_MASTER_PORT = sentinel.discover_master(
REDIS_CLUSTER_NAME
)
REDIS_HOST = REDIS_MASTER_IP or os.environ.get('REDIS_HOST', '127.0.0.1')
REDIS_PORT = REDIS_MASTER_PORT or int(os.environ.get('REDIS_PORT', 6379))
REDIS_DB = int(os.environ.get('REDIS_DB', 0))
REDIS_PASSWORD = os.environ.get('REDIS_PASSWORD', None)
REDIS_SSL = bool_from_env('REDIS_SSL', False)
def getSentinelConnection(sentinelHosts):
sentinel = Sentinel(sentinelHosts, socket_timeout=0.1)
master = sentinel.discover_master('mymaster')
r = redis.StrictRedis(master[0], port=int(master[1]), db=int(config["redisDb"]))
return r
###########################################################
# 1.确保 所有哨兵端口都开启
# 2.所有的节点端口开启
# 3.配置文件 bind 不能为127.0.0.1 应该是 0.0.0.0
from redis.sentinel import Sentinel
# 连接哨兵服务器(主机名也可以用域名)
sentinel = Sentinel([('192.168.56.101', 26379)], socket_timeout=0.5)
# 获取主服务器地址
master = sentinel.discover_master('mymaster')
print(master)
# 输出：('172.31.0.2', 5001)
# 获取从服务器地址
slave = sentinel.discover_slaves('mymaster')
print(slave)
# 输出：[('172.31.3', 5001), ('172.31.0.4', 5001), ('172.31.0.5', 5001)]
# 获取主服务器进行写入
master = sentinel.master_for('mymaster', socket_timeout=0.5, db=15)
w_ret = master.set('foo', 'bar')
# 输出：True
# print 'error starting slave instance.'
# sys.exit(99)
raw_input('./start-slave.sh')
b = StrictRedis('localhost', 44411)
wait_for_redis(b)
# err = subprocess.call('./start-sentinel.sh')
# if err:
# print 'error starting sentinel instance.'
# sys.exit(99)
raw_input('./start-sentinel.sh')
c = StrictRedis('localhost', 55511)
wait_for_redis(c)
sentinel = Sentinel([('localhost', 55511)], socket_timeout=0.1)
if sentinel.discover_master(NAME)[1] != 33311:
print 'unexpected situation: expecting port 33311 to be the master'
sys.exit(99)
print """>> Getting an High Availability connection through the sentinel.
"""
mha = sentinel.master_for(NAME)
if int(mha.config_get()['port']) != 33311:
print 'unexpected master. expecting master to be on port 33311.'
sys.exit(99)
assert mha.ping() and a.ping()
print """>> Currently: (MASTER:{})<==(SLAVES:{})""".format(sentinel.discover_master(NAME),
sentinel.discover_slaves(NAME))
class agent(object):
def __init__(self):
pass
#by default,load the agent cnfig file
def load_config(self,config_file='./conf/agent.yml'):
self.config_file=config_file
with open(self.config_file,'r') as fd:
self.config = yaml.load(fd)
return self.config
def logger(self,path):
self.logger=logging.getLogger()
self.logger.setLevel(logging.WARNING)
fh=logging.FileHandler(path)
formatter = logging.Formatter("%(asctime)-15s %(filename)s [%(levelname)-8s] %(message)s")
fh.setFormatter(formatter)
self.logger.addHandler(fh)
return self.logger
#load the twemproxy config file by agent_conf_sn we specified
def load_twem_config(self,conf_sn):
self.config=self.load_config()
self.twem_config_file=self.config['agents'][conf_sn]['twem_config']
self.twem_config=self.load_config(self.twem_config_file)
return self.twem_config
#restart the twemproxy
def twem_restart(self,conf_sn): # conf_sn(config file session name) is 'cli1','cli2' etc. we can regard him as primary key in sql.
self.load_config()
cmd=self.config['agents'][conf_sn]['twem_cmd']
rs,rt = commands.getstatusoutput(cmd) # 'rs' means result, 'rt' mean return
self.logger.warning("%s"%rt)
return rt
def update_twem_master(self,conf_sn,sv_alias,new_ip,new_port):
self.load_twem_config(conf_sn)
self.twem_servers=self.twem_config['twem1']['servers']
self.logger.warning("NO,%s:%s twem_master != sentinel_master,Begin update twemproxy "%(conf_sn,sv_alias))
for i in range(len(self.twem_servers)):
if ( self.twem_servers[i].split(" ")[1] == sv_alias ):
self.logger.warning("twem old master:%s"%self.twem_servers[i].split(" ")[0].split(":"))
#print "[",datetime.now(),"]","old_master :",self.twem_servers[i].split(" ")[0].split(":")
weight=self.twem_servers[i].split(" ")[0].split(":")[2]
new_addr=new_ip+':'+str(new_port)+':'+weight+' '+sv_alias
self.logger.warning("twem new master:%s"%new_addr)
#print "[",datetime.now(),"]","new_master",new_addr
self.twem_servers[i]=new_addr
with open(self.twem_config_file,'w') as stream:
yaml.dump(self.twem_config,stream,allow_unicode=True,default_flow_style=False)
self.twem_restart(conf_sn)
#return the twemproxy's master addr (IP:PORT)
def get_twem_master(self,conf_sn,sv_alias):
self.load_twem_config(conf_sn)
self.twem_servers=self.twem_config['twem1']['servers']
for i in range(len(self.twem_servers)):
if( self.twem_servers[i].split(" ")[1] == sv_alias ):
self.twem_master=self.twem_servers[i].split(" ")[0].split(':')[0]+':'+self.twem_servers[i].split(" ")[0].split(':')[1]
#print 'twemproxy_master:',self.twem_master
self.logger.info("get twemproxy %s:%s master:%s"%(conf_sn,sv_alias,self.twem_master))
return self.twem_master
#retun the sentinel_master addr
def get_sentinel_master(self,conf_sn,sv_alias):
stats=1
self.load_config()
self.sentinel_host=self.config['agents'][conf_sn]['sentinel_host']
self.sentinel_port=self.config['agents'][conf_sn]['sentinel_port']
try:
self.sentinel=Sentinel([(self.sentinel_host,self.sentinel_port)],socket_timeout=1)
except :
self.logger.error('connection to Sentinel %s:%s failed '%(self.sentinel_host,self.sentinel_port))
while stats:
try:
self.sentinel_master=self.sentinel.discover_master(sv_alias)
stats=0
except:
time.sleep(0.5)
self.logger.error('get %s sentinel %s:%s failed %d times'%(conf_sn,self.sentinel_host,self.sentinel_port,stats))
stats = stats + 1
while (stats == 5):
stats=0
self.logger.error('retry failed,skip this sentinel %s:%s,please check it'%(self.sentinel_host,self.sentinel_port))
self.sentinel_master=self.sentinel_master[0]+':'+str(self.sentinel_master[1])
if (len(self.sentinel_master)<7):
return False
self.logger.error("get sentinel %s,%s master Failed"%(conf_sn,sv_alias))
else:
self.logger.info("get sentinel %s,%s master:%s"%(conf_sn,sv_alias,self.sentinel_master))
return self.sentinel_master
def __del__(self):
pass
signal.signal(signal.SIGINT,signal_handler)
try:
parser = OptionParser()
parser.add_option("-n",dest='numofkeys')
(options,args) = parser.parse_args()
if len(sys.argv) == 1:
print('please suppply sentinel ipaddrs')
sys.exit(1)
sentinel = Sentinel(
[(arg.split(':')[0],arg.split(':')[1]) for arg in args[1:]],
socket_timeout = 0.5
)
print('detected master is %s' % str(sentinel.discover_master('mymaster')))
time.sleep(5)
master = sentinel.master_for('mymaster',socket_timeout = 0.1)
#master = redis.StrictRedis(host='172.17.0.7')
except:
print(sys.exc_info()[0])
sys.exit(1)
print("key insertion to commence in 3s ...")
time.sleep(3)
value = datetime.datetime.now()
key,value = keygen.generate_kv_pair()
num = int(options.numofkeys) if options.numofkeys else -1
count = 0
import time
from redis.sentinel import Sentinel
sentinel = Sentinel([('192.168.11.35', 26379),('192.168.11.37', 26379),('192.168.11.37', 26379)], socket_timeout=0.1)
for i in range(1,1000):
print "ciclo " + str(i)
try:
print "master: " + str(sentinel.discover_master('mymaster'))
print "slave/s: " + str(sentinel.discover_slaves('mymaster'))
master = sentinel.master_for('mymaster', socket_timeout=0.1)
slave = sentinel.slave_for('mymaster', socket_timeout=0.1)
except:
print "sentinel eligiendo a un master"
clave = "clave" + str(i)
valor = "valor" + str(i)
try:
print "master.set " + clave + " " + valor
master.set(clave, valor)
print "master.get " + clave + " = " + str(master.get(clave))
print "master.set " + clave + " = " + str(slave.get(clave))
print "master.delete " + clave
master.delete(clave)
except:
print "cluster en modo solo lectura"
#!/usr/bin/python
from redis.sentinel import Sentinel
sentinel = Sentinel([('localhost', 19001)], socket_timeout=0.1)
master = sentinel.discover_master('redis-cluster')
slaves = sentinel.discover_slaves('redis-cluster')
print ""
print "Master(" + str(master) + ")"
print ""
print "Slaves(" + str(len(slaves)) + ") Nodes(" + str(slaves) + ")"
print ""
class Database:
"""
Provide an interface using which to communicate with a Redis server.
A new connection to the server is created by each class instance.
Custom exception hierarchy:
Error
+-- ConnectionError
+-- WaitTimeoutError
"""
socket_timeout = 1
pubsubs = {}
sentinel = None
class Error(Exception):
pass
# print "Error"
class ConnectionError(Error):
pass
# print "ConnectionError"
class WaitTimeoutError(Error):
def __str__(self):
return self.msg
def __init__(self, msg, key, value, timeout):
msg = str(msg) # in the event msg was an Exception, for example
self.msg = msg
self.key = key
self.value = value
self.timeout = timeout
super(RedisClient.WaitTimeoutError, self).__init__(self, msg) # this raises the exception
# Note: In Python 2.x, self.__class__ should not be used in place of base class name.
def state_connection(self):
"""
Return a dict containing keys and values providing information about
the state of the connection to the Redis server.
"""
if self.logger:
self.logger.debug("state_connection")
# conn_kwargs = self._pool.connection_kwargs
# server_addr = (conn_kwargs["host"], conn_kwargs["port"])
# state = {"Server address": "{}:{}".format(*server_addr),}
state = {}
# Do ping test
try:
# connection = redis.Redis(connection_pool=self._pool)
state["Server pingable"] = self.redis.ping() # returns True
except redis.exceptions.ConnectionError: # as e:
#self.logger.error("{} Console Redis ping failure: {}".format(tools.cm_name(), e))
# intentionally not self.logger.exception
state["Server pingable"] = False
# # Test availability of a key
# try:
# self["MONO_SV"]
# except self.ConnectionError: # as e:
# #self.logger.error("{} Console Redis test failure: {}".format(tools.cm_name(), e))
# # intentionally not self.logger.exception
# state["Test result"] = "Failed"
# else:
# state["Test result"] = "Passed"
return state
def state_server(self):
"""Return a dict containing keys and values providing some properties
about the the Redis server."""
if self.logger:
self.logger.debug("state_server")
try:
# connection = redis.Redis(connection_pool=self._pool)
info = self.redis.info()
except redis.exceptions.ConnectionError as e:
if self.logger:
self.logger.error("Redis info acquisition failure: {}".format(e))
# intentionally not self.logger.exception
state = {}
else:
state = {"Number of connected clients": info["connected_clients"],
"Redis version": info["redis_version"],
"Uptime": datetime.datetime.utcnow() - datetime.datetime.utcfromtimestamp(time.time() - info["uptime_in_seconds"]),
}
return state
def __init__(self,
host="localhost",
port=6379,
db=0,
password=None,
sentinels=None,
master=None,
settings=None,
logger=False):
"""Initialize the client."""
# Save passed-in variables
self.host = host
self.port = port
self.db = db
self.password = password
self.sentinels = sentinels
self.master = master
self.logger = logger
# Use the RAPD CONTROL_DATABASE_SETTINGS object
if settings:
# Sentinel
if settings["REDIS_CONNECTION"] == "sentinel":
self.host = None
self.port = None
self.db = None
self.password = settings.get("REDIS_PASSWORD", None)
self.sentinels = settings["REDIS_SENTINEL_HOSTS"]
self.master = settings["REDIS_MASTER_NAME"]
# Standard
else:
self.host = settings["REDIS_HOST"]
self.port = settings["REDIS_PORT"]
self.db = settings["REDIS_DB"]
self.password = settings.get("REDIS_PASSWORD", None)
self.sentinels = None
self.master = None
self._ConnectionError_last_log_time = float("-inf")
# Connect to server
self._connect_lock = threading.Lock()
self._connect() # "with self._connect_lock:" is not necessary in __init__
atexit.register(self._disconnect)
###############
# ADMIN Methods
###############
def _connect(self):
"""Connect to server."""
# self.logger.debug(tools.cm_name())
socket_timeout = self.socket_timeout
# Sentinel connection
if self.sentinels:
self.sentinel = Sentinel(self.sentinels)
# Assign master connection to self.redis
self.redis = self.sentinel.master_for(self.master)
# Standard connection
else:
attempt_count = 0
while attempt_count < CONNECTION_ATTEMPT_LIMIT:
try:
# Connect
self.redis = redis.Redis(host=self.host,
port=self.port,
db=self.db,
password=self.password)
# self._pool = redis.ConnectionPool(host=self.host, port=self.port, db=0)
# Make sure redis server is up
_ = self.redis.ping()
break
# self._connection = redis.Redis(host, socket_timeout=socket_timeout)
# redis module reconnects automatically as needed.
# socket_timeout is not set to a higher value because it blocks.
except redis.exceptions.ConnectionError as e:
print "except"
if self.logger:
self.logger.debug("Connection error {}".format(e))
time.sleep(1)
attempt_count += 1
# self._raise_ConnectionError(e)
# # Make sure redis server is up
# try:
# _ = self.redis.ping()
# except redis.sentinel.MasterNotFoundError as error:
# self._raise_ConnectionError(error)
def _disconnect(self):
"""Disconnect from server."""
# self.logger.debug(tools.cm_name())
try:
pool = self._pool
except AttributeError:
pass
else:
pool.disconnect()
#self.logger.info("{} Disconnected Redis client from Redis server {host}:{port}.".format(tools.cm_name(), **connection.connection_pool.connection_kwargs))
def _raise_ConnectionError(self, exc):
"""
Raise ConnectionError exception with an error message corresponding
to the provided exception.
"""
# TODO: Implement functionality of _raise_ConnectionError directly into ConnectionError exception instead. Delete method definition.
err_msg = str(exc)
if err_msg: err_msg = ": {}".format(err_msg)
err_msg = "Redis connection error{}".format(err_msg)
# Conditionally log error
# log_interval = backend.settings.redis["Console"]["ConnectionError log interval"]
# if time.time() > (self._ConnectionError_last_log_time + log_interval):
# self._ConnectionError_last_log_time = time.time()
# logger = self.logger.info
# else:
if self.logger:
logger = self.logger.debug
logger("{}".format(err_msg))
raise self.ConnectionError(err_msg)
def discover_master(self):
"""
Return the master instance (host, ip)
"""
if self.sentinel and self.master:
attempts = 0
while attempts < ATTEMPT_LIMIT:
try:
attempts += 1
master = self.sentinel.discover_master(self.master)
break
except redis.sentinel.MasterNotFoundError as e:
# Pause for specified time
print "try %d" % attempts
time.sleep(ATTEMPT_PAUSE)
else:
self._raise_ConnectionError(e)
return master
else:
self._raise_ConnectionError("Sentinels not properly defined")
def discover_slaves(self):
"""
Return the slave instances [(host, ip),]
"""
if self.sentinel and self.master:
attempts = 0
while attempts < ATTEMPT_LIMIT:
try:
attempts += 1
master = self.sentinel.discover_slaves(self.master)
break
except redis.sentinel.SlaveNotFoundError as e:
# Pause for specified time
print "try %d" % attempts
time.sleep(ATTEMPT_PAUSE)
else:
self._raise_ConnectionError(e)
return master
else:
self._raise_ConnectionError("Sentinels not properly defined")
#############
# PIPELINE Methods
#############
def pipeline(self):
"""return a pipeline instance"""
return self.redis.pipeline()
#############
# GET Methods
#############
def __getitem__(self, key, return_dict=False):
"""
Return the value of the specified key(s).
key can be either a str, or a tuple or list of strs. It is recommended
that key(s) be present in backend.redis_console_keys.keys.
`transform` indicates whether to apply transformations to the value
before returning it, as defined in the self._transform_value method.
`return_dict` indicates whether to return values in a dict. If True,
the keys in the returned dict match the specified key(s).
"""
# Get value(s)
if isinstance(key, str):
return self.get(key, return_dict)
elif isinstance(key, tuple) or isinstance(key, list):
return self.mget(key, return_dict)
get = __getitem__
@connectionErrorWrapper
def get(self, key, return_dict=False):
"""
Return the value of `key`.
`return_dict` indicates whether to return values in a dict. If True,
the keys in the returned dict match the specified key(s).
"""
# self.logger.debug("get key:{} return_dict:{}".format(key, return_dict))
# Retrieve value
value = self.redis.get(key)
# Wrap in dict if requested
return ({key: value} if return_dict else value)
# def get(self, key, return_dict=False):
# """
# Return the value of `key`.
# `return_dict` indicates whether to return values in a dict. If True,
# the keys in the returned dict match the specified key(s).
# """
# # self.logger.debug("get key:{} return_dict:{}".format(key, return_dict))
# # Retrieve value
# attempts = 0
# while attempts < ATTEMPT_LIMIT:
# try:
# attempts += 1
# value = self.redis.get(key)
# break
# except redis.exceptions.ConnectionError as e:
# # Pause for specified time
# print "try %d" % attempts
# time.sleep(ATTEMPT_PAUSE)
# else:
# self._raise_ConnectionError(e)
# return ({key: value} if return_dict else value)
@connectionErrorWrapper
def mget(self, keys, return_dict=False):
"""
Return the values of the sequence `keys`.
`transform` indicates whether to apply transformations to the value
before returning it, as defined in the self._transform_value method.
`return_dict` indicates whether to return values in a dict. If True,
the keys in the returned dict match the specified key(s).
"""
# Retrieve values
values = self.redis.mget(keys)
return (dict(zip(keys,values)) if return_dict else values)
# def mget(self, keys, return_dict=False):
# """
# Return the values of the sequence `keys`.
# `transform` indicates whether to apply transformations to the value
# before returning it, as defined in the self._transform_value method.
# `return_dict` indicates whether to return values in a dict. If True,
# the keys in the returned dict match the specified key(s).
# """
# # self.logger.log(5, "{} keys:{} transform:{} return_dict:{}".format(tools.cm_name(), keys, transform, return_dict))
# # Retrieve values
# try:
# connection = redis.Redis(connection_pool=self._pool)
# values = connection.mget(keys)
# except redis.exceptions.ConnectionError as e:
# self._raise_ConnectionError(e)
# # self.logger.log(5 if transform else logging.DEBUG, "{} keys:{} values:{}".format(tools.cm_name(), keys, repr(values)))
# # Transform and return values
# # if transform: values = self._transform_value(keys, values)
# return (dict(zip(keys,values)) if return_dict else values)
#############
# SET Methods
#############
@connectionErrorWrapper
def set(self, key, value):
"""
Set the indicated key value pair.
"""
if self.logger:
self.logger.debug("set key:{} value:{}".format(key, value))
self.redis.set(key, str(value))
# def set(self, key, value):
# """
# Set the indicated key value pair.
# """
# self.logger.debug("set key:{} value:{}".format(key, value))
# # Set
# attempts = 0
# while attempts < ATTEMPT_LIMIT:
# try:
# attempts += 1
# self.redis.set(key, str(value))
# break
# except redis.exceptions.ConnectionError as error:
# # Pause for specified time
# print "try %d" % attempts
# time.sleep(ATTEMPT_PAUSE)
# else:
# self._raise_ConnectionError(error)
__setitem__ = set
@connectionErrorWrapper
def mset(self, mapping):
"""
Set the key value pairs which are in the dict `mapping`.
"""
# self.logger.debug("{} mapping:{}".format(tools.cm_name(), mapping))
# Get the mapping ready
mapping = {k: str(v) for k, v in mapping.items()}
# Set in redis
self.redis.mset(mapping)
# def mset(self, mapping):
# """
# Set the key value pairs which are in the dict `mapping`.
# """
# # self.logger.debug("{} mapping:{}".format(tools.cm_name(), mapping))
# mapping = {k: str(v) for k, v in mapping.items()}
# # Set
# attempts = 0
# while attempts < ATTEMPT_LIMIT:
# try:
# attempts += 1
# self.redis.mset(mapping)
# break
# except redis.exceptions.ConnectionError as error:
# # Pause for specified time
# # print "try %d" % attempts
# time.sleep(ATTEMPT_PAUSE)
# else:
# self._raise_ConnectionError(error)
@connectionErrorWrapper
def setex(self, key, expire_time, value):
"""
Set the indicated key value pair with expire time.
"""
#if self.logger:
# self.logger.debug("set key:{} value:{}".format(key, value))
self.redis.setex(key, int(expire_time), value)
#############
# KEY Methods
#############
@connectionErrorWrapper
def keys(self, template):
"""Searches for keys fitting template."""
#self.logger.debug("set key:{} value:{}".format(key, value))
return self.redis.keys(template)
@connectionErrorWrapper
def delete(self, key):
"""Delete a key"""
#self.logger.debug("set key:{} value:{}".format(key, value))
return self.redis.delete(key)
##############
# LIST Methods
##############
@connectionErrorWrapper
def llen(self, key):
"""
LLEN get length of list
"""
value = self.redis.llen(key)
return value
@connectionErrorWrapper
def lpop(self, key):
"""
LPOP a value off a given list
"""
value = self.redis.lpop(key)
return value
@connectionErrorWrapper
def lpush(self, key, value):
"""
LPUSH a values onto a given list
"""
self.redis.lpush(key, value)
@connectionErrorWrapper
def brpop(self, key):
"""
RPOP a value off a given list
"""
value = self.redis.brpop(key)
return value
@connectionErrorWrapper
def rpop(self, key):
"""
RPOP a value off a given list
"""
value = self.redis.rpop(key)
return value
@connectionErrorWrapper
def rpoplpush(self, list1, list2):
"""
RPOPLPUSH pop a value off a given list and push on another list
"""
value = self.redis.rpoplpush(list1, list2)
#return value
##############
# HASH Methods
##############
@connectionErrorWrapper
def hget(self, key, field):
"""
HGET field on a key
"""
return self.redis.hget(key, field)
@connectionErrorWrapper
def hgetall(self, key):
"""
HGETALL a mapping on a key
"""
return self.redis.hgetall(key)
@connectionErrorWrapper
def hmset(self, key, mapping):
"""
HMSET a key with the attached mapping
"""
self.redis.hmset(key, mapping)
@connectionErrorWrapper
def hset(self, key, field, value):
"""
Sets field in the hash stored at key to value.
"""
self.redis.hset(key, field, value)
################
# PUBSUB Methods
################
@connectionErrorWrapper
def publish(self, key, value):
"""
Publish a value on a given key
"""
# print "publish {} {}".format(key, value)
self.redis.publish(key, value)
####################
# EXPIRATION Methods
####################
@connectionErrorWrapper
def expire(self, key, seconds):
"""
Set a timeout on key
"""
self.redis.expire(key, seconds)
# def publish(self, key, value):
# """
# Publish a value on a given key
# """
# # print "publish {} {}".format(key, value)
# attempts = 0
# while attempts < ATTEMPT_LIMIT:
# try:
# attempts += 1
# self.redis.publish(key, value)
# break
# except redis.exceptions.ConnectionError as error:
# # Pause for specified time
# print "try %d" % attempts
# time.sleep(ATTEMPT_PAUSE)
# else:
# self._raise_ConnectionError(error)
@connectionErrorWrapper
def get_message(self, id):
"""
Get message on pubsub connection
"""
message = self.pubsubs[id].get_message()
return message
# def get_message(self, id):
# """
# Get message on pubsub connection
# """
# attempts = 0
# while attempts < ATTEMPT_LIMIT:
# try:
# attempts += 1
# message = self.pubsubs[id].get_message()
# break
# except redis.exceptions.ConnectionError as error:
# # Pause for specified time
# print "try %d" % attempts
# time.sleep(ATTEMPT_PAUSE)
# else:
# self._raise_ConnectionError(error)
# return message
def get_pubsub(self):
"""
Returns a pubsub connection - no error is triggered if no connection
"""
# Create the pubsub object
ps = self.redis.pubsub()
# Give it an id
id = len(self.pubsubs) + 1
ps._id = id
# Store a reference
self.pubsubs[id] = ps
# Return the id
return id
@connectionErrorWrapper
def psubscribe(self, id=False, pattern=False):
"""
Add pattern subscription to a pubsub object and return id of pubsub object
"""
if pattern:
# No id passed in - create pubsub object
if not id:
id = self.get_pubsub()
self.pubsubs[id].psubscribe(pattern)
return id
else:
raise TypeError("pattern must be specified")
# def psubscribe(self, id=False, pattern=False):
# """
# Add pattern subscription to a pubsub object and return id of pubsub object
# """
# # No id passed in - create pubsub object
# if not id:
# id = self.get_pubsub()
# attempts = 0
# while attempts < ATTEMPT_LIMIT:
# try:
# attempts += 1
# self.pubsubs[id].psubscribe(pattern)
# break
# except redis.exceptions.ConnectionError as error:
# # Pause for specified time
# # print "try %d" % attempts
# time.sleep(ATTEMPT_PAUSE)
# else:
# self._raise_ConnectionError(error)
# return id
@connectionErrorWrapper
def subscribe(self, id=None, channel=None):
"""
Add subscription to a pubsub object and return id of pubsub object
"""
if channel:
# No id passed in - create pubsub object
if not id:
id = self.get_pubsub()
self.pubsubs[id].subscribe(channel)
return id
else:
raise TypeError("channel must be specified")