class RedisDriver:
def __init__(self, config):
self.config = config
sentinels = [(c['ip'], c['port']) for c in self.config['sentinels']]
self.sentinel = Sentinel(sentinels,
socket_timeout=0.1,
db=self.config['db'],
decode_responses=True)
self.__master = None
self.__slave = None
@property
def master(self):
if False:
if not self.__master:
self.__master = self.sentinel.master_for(
self.config['name'], password=self.config.get('auth', ''))
return self.__master
else:
return self.sentinel.master_for(self.config['name'],
password=self.config.get(
'auth', ''))
@property
def slave(self):
if False:
if not self.__slave:
self.__slave = self.sentinel.slave_for(
self.config['name'], password=self.config.get('auth', ''))
return self.__slave
else:
return self.sentinel.slave_for(self.config['name'],
password=self.config.get(
'auth', ''))
class redisSentinelHelper():
def __init__(self,sentinel_list,service_name,db):
self.sentinel = Sentinel(sentinel_list,socket_timeout=0.5)
self.service_name = service_name
self.db = db
def get_master_redis(self):
return self.sentinel.discover_master(self.service_name)
def get_slave_redis(self):
return self.sentinel.discover_slaves(self.service_name)
def set_key(self,key,value):
master = self.sentinel.master_for(
service_name=self.service_name,
socket_timeout=0.5,
db=self.db
)
return master.set(key,value)
def set_hash_key(self,hkey,key,value):
master = self.sentinel.master_for(
service_name=self.service_name,
socket_timeout=0.5,
db=self.db
)
return master.hset(hkey,key,value)
def get_key(self,key):
slave = self.sentinel.slave_for(
service_name=self.service_name,
socket_timeout=0.5,
db=self.db
)
return slave.get(key)
def get_hash_key(self,hkey,hey):
slave = self.sentinel.slave_for(
service_name=self.service_name,
socket_timeout=0.5,
db=self.db
)
return slave.hget(hkey,hey)
def delete(self,key):
master = self.sentinel.master_for(
service_name=self.service_name,
socket_timeout=0.5,
db=self.db
)
return master.delete(key)
def run(name):
sentinel = Sentinel([
('10.5.5.194', 26379),
('10.5.5.155', 26379),
('10.5.5.128', 26379),
],
socket_timeout=0.5)
master = sentinel.master_for('mymaster',
socket_timeout=0.5,
password='******',
db=0)
slave = sentinel.slave_for('mymaster',
socket_timeout=0.5,
password='******',
db=0)
# 判断是否所传参数对应的value值是否为1
while True:
time.sleep(0.05)
if slave.get(name) == "0" or slave.get(name) is None:
continue
else:
break
master.set(name, "0")
f_do = sys.argv[1] + '.do'
f_dat = sys.argv[1] + '.dat'
f = open(f_dat, 'w')
f.write('0' + '\n')
f.write('success' + '\n')
f.write('set ' + name + '为0 \n')
f.close()
f = open(f_do, 'w')
f.write('')
f.close()
def _init(self, server, params):
super(CacheClass, self).__init__(params)
self._server = server
self._params = params
self.service_name = self.options.get("SERVICE_NAME", 'mymaster')
self.slave_read = self.options.get("READ_SLAVE", False)
if self.server:
def _get_hosttupe(conn):
host_lst = conn.rsplit(':', 1)
return host_lst[0], int(host_lst[1])
conn_list = [_get_hosttupe(conn) for conn in self.server]
else:
raise ImproperlyConfigured("sentinel server config error.")
kwargs = self.connection_pool_class_kwargs
max_connections = kwargs.pop('max_connections')
sentinel_manager = Sentinel(conn_list, sentinel_kwargs=kwargs)
kwargs.update({
'db': self.db,
'password': self.password,
'max_connections': max_connections
})
self._client = sentinel_manager.master_for(self.service_name,
connection_pool_class=self.connection_pool_class,
**kwargs)
if self.slave_read:
self._slave_client = sentinel_manager.slave_for(self.service_name,
connection_pool_class=self.connection_pool_class,
**kwargs)
class TestApplication(Application):
def __init__(self):
# Call to the superclass to bootstrap.
super(TestApplication, self).__init__(name='krux-redis-sentinel')
self.logger.debug('Parsing sentinels from args: %r', self.args.sentinel)
sentinels = hostports_from_args(self.args)
self.logger.debug('Parsed sentinel host, port pairs: %r', sentinels)
self.logger.debug('Initializing Sentinel instance...')
self.sentinel = Sentinel(sentinels, socket_timeout=SOCKET_TIMEOUT)
self.logger.debug('Initialized Sentinel instance: %r', self.sentinel)
def add_cli_arguments(self, parser):
add_sentinel_cli_arguments(parser)
def run(self):
master = self.sentinel.master_for(self.name)
slave = self.sentinel.slave_for(self.name)
try:
self.logger.info('Ping master: %s', master.ping())
except ConnectionError as err:
self.logger.warning('Could not connect to master!')
self.logger.error(err)
try:
self.logger.info('Ping slave: %s', slave.ping())
except ConnectionError as err:
self.logger.warning('Could not connect to slave!')
self.logger.error(err)
def create_app():
'''工厂方法'''
# 创建app对象
app = Flask(__name__)
# 加载配置
config = config_dict.get(app.config.get('ENV'))
app.config.from_object(config)
# 加载数据库配置
from common.models import db
db.init_app(app)
# 记录日志信息
log_file(config.LOG_LV)
# redis集群加载
from rediscluster import RedisCluster
app.redis_cluster = RedisCluster(
startup_nodes=app.config.get('REDIS_CLUSTER'))
# redis哨兵,主从配置
from redis.sentinel import Sentinel
sentinel = Sentinel(app.config.get('REDIS_SENTINELS'))
app.redis_master = sentinel.master_for(
app.config.get('REDIS_SENTINEL_SERVICES_NAME'))
app.redis_slave = sentinel.slave_for(
app.config.get('REDIS_SENTINEL_SERVICES_NAME'))
# 注册用户蓝图到app中
from .resources.users import user_blue
app.register_blueprint(user_blue)
return app
def get_redis_connection(config, is_master=True):
"""
Returns redis connection basing on configuration dict
Args:
config: Dict with configuration, if key `SENTINELS` is present
sentinel connection pool will be used.
is_master: Whether connection should be master or slave,
applies only if sentinels are used
Returns: StrictRedis connection
"""
if 'SENTINELS' in config:
sentinel = Sentinel(
config['SENTINELS'],
db=config.get('DB'),
password=config.get('PASSWORD'),
socket_timeout=config.get('TIMEOUT'),
socket_connect_timeout=config.get('CONNECT_TIMEOUT'),
)
if is_master:
return sentinel.master_for(settings.REDIS_CLUSTER_NAME)
else:
return sentinel.slave_for(settings.REDIS_CLUSTER_NAME)
else:
return StrictRedis(
host=config['HOST'],
port=config['PORT'],
db=config.get('DB'),
password=config.get('PASSWORD'),
socket_timeout=config.get('TIMEOUT'),
socket_connect_timeout=config.get('CONNECT_TIMEOUT'),
)
def init_redis_client(cls):
# 防止多线程竞争，获取到锁后依然可能已经初始化了
if cls.redis_objs:
return
cls.instance_name = conf.get('sentinel', "")
cls.socket_timeout = conf.get("socket_timeout", 5)
cls.connect_timeout = conf.get("connect_timeout", 0.1)
if not cls.instance_name:
# 单例模式
host = conf["host"]
cls.redis_objs[cls.MODE_READ] = Redis(
host=host.split(':')[0],
port=host.split(':')[1],
socket_timeout=cls.socket_timeout,
socket_connect_timeout=cls.connect_timeout,
retry_on_timeout=1
)
cls.redis_objs[cls.MODE_WRITE] = cls.redis_objs[cls.MODE_READ]
else:
# 哨兵模式
sentinel = Sentinel(
cls.parse_config(),
socket_timeout=cls.socket_timeout,
socket_connect_timeout=cls.connect_timeout,
retry_on_timeout=1
)
cls.redis_objs[cls.MODE_READ] = sentinel.slave_for(cls.instance_name)
cls.redis_objs[cls.MODE_WRITE] = sentinel.master_for(cls.instance_name)
def get_connection(self, is_read_only=False) -> redis.StrictRedis:
"""
Gets a StrictRedis connection for normal redis or for redis sentinel based upon redis mode in configuration.
:type is_read_only: bool
:param is_read_only: In case of redis sentinel, it returns connection to slave
:return: Returns a StrictRedis connection
"""
if self.connection is not None:
return self.connection
if self.is_sentinel:
kwargs = dict()
if self.password:
kwargs["password"] = self.password
sentinel = Sentinel([(self.host, self.port)], **kwargs)
if is_read_only:
connection = sentinel.slave_for(self.sentinel_service, decode_responses=True)
else:
connection = sentinel.master_for(self.sentinel_service, decode_responses=True)
else:
connection = redis.StrictRedis(host=self.host, port=self.port, decode_responses=True,
password=self.password)
self.connection = connection
return connection
def _connection(self):
sentinel = Sentinel(self._sentinels,
socket_timeout=self._socket_timeout)
slave = sentinel.slave_for(self._sentinel_master,
socket_timeout=self._socket_timeout,
db=self._db)
return slave
def test_simple_sentinel(self):
# Setup Master & Slave
master = RedisDeployment('~/redis-master', port=3428)
salve = RedisDeployment('~/redis-slave', port=3429)
master.start()
salve.start(master_ip='127.0.0.1', master_port=3428)
# Setup Sentinel
redis_sentinel = RedisSentinel('~/redis-sentinel')
redis_sentinel.start(master_ip='127.0.0.1',master_port=3428, master_name='mymaster')
sleep(3)
## len(RedisSentinel.list_running_instances()) == 1
## len(RedisDeployment.list_running_instances()) == 2
# Client API
sentinel = Sentinel([('localhost', redis_sentinel.port)], socket_timeout=0.1)
master_client = sentinel.master_for('mymaster')
slave_client = sentinel.slave_for('mymaster')
master_client.set('foo', 'bar')
slave_client.get('foo') # bar
self.assertTrue(len(RedisSentinel.list_running_instances()) == 1)
self.assertTrue(len(RedisDeployment.list_running_instances()) == 2)
self.assertTrue(str(slave_client.get('foo').decode('utf-8')) == 'bar')
# Close down the servers
master.stop()
salve.stop()
redis_sentinel.stop()
def __init__(self):
"""
init setting and create redis instance
"""
settings = Settings.get("redis")
self.prefix = settings.get("prefix")
self.queue_max_len = settings.get("queue_max_len")
self.sentinel = settings.get("sentinel")
if self.sentinel:
sentinel = Sentinel(sentinels=map(lambda x: x.split(":"),
settings.get("sentinel_hosts")))
kwargs = dict(
service_name=settings.get("sentinel_master"),
password=settings.get("password"),
decode_responses=True,
)
self.master = sentinel.master_for(**kwargs)
self.slave = sentinel.slave_for(**kwargs)
else:
pool = redis.ConnectionPool(
host=settings.get("host"),
port=settings.get("port"),
db=settings.get("db"),
password=settings.get("password"),
decode_responses=True,
)
self.master = self.slave = redis.StrictRedis(connection_pool=pool)
def connect2redis(host, port=6379, db=0, connectionPool=True, logging=None, sentinel_service=prms.REDIS_SENTINEL_SERVICE_NAME, master=True):
if sentinel_service:
retry_count = 0
while retry_count < 10:
sentinel = Sentinel([(host, 26379)])
if master:
client = sentinel.master_for(sentinel_service)
else:
client = sentinel.slave_for(sentinel_service)
if is_available(client, logging=logging):
logging.info("Connected to Redis succesfully.")
return client
else:
print(f"Waiting for {pow(2, retry_count)} secs.")
time.sleep(pow(2, retry_count))
retry_count += 1
else:
retry_count = 0
while retry_count < 10:
if connectionPool:
pool = redis.ConnectionPool(host=host, port=port, db=db)
client = redis.StrictRedis(connection_pool=pool)
else:
client = redis.StrictRedis(host=host, port=port, db=db)
if is_available(client, logging=logging):
logging.info("Connected to Redis succesfully.")
return client
else:
print(f"Waiting for {pow(2, retry_count)} secs.")
time.sleep(pow(2, retry_count))
retry_count += 1
logging.error("Exiting...")
exit(3)
class SentinelProxy:
"""Proxy for Redis sentinel."""
def __init__(self,
sentinel_host,
master_name,
socket_timeout=0.1,
**kwargs):
"""Initialize Redis sentinel connection.
:params: sentinel_host: (host, port)
"""
self.sentinel = Sentinel([sentinel_host],
socket_timeout=socket_timeout)
self.master = self.sentinel.master_for(master_name,
socket_timeout=socket_timeout,
**kwargs)
self.slave = self.sentinel.slave_for(master_name,
socket_timeout=socket_timeout,
**kwargs)
def __getattr__(self, name):
"""Get attribute from Redis master or slave."""
master_key = ('set', 'hset', 'hmset', 'lset', 'lpush', 'blpop',
'brpop', 'rpush', 'expire', 'delete', 'incr')
if name not in master_key:
target = self.slave
else:
target = self.master
return getattr(target, name)
def get_connection(self, is_read_only=False) -> redis.Redis:
"""
Gets a Redis connection for normal redis or for redis sentinel based upon redis mode in configuration.
:type is_read_only: bool
:param is_read_only: In case of redis sentinel, it returns connection to slave
:return: Returns a Redis connection
"""
if self.connection is not None:
return self.connection
if self.is_sentinel:
kwargs = dict()
if self.password:
kwargs["password"] = self.password
sentinel = Sentinel([(self.host, self.port)], **kwargs)
if is_read_only:
connection = sentinel.slave_for(self.sentinel_service,
decode_responses=True)
else:
connection = sentinel.master_for(self.sentinel_service,
decode_responses=True)
else:
connection = redis.Redis(host=self.host,
port=self.port,
decode_responses=True,
password=self.password)
self.connection = connection
return connection
def redis_conn(readonly=True):
global sentinel, master_client, slave_client
if not sentinel:
sentinel = Sentinel(settings.REDIS_SENTINEL, socket_timeout=1)
if not master_client:
master_client = sentinel.master_for(settings.REDIS_CLUSTER)
if not slave_client:
slave_client = sentinel.slave_for(settings.REDIS_CLUSTER)
return slave_client if readonly else master_client
def get_database(config, master=True):
if config['sentinel']:
sentinal = Sentinel(config['sentinel'], socket_timeout=0.1,
password=config['redis_password'], db=config['redis_database'])
if master:
return sentinal.master_for(config['sentinel_cluster_name'])
else:
return sentinal.slave_for(config['sentinel_cluster_name'])
else:
return Redis.from_url(config['redis'])
def __conn_redis(self):
sentinel = Sentinel(
self.REDIS_CONFIG['SENTINELS'],
socket_timeout=0.1,
decode_responses=self.REDIS_CONFIG['DECODE_RESPONSES'],
**self.conn_kwargs)
self.redis_master = sentinel.master_for(
self.REDIS_CONFIG['SERVICE_NAME'], socket_timeout=0.1)
self.redis_slave = sentinel.slave_for(
self.REDIS_CONFIG['SERVICE_NAME'], socket_timeout=0.1)
def get_redis_conn(readonly=False):
global sentinel, _cached_master_client, _cached_slave_client
if not sentinel:
sentinel = Sentinel(settings.REDIS_SENTINEL, socket_timeout=0.1)
if not _cached_master_client:
_cached_master_client = sentinel.master_for(
settings.REDIS_SNS_CLUSTER_NAME)
if not _cached_slave_client:
_cached_slave_client = sentinel.slave_for(
settings.REDIS_SNS_CLUSTER_NAME)
return _cached_slave_client if readonly else _cached_master_client
def create_sentinel_redis(config):
logger.info('start to connect sentinel redis: host-port:%s, db: %s' %
(config['host-pot'], config['db']))
host_port = parse_host(config["host-port"])
sentinel = Sentinel(host_port,
socket_timeout=2,
password=config['pwd'],
db=config['db'])
master = sentinel.master_for('mymaster', socket_timeout=5)
slave = sentinel.slave_for('mymaster', socket_timeout=2)
return master, slave
def main():
rd = redis.Redis(host='localhost',
port=6379,
db=0,
password=None,
socket_timeout=None)
#rd = redis.StrictRedis( host = 'localhost', port = 6379, db = 0, password = None, socket_timeout = None )
key = "test1"
value = "test1_val"
rd.set(key, value)
result = rd.get(key)
print("result =\n")
print(f'{ key } = { result }\n')
# mset
data = {"test2": "test2_val", "test3": "test3_val", "test4": "test4_val"}
rd.mset(data)
result = rd.get("test2")
print("result =\n")
print(f'{ key } = { result }\n')
# value: JSON data
key = "test1_sub"
value = {
"test1_sub1": "test1_sub1_value",
"test1_sub1": "test1_sub1_value",
"test1_sub1": "test1_sub1_value"
}
value = json.dumps(value, ensure_ascii=False).encode("utf-8")
rd.set(key, value)
result = rd.get(key)
result = result.decode("utf-8")
result = dict(json.loads(result))
print("result =\n")
print(f'{ key } = { result }\n')
rd.close()
# Sentinel
sentinel = Sentinel([('localhost', 26379)], socket_timeout=0.1)
master = sentinel.master_for('my_redis_master', socket_timeout=0.1)
key = "master_test1"
value = "master_test1_val"
master.set(key, value)
result = master.get(key)
print("result (master) =\n")
print(f'{ key } = { result }\n')
slave = sentinel.slave_for('my_redis_master', socket_timeout=0.1)
result = slave.get(key)
print("result (slave) =\n")
print(f'{ key } = { result }\n')
sentinel.close()
def get_redis_conn():
global MYSETINEL
global MASTER
global SLAVE
# 如果哨兵连接实例已存在, 不重复连接, 当连接失效时, 重新连接
if not MYSETINEL: # 连接哨兵
MYSETINEL = Sentinel(SENTINEADDRESS,
socket_timeout=2000) # 尝试连接最长时间单位毫秒, 1000毫秒为1秒
# 通过哨兵获取主数据库连接实例 参数1: 主数据库的名字(集群部署时在配置文件里指明)
MASTER = MYSETINEL.master_for('mymaster', socket_timeout=2000)
# 通过哨兵获取从数据库连接实例 参数1: 从数据的名字(集群部署时在配置文件里指明)
SLAVE = MYSETINEL.slave_for('mymaster', socket_timeout=2000)
def create_app(Config, enable_config_file=False):
"""
创建应用的工厂函数
:param config: 配置信息类
:param enable_config_file: 是否允许环境变量中的配置文件覆盖已加载的配置信息
:return: 应用对象
"""
app = create_flask_app(Config, enable_config_file)
"""项目运行需要的配置"""
# 配置日志
from utils.logging import create_logger
create_logger(app)
# 配置redis哨兵
from redis.sentinel import Sentinel
_sentinel = Sentinel(app.config.get('REDIS_SENTINELS'))
# 根据哨兵设置主、从服务
app.redis_master = _sentinel.master_for(
app.config['REDIS_SENTINEL_SERVICE_NAME'])
app.redis_slave = _sentinel.slave_for(
app.config['REDIS_SENTINEL_SERVICE_NAME'])
# Redis集群
from rediscluster import StrictRedisCluster
app.redis_cluster = StrictRedisCluster(
startup_nodes=app.config['REDIS_CLUSTER'])
# MySQL数据库连接初始化
from models import db
db.init_app(app)
# 创建Snowflake ID worker--雪花算法
from utils.snowflake.id_worker import IdWorker
app.id_worker = IdWorker(app.config['DATACENTER_ID'],
app.config['WORKER_ID'], app.config['SEQUENCE'])
"""添加请求钩子"""
from utils.middlewares import jwt_authentication
app.before_request(jwt_authentication)
"""注册蓝图"""
# 用户蓝图
from toutiao.resources.user import user_bp
app.register_blueprint(user_bp)
"""定时任务，每天3点更正我们redis和mysql的数据"""
# 初始化调度器，并配置最大开始10个线程(不指定，默认10个)
bg_scheduler = BackgroundScheduler(
executor={'default': ThreadPoolExecutor()})
# 添加任务函数
# bg_scheduler.add_job('任务函数', '执行器', '执行周期时间')
# bg_scheduler.add_job('任务函数', 'cron', hours=3) # 每天3点执行
from toutiao.aps_scheduler.statistic_data import fix_process
bg_scheduler.add_job(fix_process, 'date', args=[app]) # 为了测试让他立即执行
# 执行任务
bg_scheduler.start()
return app
def save_to_redis(self):
'''
Save JSON dump into Redis storage
as covid_data_ru and covid_data_en objects
'''
status = False
data = {}
for language in ('ru', 'en'):
data[language] = self.create_geojson(language)
try:
logger.info('Connecting to Redis Sentinel')
sentinel_client = Sentinel(
[('redis-sentinel', 26379)], socket_timeout=0.1)
logger.info('Discovering Redis Master')
sentinel_client.discover_master(REDIS_MASTER)
master = sentinel_client.master_for(
REDIS_MASTER, socket_timeout=0.1)
logger.info('Discovering Redis Slave')
slave = sentinel_client.slave_for(REDIS_MASTER, socket_timeout=0.1)
for language in ('ru', 'en'):
logger.info(f'Saving {language.upper()} covid_data into Slave')
status = slave.set('covid_data_' + language,
json.dumps(data[language], ensure_ascii=False))
if not status:
return False
return status
except:
logger.warning(
'Can not proceed with Redis. Saving JSON as a file.')
try:
for language in ('ru', 'en'):
with open(language + '.json', 'w') as outfile:
logger.info(f'Saving {language.upper()} as {language}.json')
json.dump(data[language], outfile, ensure_ascii=False)
except:
logger.warning('Can not save data as a file.')
return False
def wrapper(*args, **kwargs):
if os.environ.get('PYBOSSA_REDIS_CACHE_DISABLED') is None:
key = "%s::%s" % (settings.REDIS_KEYPREFIX, key_prefix)
sentinel = Sentinel(settings.REDIS_SENTINEL, socket_timeout=0.1)
master = sentinel.master_for('mymaster')
slave = sentinel.slave_for('mymaster')
output = slave.get(key)
if output:
return pickle.loads(output)
else:
output = f(*args, **kwargs)
master.setex(key, timeout, pickle.dumps(output))
return output
else:
return f(*args, **kwargs)
def setUpClass(cls):
cls.env_num = cls.a.get_num_name("环境")
cls.apiName_num = cls.a.get_num_name("接口名称")
cls.method_num = cls.a.get_num_name("请求方法")
cls.headers_num = cls.a.get_num_name("请求头")
cls.para_num = cls.a.get_num_name("请求参数")
cls.desc_num = cls.a.get_num_name("用例描述")
cls.data_num = cls.a.get_num_name("请求体")
cls.expect_num = cls.a.get_num_name("预期结果")
cls.isSkip_num = cls.a.get_num_name("是否跳过该用例")
cls.relateData_num = cls.a.get_num_name("接口关联参数")
# 线上楼盘ID
sss["Restful_estateId2"] = json.dumps({"estateId": "1041394"})
sss["Restful_estateId3"] = json.dumps({"estateId": "22374"})
# 查看线上签约信息
sss["Restful_signId"] = json.dumps({
"estateId": "22374",
"signId": "116"
})
# 查看测试环境签约信息
sss["Restful_signId_test"] = json.dumps({
"estateId": "889",
"signId": "66"
})
# #
SENTINEADDRESS = [('10.50.255.117', 26380), ('10.50.255.118', 26380),
('10.50.255.119', 26380)]
# 尝试连接最长时间单位毫秒, 1000毫秒为1秒
MYSETINEL = Sentinel(SENTINEADDRESS, socket_timeout=3000)
# 通过哨兵获取从数据库连接实例,用于数据读取
SLAVE = MYSETINEL.slave_for('redis.bp.fdd', socket_timeout=3000)
# 通过哨兵获取主数据库连接实例，用于数据写入
MASTER = MYSETINEL.master_for('redis.bp.fdd', socket_timeout=3000)
# 读取redis中的签约时间戳
sign_end_time = SLAVE.get('market_dev.ddxf.bp.fdd:sign_end_time')
if sign_end_time is None:
# 初始化一个时间戳
MASTER.setex('market_dev.ddxf.bp.fdd:sign_end_time', '86400',
1582819199999)
sss["sign_end_time"] = 1582819199999
else:
sss["sign_end_time"] = int(sign_end_time)
new_end_time = int(sign_end_time) + 86400000
# 将新的时间戳写入redis中，有效期1天
MASTER.setex('market_dev.ddxf.bp.fdd:sign_end_time', '86400',
new_end_time)
def get_client(self, mode, timeout=None):
# def _curr_session_time_out():
# return timeout if timeout is not None else self.sock_timeout
if timeout is not None:
# 如果指定了超时，就临时生成一个符合这个超时的连接，使用短连接，这个场景主要应对DI轮训服务，不能频繁超时
if not self.__class__.instance_name:
host = conf["host"]
return Redis(
host=host.split(':')[0],
port=host.split(':')[1],
socket_timeout=self.socket_timeout+2,
socket_connect_timeout=self.connect_timeout,
retry_on_timeout=1,
socket_keepalive=False
)
curr_sentinel = Sentinel(
self.parse_config(),
socket_timeout=self.socket_timeout,
socket_connect_timeout=self.connect_timeout,
)
if mode == self.MODE_WRITE:
return curr_sentinel.master_for(
self.instance_name,
socket_timeout=timeout+2,
socket_connect_timeout=self.connect_timeout,
socket_keepalive=False, # 短连接
retry_on_timeout=True
)
else:
return curr_sentinel.slave_for(
self.instance_name,
socket_timeout=timeout+2,
socket_connect_timeout=self.connect_timeout,
socket_keepalive=False,
retry_on_timeout=True
)
else:
return self.__class__.redis_objs[mode]
class RedisStorage(Storage):
def __init__(self, env: dict):
self._redis_service_name = env.get("YOCOTTO_URL_REDIS_SERVICE_NAME",
"mymaster")
self._redis_socket_timeout = float(
env.get("YOCOTTO_URL_REDIS_SOCKET_TIMEOUT", 0.1))
self._redis_sentinel = Sentinel(
json.loads(
env.get("YOCOTTO_URL_REDIS_SENTINELS",
'[["redis-sentinel", 26379]]')))
self._redis_master = self._redis_sentinel.master_for(
self._redis_service_name,
socket_timeout=self._redis_socket_timeout)
self._redis_slave = self._redis_sentinel.slave_for(
self._redis_service_name,
socket_timeout=self._redis_socket_timeout)
def get(self, url_id: str) -> ShortenedUrl:
return ShortenedUrl(**json.loads(self._redis_slave.get(url_id)))
def put(self, shortened_url: ShortenedUrl) -> ShortenedUrl:
url_id = shortened_url.url_id
res = self._redis_master.setnx(url_id,
json.dumps(asdict(shortened_url)))
if res == 1:
return shortened_url
else:
raise UrlAlreadyExists(f"The url_id: {url_id} already taken.")
def save(self, long_url: str, retry_count=5) -> ShortenedUrl:
if retry_count == 0:
raise StorageError("Failed to save URL.")
try:
shortened_url = ShortenedUrl(url_id=self._next_url_id(),
long_url=long_url)
self.put(shortened_url)
return shortened_url
except UrlAlreadyExists:
return self.save(long_url, retry_count=retry_count - 1)
def _next_url_id(self):
url_id = base_encode(self._redis_master.incr("yocotto_url_counter"))
return url_id
def connect_to_redis(read_only=False):
"""
Get a connection to the master redis service.
"""
logger.debug("Connecting to redis", server=redis_host, port=redis_sentinel_port)
if config.REDIS_USE_SENTINEL:
sentinel_service = Sentinel([(redis_host, redis_sentinel_port)], password=redis_pass, socket_timeout=5)
sentinel_name = config.REDIS_SENTINEL_NAME
if read_only:
logger.debug("Looking up read only redis slave using sentinel protocol")
r = sentinel_service.slave_for(sentinel_name)
else:
logger.debug("Looking up redis master using sentinel protocol")
r = sentinel_service.master_for(sentinel_name)
else:
r = redis.StrictRedis(host=redis_host, password=redis_pass)
return r
def create_app(config, enable_config_file=False):
"""
创建flask应用 并 初始化各组件
:param config: 配置类
:param enable_config_file: 是否允许运行环境中的配置文件覆盖已加载的配置信息
:return: flask应用
"""
app = create_flask_app(config, enable_config_file)
# 添加自定义正则转换器
from utils.converters import register_converters
register_converters(app)
# TODO 创建redis哨兵
from redis.sentinel import Sentinel
_sentinel = Sentinel(app.config['REDIS_SENTINELS'])
# 获取redis主从连接对象
app.redis_master = _sentinel.master_for(
app.config['REDIS_SENTINEL_SERVICE_NAME'])
app.redis_slave = _sentinel.slave_for(
app.config['REDIS_SENTINEL_SERVICE_NAME'])
# 配置日志
from utils.logging import create_logger
create_logger(app)
# 限流器
from utils.limiter import limiter as lmt
lmt.init_app(app)
# 注册用户模块蓝图
from .resources.user import user_bp
app.register_blueprint(user_bp)
# 注册新闻模块蓝图
from .resources.news import news_bp
app.register_blueprint(news_bp)
# 注册搜索模块蓝图
from .resources.search import search_bp
app.register_blueprint(search_bp)
return app
def connect_to_redis(read_only=False):
"""
Get a connection to the master redis service.
"""
logger.debug("Connecting to redis")
if config.REDIS_USE_SENTINEL:
sentinel = Sentinel([(redis_host, 26379)],
password=redis_pass,
socket_timeout=5)
if read_only:
logger.debug(
"Looking up read only redis slave using sentinel protocol")
r = sentinel.slave_for('mymaster')
else:
logger.debug("Looking up redis master using sentinel protocol")
r = sentinel.master_for('mymaster')
else:
r = redis.StrictRedis(host=redis_host, password=redis_pass)
return r
def __init__(self,
host="redis-master",
port=6379,
master_port=26379,
db=0):
super(Persister, self).__init__()
self.handler.module = "Redis Persister"
self.handler.log(message="Preparing redis connection")
master_node = None
sentinel = None
slave_nodes = [(host, port)]
try:
self.handler.log(
message="Checking if redis instance passed is a cluster")
sentinel = Sentinel([(host, master_port)], socket_timeout=0.1)
master_node = sentinel.discover_master('redis')
self.handler.log(message="It is a cluster. Setting master node")
except MasterNotFoundError:
self.handler.log(
message="No cluster found; using single redis instance only")
except ResponseError:
self.handler.log(
message="No cluster found; using single redis instance only")
except Exception:
raise
if master_node:
self.handler.log(message="Setting redis master for writes")
self._redis_master = sentinel.master_for("redis",
socket_timeout=0.1)
self._redis_connection = sentinel.slave_for("redis",
socket_timeout=0.1)
else:
self._redis_connection = redis.StrictRedis(host=host,
port=port,
db=db)
self.handler.log(message="Pointing redis master to connection")
self._redis_master = self._redis_connection
def index():
try:
stream = StringIO()
sentinel = Sentinel([('redis-sentinel', 26379)], socket_timeout=0.1)
dm = sentinel.discover_master('dpf')
ds = sentinel.discover_slaves('dpf')
print('master:', dm, file=stream)
print('slaves:', ds, file=stream)
master = sentinel.master_for('dpf', socket_timeout=0.1)
master.set('foo', time.asctime())
slave = sentinel.slave_for('dpf', socket_timeout=0.1)
print('get from redis:', slave.get('foo'), file=stream)
except MasterNotFoundError as e:
print('redis master is GONE, waiting for failover...', e, file=stream)
except Exception as e:
print('exception happened, connection lost?', e, file=stream)
return '{}\n'.format(stream.getvalue())
def connection_examples():
# Connect to a standard Redis deployment.
client = redis.Redis("localhost",
port=6379,
decode_responses=True,
max_connections=20)
# Read and write through the client.
client.set("foo", "bar")
client.get("foo")
# Connect to a Redis Sentinel deployment.
sentinel = Sentinel([('localhost', 26379)], socket_timeout=0.1)
master = sentinel.master_for("cache", socket_timeout=0.1)
slave = sentinel.slave_for("cache", socket_timeout=0.1)
# Run write operations with the master.
master.set("foo", "bar")
# Run read operations with the slave.
slave.get("foo")
def _create_sentinel_redis(self, config: dict) -> Tuple[Redis, Redis]:
"""
创建一个Redis的主从链接
:param config:
:return:
"""
def _parse_host(val: str) -> List[Tuple[str, int]]:
"""
@attention: 分解host,把10.1.113.158-26379分割为("10.1.113.158",26379)
"""
info = re.findall(r"([\d\.]+)-(\d+)", val)
return [(item[0], int(item[1])) for item in info]
host_port = _parse_host(config["host-port"])
sentinel = Sentinel(host_port,
socket_timeout=2,
password=config['pwd'],
db=config['db'])
master = sentinel.master_for('mymaster', socket_timeout=5)
slave = sentinel.slave_for('mymaster', socket_timeout=2)
return master, slave
def wrapper(*args, **kwargs):
if os.environ.get('PYBOSSA_REDIS_CACHE_DISABLED') is None:
key = "%s:%s_args:" % (settings.REDIS_KEYPREFIX, f.__name__)
key_to_hash = ""
for i in args:
key_to_hash += ":%s" % i
key = key + ":" + hashlib.md5(key_to_hash).hexdigest()
#key += "_kwargs"
#for i in frozenset(kwargs.items()):
# key += ":%s" % i
sentinel = Sentinel(settings.REDIS_SENTINEL, socket_timeout=0.1)
master = sentinel.master_for(settings.REDIS_MASTER)
slave = sentinel.slave_for(settings.REDIS_MASTER)
output = slave.get(key)
if output:
return pickle.loads(output)
else:
output = f(*args, **kwargs)
master.setex(key, timeout, pickle.dumps(output))
return output
else:
return f(*args, **kwargs)
class RedisInfo:
def __init__(self, sentinel_use=1):
cfg = Config()
self.sentinel_use = sentinel_use
self.rmq = {}
self.grp = {}
# Redis master, slave Context
#self.redis_server = redis.StrictRedis(host=str(self.host), port=int(self.port), db=int(self.dbnm)) #, socket_timeout=-1)
#self.rdm_slave = redis.StrictRedis(host=str(self.host_slave), port=int(self.port_slave), db=int(self.dbnm_slave) ) #, socket_timeout=1)
#self.redis_server = self.redis_server
#############################################################################
# Redis-Sentinel]
# istSentinel = [('172.31.5.41', 26379)]
# strServiceName = mymaster
# strRedisPass =
# nDB = 0
##############################################################################
if self.sentinel_use == 1:
self.redis_sentinel = eval(cfg.get('Redis-Sentinel', 'istSentinel'))
self.redis_servicename = cfg.get('Redis-Sentinel', 'strServiceName')
self.redis_redispass = cfg.get('Redis-Sentinel', 'strRedisPass')
self.redis_ndb = int(cfg.get('Redis-Sentinel', 'nDB'))
self.sentinel = Sentinel(self.redis_sentinel) #socket_timeout=0.1)
#master = self.sentinel.discover_master(self.redis_servicename)
#slave = self.sentinel.discover_slaves(self.redis_servicename)
###############################################################################
self.redis_list = eval(cfg.get('Redis-List', 'list'))
self.redis_list_len = len(self.redis_list)
self.rdm_server = {}
self.redis_server = None
self.redis_listup()
self.redis_current()
# Redis List Context
def redis_listup(self):
for x in range(self.redis_list_len):
self.rdm_server[x] = redis.StrictRedis( host=self.redis_list[x]['hostname'], port=self.redis_list[x]['port'], db=self.redis_list[x]['db'] )
# Current Live Redis Check
# 2015-09-25 add
# => Redis Sentinel Master Context
def redis_current(self):
if self.sentinel_use == 1:
master = self.sentinel.master_for(self.redis_servicename, password=self.redis_redispass, db=self.redis_ndb) #socket_timeout=0.1)
slave = self.sentinel.slave_for(self.redis_servicename, password=self.redis_redispass, db=self.redis_ndb) #socket_timeout=0.1)
self.redis_server = master
return self.redis_server
else:
for x in self.rdm_server:
try:
if self.rdm_server[x].ping():
self.redis_server = self.rdm_server[x]
break
except:
pass
try:
return self.redis_server
except:
self.redis_listup()
self.redis_current()
time.sleep(1)
# Redis Real Time Check
def redis_ping(self):
l = {}
for x in self.rdm_server:
try:
c = self.rdm_server[x].ping()
except:
c = False
l[x] = {'hostname':self.redis_list[x]['hostname'], 'status':c}
try:
return l
except:
self.redis_listup()
self.redis_ping()
# Quant Default Meta Setting
def set(self):
#self.rdm = redis.StrictRedis(host=str(self.host), port=int(self.port), db=int(self.dbnm), socket_timeout=1)
try:
d = {
0:{'hostname':'172.31.21.219', 'port':5672, 'user':'******', 'pass':'******', 'vhost':'dev-quant', 'exchange_name':'quant'},
1:{'hostname':'172.31.16.52' , 'port':5672, 'user':'******', 'pass':'******', 'vhost':'dev-quant', 'exchange_name':'quant'}
}
self.redis_server.set('rabbitmq', d)
d = {'state':{'AA':'XKRX-CS-AA', 'KR':'XKRX-CS-KR'}}
self.redis_server.set('group', d)
d = {'172.31.7.219':'data', '172.31.7.217':'state', '172.31.7.218':'class', '172.31.7.216':'web'}
self.redis_server.set('service', d)
except Exception, e:
print e
MasterNotFoundError, SlaveNotFoundError)
# Redis 접속 기본 설정값
listSentinel = [('172.31.5.41', 26379)]
strServiceName = 'mymaster'
strRedisPass = ''
nDB = 0
nMaxUser = 1000
sentinel = Sentinel(listSentinel, socket_timeout=0.1)
try:
#sentinel.discover_master(strServiceName) # No need for this
#sentinel.discover_slaves(strServiceName)
master = sentinel.master_for(strServiceName, password=strRedisPass, db=nDB, socket_timeout=0.1)
slave = sentinel.slave_for(strServiceName, password=strRedisPass, db=nDB, socket_timeout=0.1)
print master
"""
m = sentinel.discover_master(strServiceName)
s = sentinel.discover_slaves(strServiceName)
"""
except MasterNotFoundError:
print 'Master not found or Sentinel instances not runnung'
sys.exit()
except SlaveNotFoundError:
print 'Slave not found or Sentinel instances not runnung'
sys.exit()
except ConnectionError:
print 'Connection Error. Check if Sentinel instances are running'
sys.exit()
#from Redis import RedisInfo
#r = RedisInfo()
from redis import Redis, exceptions, RedisError
from redis.sentinel import (Sentinel, SentinelConnectionPool,ConnectionError, MasterNotFoundError, SlaveNotFoundError)
# Redis 접속 기본 설정값
listSentinel = [('172.31.5.41', 26379)]
strServiceName = 'mymaster'
strRedisPass = ''
nDB = 0
sentinel = Sentinel(listSentinel)
master = sentinel.master_for(strServiceName, password=strRedisPass, db=nDB)
slave = sentinel.slave_for(strServiceName, password=strRedisPass, db=nDB)
rds = master
rds_pub = rds.pubsub()
rds_pub.subscribe('0_Text_0')
"""
print sys.argv[1]
"""
while True:
try:
for x in rds_pub.listen():
print "Text :", x['data']
except Exception, e:
print "Exception :", e
# Redis Sentinel
from redis.sentinel import Sentinel
sentinel = Sentinel([('localhost', 26379)], socket_timeout=0.3, db=15, decode_responses=True)
master = sentinel.master_for('madeira')
slave = sentinel.slave_for('madeira')
# redis 增删改查
master.set('foo', 'bar')
slave.get('foo')
master.delete('foo')
login_manager.login_view = 'account.signin'
login_manager.login_message = u"Please sign in to access this page."
# Configure theme
try: # pragma: no cover
# First with local settings
import settings_local
theme = settings_local.THEME
except:
# Otherwise try with default theme
theme = settings.THEME
# Create app
app = create_app(theme=theme)
sentinel = Sentinel(app.config['REDIS_SENTINEL'], socket_timeout=0.1)
redis_master = sentinel.master_for('mymaster')
redis_slave = sentinel.slave_for('mymaster')
#toolbar = DebugToolbarExtension(app)
db = SQLAlchemy(app)
mail = Mail(app)
signer = URLSafeTimedSerializer(app.config['ITSDANGEORUSKEY'])
if app.config.get('SENTRY_DSN'): # pragma: no cover
sentr = Sentry(app)
babel = Babel(app)
@babel.localeselector
def get_locale():
if current_user.is_authenticated():
lang = current_user.locale
from bottle import route, run, static_file
from math import ceil
from redis.sentinel import Sentinel
#Slave/Master distribution
sentinel = Sentinel('TODO: Get some Redis instances in Amazons AWS')
master = sentinel.master_for('.')
slave = sentinel.slave_for('.')
@route('/')
def home():
return static_file('index.html', '..')
@route('/get/stats')
def getStats():
return generate_json()
@route('/<filename>')
def server_static(filename):
return static_file(filename, root='..')
def generate_json():
# Pipeline all the requests to the SLAVE (All read)
pipe = slave.pipeline()
#Pipeline returns all responses in a list
import time
from redis.sentinel import Sentinel
sentinel = Sentinel([('192.168.11.35', 26379),('192.168.11.37', 26379),('192.168.11.37', 26379)], socket_timeout=0.1)
for i in range(1,1000):
print "ciclo " + str(i)
try:
print "master: " + str(sentinel.discover_master('mymaster'))
print "slave/s: " + str(sentinel.discover_slaves('mymaster'))
master = sentinel.master_for('mymaster', socket_timeout=0.1)
slave = sentinel.slave_for('mymaster', socket_timeout=0.1)
except:
print "sentinel eligiendo a un master"
clave = "clave" + str(i)
valor = "valor" + str(i)
try:
print "master.set " + clave + " " + valor
master.set(clave, valor)
print "master.get " + clave + " = " + str(master.get(clave))
print "master.set " + clave + " = " + str(slave.get(clave))
print "master.delete " + clave
master.delete(clave)
except:
print "cluster en modo solo lectura"
from mickey.commonconf import SINGLE_MODE, REDIS_PASSWORD
if SINGLE_MODE:
import redis
if REDIS_PASSWORD:
_sentinel = redis.StrictRedis(host='localhost', port=6379, db=0, password=REDIS_PASSWORD, socket_timeout=5.0)
else:
_sentinel = redis.StrictRedis(host='localhost', port=6379, db=0, socket_timeout=5.0)
_sentinel_salve = _sentinel
_sentinel_master = _sentinel
else:
from redis.sentinel import Sentinel
_sentinel = Sentinel([('localhost', 26379)], socket_timeout = 1)
_sentinel_salve = _sentinel.slave_for('master', socket_timeout = 0.5)
_sentinel_master = _sentinel.master_for('master', socket_timeout = 0.5)
def write_to_redis(key, value, expire = None):
if not key or not value:
return
try:
_sentinel_master.set(key, value)
if expire:
_sentinel_master.expire(key, expire)
except Exception as e:
logging.error("save information to redis failed {0}".format(e))
def read_from_redis(key, defaultvalue = None):
class BaseCache(object):
'''
This class uses redis sentinel to establish a connection with redis.
It ensures that all write operations goes to master instance and all read operations
served by slave instances.
'''
# To support the *args, passed from ReConnectingCache
def __init__(self, host='localhost', port=6379, db=0, password=None, socket_timeout=None):
self.sentinel = None
self.master = None
self.selected_db = db
self.slave = None
self.socket_timeout = socket_timeout
self.logger = logging.getLogger("django_sentinel")
self.initialize()
def initialize(self):
try:
self.sentinel = Sentinel(settings.LIST_OF_SENTINEL, self.socket_timeout)
self.discover_master()
self.discover_slave()
except:
self.logger.exception("Exception !! %s", traceback.format_exc())
raise
self.logger.info("RedisHA initialized, db is %d", self.selected_db)
def get_master(self):
return self.master
def get_slave(self):
return self.slave
def discover_instance(func):
'''
It discover the new master/slave, if the current master or slave is down.
'''
@wraps(func)
def inner(self, *args, **kwargs):
try:
res = func(self, *args, **kwargs)
except SlaveNotFoundError:
self.logger.exception("Exception : SlaveNotFoundError")
self.discover_slave()
res = func(self, *args, **kwargs)
except MasterNotFoundError:
self.logger.exception("Exception : MasterNotFoundError")
self.discover_master()
res = func(self, *args, **kwargs)
except:
self.logger.exception("Exception !! %s", traceback.format_exc())
raise
return res
return inner
@discover_instance
def get(self, key):
return self.slave.get(key)
@discover_instance
def set(self, key, value, expiry=7776000):
return self.master.set(key, value, expiry)
@discover_instance
def delete(self, key):
return self.master.delete(key)
@discover_instance
def keys(self):
return self.slave.keys()
@discover_instance
def lpop(self, name):
return self.master.lpop(name)
@discover_instance
def rpop(self, name):
return self.master.rpop(name)
@discover_instance
def lpush(self, name, *values):
return self.master.lpush(name, *values)
@discover_instance
def rpush(self, name, *values):
return self.master.rpush(name, *values)
@discover_instance
def expire(self, name, time):
return self.master.expire(name, time)
@discover_instance
def incr(self, name, amount=1):
return self.master.incr(name, amount)
@discover_instance
def sadd(self, name, *values):
return self.master.sadd(name, *values)
@discover_instance
def srem(self, name, *values):
return self.master.srem(name, *values)
@discover_instance
def scard(self, name):
return self.slave.scard(name)
@discover_instance
def smembers(self, name):
return self.slave.smembers(name)
@discover_instance
def zadd(self, name, *args, **kwargs):
return self.master.zadd(name, *args, **kwargs)
@discover_instance
def zrem(self, name, *values):
return self.master.zrem(name, *values)
@discover_instance
def hset(self, name, key, value):
return self.master.hset(name, key, value)
@discover_instance
def hget(self, name, key):
return self.slave.hget(name, key)
@discover_instance
def hdel(self, name, *keys):
return self.master.hdel(name, *keys)
@discover_instance
def hgetall(self, name):
return self.slave.hgetall(name)
@discover_instance
def hmget(self, name, keys, *args):
return self.slave.hmget(name, keys, *args)
@discover_instance
def flushdb(self):
return self.master.flushdb()
@discover_instance
def dbsize(self):
return self.slave.dbsize()
@discover_instance
def zrangebyscore(self, name, min, max, start=None, num=None, withscores=False, score_cast_func=float):
return self.slave.zrangebyscore(name, min, max, start, num, withscores, score_cast_func)
def discover_master(self):
try:
self.master = self.sentinel.master_for(settings.SENTINEL_MASTER, db=self.selected_db, socket_timeout=None)
except:
self.logger.exception("Exception !! %s", traceback.format_exc())
raise
def discover_slave(self):
try:
self.slave = self.sentinel.slave_for(settings.SENTINEL_MASTER, db=self.selected_db, socket_timeout=None)
except:
self.logger.exception("Exception !! %s", traceback.format_exc())
raise
def __getattr__(self, name):
return getattr(self.get_slave(), name)