class RedisDriver: def __init__(self, config): self.config = config sentinels = [(c['ip'], c['port']) for c in self.config['sentinels']] self.sentinel = Sentinel(sentinels, socket_timeout=0.1, db=self.config['db'], decode_responses=True) self.__master = None self.__slave = None @property def master(self): if False: if not self.__master: self.__master = self.sentinel.master_for( self.config['name'], password=self.config.get('auth', '')) return self.__master else: return self.sentinel.master_for(self.config['name'], password=self.config.get( 'auth', '')) @property def slave(self): if False: if not self.__slave: self.__slave = self.sentinel.slave_for( self.config['name'], password=self.config.get('auth', '')) return self.__slave else: return self.sentinel.slave_for(self.config['name'], password=self.config.get( 'auth', ''))
class redisSentinelHelper(): def __init__(self,sentinel_list,service_name,db): self.sentinel = Sentinel(sentinel_list,socket_timeout=0.5) self.service_name = service_name self.db = db def get_master_redis(self): return self.sentinel.discover_master(self.service_name) def get_slave_redis(self): return self.sentinel.discover_slaves(self.service_name) def set_key(self,key,value): master = self.sentinel.master_for( service_name=self.service_name, socket_timeout=0.5, db=self.db ) return master.set(key,value) def set_hash_key(self,hkey,key,value): master = self.sentinel.master_for( service_name=self.service_name, socket_timeout=0.5, db=self.db ) return master.hset(hkey,key,value) def get_key(self,key): slave = self.sentinel.slave_for( service_name=self.service_name, socket_timeout=0.5, db=self.db ) return slave.get(key) def get_hash_key(self,hkey,hey): slave = self.sentinel.slave_for( service_name=self.service_name, socket_timeout=0.5, db=self.db ) return slave.hget(hkey,hey) def delete(self,key): master = self.sentinel.master_for( service_name=self.service_name, socket_timeout=0.5, db=self.db ) return master.delete(key)
def run(name): sentinel = Sentinel([ ('10.5.5.194', 26379), ('10.5.5.155', 26379), ('10.5.5.128', 26379), ], socket_timeout=0.5) master = sentinel.master_for('mymaster', socket_timeout=0.5, password='******', db=0) slave = sentinel.slave_for('mymaster', socket_timeout=0.5, password='******', db=0) # 判断是否所传参数对应的value值是否为1 while True: time.sleep(0.05) if slave.get(name) == "0" or slave.get(name) is None: continue else: break master.set(name, "0") f_do = sys.argv[1] + '.do' f_dat = sys.argv[1] + '.dat' f = open(f_dat, 'w') f.write('0' + '\n') f.write('success' + '\n') f.write('set ' + name + '为0 \n') f.close() f = open(f_do, 'w') f.write('') f.close()
def _init(self, server, params): super(CacheClass, self).__init__(params) self._server = server self._params = params self.service_name = self.options.get("SERVICE_NAME", 'mymaster') self.slave_read = self.options.get("READ_SLAVE", False) if self.server: def _get_hosttupe(conn): host_lst = conn.rsplit(':', 1) return host_lst[0], int(host_lst[1]) conn_list = [_get_hosttupe(conn) for conn in self.server] else: raise ImproperlyConfigured("sentinel server config error.") kwargs = self.connection_pool_class_kwargs max_connections = kwargs.pop('max_connections') sentinel_manager = Sentinel(conn_list, sentinel_kwargs=kwargs) kwargs.update({ 'db': self.db, 'password': self.password, 'max_connections': max_connections }) self._client = sentinel_manager.master_for(self.service_name, connection_pool_class=self.connection_pool_class, **kwargs) if self.slave_read: self._slave_client = sentinel_manager.slave_for(self.service_name, connection_pool_class=self.connection_pool_class, **kwargs)
class TestApplication(Application): def __init__(self): # Call to the superclass to bootstrap. super(TestApplication, self).__init__(name='krux-redis-sentinel') self.logger.debug('Parsing sentinels from args: %r', self.args.sentinel) sentinels = hostports_from_args(self.args) self.logger.debug('Parsed sentinel host, port pairs: %r', sentinels) self.logger.debug('Initializing Sentinel instance...') self.sentinel = Sentinel(sentinels, socket_timeout=SOCKET_TIMEOUT) self.logger.debug('Initialized Sentinel instance: %r', self.sentinel) def add_cli_arguments(self, parser): add_sentinel_cli_arguments(parser) def run(self): master = self.sentinel.master_for(self.name) slave = self.sentinel.slave_for(self.name) try: self.logger.info('Ping master: %s', master.ping()) except ConnectionError as err: self.logger.warning('Could not connect to master!') self.logger.error(err) try: self.logger.info('Ping slave: %s', slave.ping()) except ConnectionError as err: self.logger.warning('Could not connect to slave!') self.logger.error(err)
def create_app(): '''工厂方法''' # 创建app对象 app = Flask(__name__) # 加载配置 config = config_dict.get(app.config.get('ENV')) app.config.from_object(config) # 加载数据库配置 from common.models import db db.init_app(app) # 记录日志信息 log_file(config.LOG_LV) # redis集群加载 from rediscluster import RedisCluster app.redis_cluster = RedisCluster( startup_nodes=app.config.get('REDIS_CLUSTER')) # redis哨兵,主从配置 from redis.sentinel import Sentinel sentinel = Sentinel(app.config.get('REDIS_SENTINELS')) app.redis_master = sentinel.master_for( app.config.get('REDIS_SENTINEL_SERVICES_NAME')) app.redis_slave = sentinel.slave_for( app.config.get('REDIS_SENTINEL_SERVICES_NAME')) # 注册用户蓝图到app中 from .resources.users import user_blue app.register_blueprint(user_blue) return app
def get_redis_connection(config, is_master=True): """ Returns redis connection basing on configuration dict Args: config: Dict with configuration, if key `SENTINELS` is present sentinel connection pool will be used. is_master: Whether connection should be master or slave, applies only if sentinels are used Returns: StrictRedis connection """ if 'SENTINELS' in config: sentinel = Sentinel( config['SENTINELS'], db=config.get('DB'), password=config.get('PASSWORD'), socket_timeout=config.get('TIMEOUT'), socket_connect_timeout=config.get('CONNECT_TIMEOUT'), ) if is_master: return sentinel.master_for(settings.REDIS_CLUSTER_NAME) else: return sentinel.slave_for(settings.REDIS_CLUSTER_NAME) else: return StrictRedis( host=config['HOST'], port=config['PORT'], db=config.get('DB'), password=config.get('PASSWORD'), socket_timeout=config.get('TIMEOUT'), socket_connect_timeout=config.get('CONNECT_TIMEOUT'), )
def init_redis_client(cls): # 防止多线程竞争,获取到锁后依然可能已经初始化了 if cls.redis_objs: return cls.instance_name = conf.get('sentinel', "") cls.socket_timeout = conf.get("socket_timeout", 5) cls.connect_timeout = conf.get("connect_timeout", 0.1) if not cls.instance_name: # 单例模式 host = conf["host"] cls.redis_objs[cls.MODE_READ] = Redis( host=host.split(':')[0], port=host.split(':')[1], socket_timeout=cls.socket_timeout, socket_connect_timeout=cls.connect_timeout, retry_on_timeout=1 ) cls.redis_objs[cls.MODE_WRITE] = cls.redis_objs[cls.MODE_READ] else: # 哨兵模式 sentinel = Sentinel( cls.parse_config(), socket_timeout=cls.socket_timeout, socket_connect_timeout=cls.connect_timeout, retry_on_timeout=1 ) cls.redis_objs[cls.MODE_READ] = sentinel.slave_for(cls.instance_name) cls.redis_objs[cls.MODE_WRITE] = sentinel.master_for(cls.instance_name)
def get_connection(self, is_read_only=False) -> redis.StrictRedis: """ Gets a StrictRedis connection for normal redis or for redis sentinel based upon redis mode in configuration. :type is_read_only: bool :param is_read_only: In case of redis sentinel, it returns connection to slave :return: Returns a StrictRedis connection """ if self.connection is not None: return self.connection if self.is_sentinel: kwargs = dict() if self.password: kwargs["password"] = self.password sentinel = Sentinel([(self.host, self.port)], **kwargs) if is_read_only: connection = sentinel.slave_for(self.sentinel_service, decode_responses=True) else: connection = sentinel.master_for(self.sentinel_service, decode_responses=True) else: connection = redis.StrictRedis(host=self.host, port=self.port, decode_responses=True, password=self.password) self.connection = connection return connection
def _connection(self): sentinel = Sentinel(self._sentinels, socket_timeout=self._socket_timeout) slave = sentinel.slave_for(self._sentinel_master, socket_timeout=self._socket_timeout, db=self._db) return slave
def test_simple_sentinel(self): # Setup Master & Slave master = RedisDeployment('~/redis-master', port=3428) salve = RedisDeployment('~/redis-slave', port=3429) master.start() salve.start(master_ip='127.0.0.1', master_port=3428) # Setup Sentinel redis_sentinel = RedisSentinel('~/redis-sentinel') redis_sentinel.start(master_ip='127.0.0.1',master_port=3428, master_name='mymaster') sleep(3) ## len(RedisSentinel.list_running_instances()) == 1 ## len(RedisDeployment.list_running_instances()) == 2 # Client API sentinel = Sentinel([('localhost', redis_sentinel.port)], socket_timeout=0.1) master_client = sentinel.master_for('mymaster') slave_client = sentinel.slave_for('mymaster') master_client.set('foo', 'bar') slave_client.get('foo') # bar self.assertTrue(len(RedisSentinel.list_running_instances()) == 1) self.assertTrue(len(RedisDeployment.list_running_instances()) == 2) self.assertTrue(str(slave_client.get('foo').decode('utf-8')) == 'bar') # Close down the servers master.stop() salve.stop() redis_sentinel.stop()
def __init__(self): """ init setting and create redis instance """ settings = Settings.get("redis") self.prefix = settings.get("prefix") self.queue_max_len = settings.get("queue_max_len") self.sentinel = settings.get("sentinel") if self.sentinel: sentinel = Sentinel(sentinels=map(lambda x: x.split(":"), settings.get("sentinel_hosts"))) kwargs = dict( service_name=settings.get("sentinel_master"), password=settings.get("password"), decode_responses=True, ) self.master = sentinel.master_for(**kwargs) self.slave = sentinel.slave_for(**kwargs) else: pool = redis.ConnectionPool( host=settings.get("host"), port=settings.get("port"), db=settings.get("db"), password=settings.get("password"), decode_responses=True, ) self.master = self.slave = redis.StrictRedis(connection_pool=pool)
def connect2redis(host, port=6379, db=0, connectionPool=True, logging=None, sentinel_service=prms.REDIS_SENTINEL_SERVICE_NAME, master=True): if sentinel_service: retry_count = 0 while retry_count < 10: sentinel = Sentinel([(host, 26379)]) if master: client = sentinel.master_for(sentinel_service) else: client = sentinel.slave_for(sentinel_service) if is_available(client, logging=logging): logging.info("Connected to Redis succesfully.") return client else: print(f"Waiting for {pow(2, retry_count)} secs.") time.sleep(pow(2, retry_count)) retry_count += 1 else: retry_count = 0 while retry_count < 10: if connectionPool: pool = redis.ConnectionPool(host=host, port=port, db=db) client = redis.StrictRedis(connection_pool=pool) else: client = redis.StrictRedis(host=host, port=port, db=db) if is_available(client, logging=logging): logging.info("Connected to Redis succesfully.") return client else: print(f"Waiting for {pow(2, retry_count)} secs.") time.sleep(pow(2, retry_count)) retry_count += 1 logging.error("Exiting...") exit(3)
class SentinelProxy: """Proxy for Redis sentinel.""" def __init__(self, sentinel_host, master_name, socket_timeout=0.1, **kwargs): """Initialize Redis sentinel connection. :params: sentinel_host: (host, port) """ self.sentinel = Sentinel([sentinel_host], socket_timeout=socket_timeout) self.master = self.sentinel.master_for(master_name, socket_timeout=socket_timeout, **kwargs) self.slave = self.sentinel.slave_for(master_name, socket_timeout=socket_timeout, **kwargs) def __getattr__(self, name): """Get attribute from Redis master or slave.""" master_key = ('set', 'hset', 'hmset', 'lset', 'lpush', 'blpop', 'brpop', 'rpush', 'expire', 'delete', 'incr') if name not in master_key: target = self.slave else: target = self.master return getattr(target, name)
def get_connection(self, is_read_only=False) -> redis.Redis: """ Gets a Redis connection for normal redis or for redis sentinel based upon redis mode in configuration. :type is_read_only: bool :param is_read_only: In case of redis sentinel, it returns connection to slave :return: Returns a Redis connection """ if self.connection is not None: return self.connection if self.is_sentinel: kwargs = dict() if self.password: kwargs["password"] = self.password sentinel = Sentinel([(self.host, self.port)], **kwargs) if is_read_only: connection = sentinel.slave_for(self.sentinel_service, decode_responses=True) else: connection = sentinel.master_for(self.sentinel_service, decode_responses=True) else: connection = redis.Redis(host=self.host, port=self.port, decode_responses=True, password=self.password) self.connection = connection return connection
def redis_conn(readonly=True): global sentinel, master_client, slave_client if not sentinel: sentinel = Sentinel(settings.REDIS_SENTINEL, socket_timeout=1) if not master_client: master_client = sentinel.master_for(settings.REDIS_CLUSTER) if not slave_client: slave_client = sentinel.slave_for(settings.REDIS_CLUSTER) return slave_client if readonly else master_client
def get_database(config, master=True): if config['sentinel']: sentinal = Sentinel(config['sentinel'], socket_timeout=0.1, password=config['redis_password'], db=config['redis_database']) if master: return sentinal.master_for(config['sentinel_cluster_name']) else: return sentinal.slave_for(config['sentinel_cluster_name']) else: return Redis.from_url(config['redis'])
def __conn_redis(self): sentinel = Sentinel( self.REDIS_CONFIG['SENTINELS'], socket_timeout=0.1, decode_responses=self.REDIS_CONFIG['DECODE_RESPONSES'], **self.conn_kwargs) self.redis_master = sentinel.master_for( self.REDIS_CONFIG['SERVICE_NAME'], socket_timeout=0.1) self.redis_slave = sentinel.slave_for( self.REDIS_CONFIG['SERVICE_NAME'], socket_timeout=0.1)
def get_redis_conn(readonly=False): global sentinel, _cached_master_client, _cached_slave_client if not sentinel: sentinel = Sentinel(settings.REDIS_SENTINEL, socket_timeout=0.1) if not _cached_master_client: _cached_master_client = sentinel.master_for( settings.REDIS_SNS_CLUSTER_NAME) if not _cached_slave_client: _cached_slave_client = sentinel.slave_for( settings.REDIS_SNS_CLUSTER_NAME) return _cached_slave_client if readonly else _cached_master_client
def create_sentinel_redis(config): logger.info('start to connect sentinel redis: host-port:%s, db: %s' % (config['host-pot'], config['db'])) host_port = parse_host(config["host-port"]) sentinel = Sentinel(host_port, socket_timeout=2, password=config['pwd'], db=config['db']) master = sentinel.master_for('mymaster', socket_timeout=5) slave = sentinel.slave_for('mymaster', socket_timeout=2) return master, slave
def main(): rd = redis.Redis(host='localhost', port=6379, db=0, password=None, socket_timeout=None) #rd = redis.StrictRedis( host = 'localhost', port = 6379, db = 0, password = None, socket_timeout = None ) key = "test1" value = "test1_val" rd.set(key, value) result = rd.get(key) print("result =\n") print(f'{ key } = { result }\n') # mset data = {"test2": "test2_val", "test3": "test3_val", "test4": "test4_val"} rd.mset(data) result = rd.get("test2") print("result =\n") print(f'{ key } = { result }\n') # value: JSON data key = "test1_sub" value = { "test1_sub1": "test1_sub1_value", "test1_sub1": "test1_sub1_value", "test1_sub1": "test1_sub1_value" } value = json.dumps(value, ensure_ascii=False).encode("utf-8") rd.set(key, value) result = rd.get(key) result = result.decode("utf-8") result = dict(json.loads(result)) print("result =\n") print(f'{ key } = { result }\n') rd.close() # Sentinel sentinel = Sentinel([('localhost', 26379)], socket_timeout=0.1) master = sentinel.master_for('my_redis_master', socket_timeout=0.1) key = "master_test1" value = "master_test1_val" master.set(key, value) result = master.get(key) print("result (master) =\n") print(f'{ key } = { result }\n') slave = sentinel.slave_for('my_redis_master', socket_timeout=0.1) result = slave.get(key) print("result (slave) =\n") print(f'{ key } = { result }\n') sentinel.close()
def get_redis_conn(): global MYSETINEL global MASTER global SLAVE # 如果哨兵连接实例已存在, 不重复连接, 当连接失效时, 重新连接 if not MYSETINEL: # 连接哨兵 MYSETINEL = Sentinel(SENTINEADDRESS, socket_timeout=2000) # 尝试连接最长时间单位毫秒, 1000毫秒为1秒 # 通过哨兵获取主数据库连接实例 参数1: 主数据库的名字(集群部署时在配置文件里指明) MASTER = MYSETINEL.master_for('mymaster', socket_timeout=2000) # 通过哨兵获取从数据库连接实例 参数1: 从数据的名字(集群部署时在配置文件里指明) SLAVE = MYSETINEL.slave_for('mymaster', socket_timeout=2000)
def create_app(Config, enable_config_file=False): """ 创建应用的工厂函数 :param config: 配置信息类 :param enable_config_file: 是否允许环境变量中的配置文件覆盖已加载的配置信息 :return: 应用对象 """ app = create_flask_app(Config, enable_config_file) """项目运行需要的配置""" # 配置日志 from utils.logging import create_logger create_logger(app) # 配置redis哨兵 from redis.sentinel import Sentinel _sentinel = Sentinel(app.config.get('REDIS_SENTINELS')) # 根据哨兵设置主、从服务 app.redis_master = _sentinel.master_for( app.config['REDIS_SENTINEL_SERVICE_NAME']) app.redis_slave = _sentinel.slave_for( app.config['REDIS_SENTINEL_SERVICE_NAME']) # Redis集群 from rediscluster import StrictRedisCluster app.redis_cluster = StrictRedisCluster( startup_nodes=app.config['REDIS_CLUSTER']) # MySQL数据库连接初始化 from models import db db.init_app(app) # 创建Snowflake ID worker--雪花算法 from utils.snowflake.id_worker import IdWorker app.id_worker = IdWorker(app.config['DATACENTER_ID'], app.config['WORKER_ID'], app.config['SEQUENCE']) """添加请求钩子""" from utils.middlewares import jwt_authentication app.before_request(jwt_authentication) """注册蓝图""" # 用户蓝图 from toutiao.resources.user import user_bp app.register_blueprint(user_bp) """定时任务,每天3点更正我们redis和mysql的数据""" # 初始化调度器,并配置最大开始10个线程(不指定,默认10个) bg_scheduler = BackgroundScheduler( executor={'default': ThreadPoolExecutor()}) # 添加任务函数 # bg_scheduler.add_job('任务函数', '执行器', '执行周期时间') # bg_scheduler.add_job('任务函数', 'cron', hours=3) # 每天3点执行 from toutiao.aps_scheduler.statistic_data import fix_process bg_scheduler.add_job(fix_process, 'date', args=[app]) # 为了测试让他立即执行 # 执行任务 bg_scheduler.start() return app
def save_to_redis(self): ''' Save JSON dump into Redis storage as covid_data_ru and covid_data_en objects ''' status = False data = {} for language in ('ru', 'en'): data[language] = self.create_geojson(language) try: logger.info('Connecting to Redis Sentinel') sentinel_client = Sentinel( [('redis-sentinel', 26379)], socket_timeout=0.1) logger.info('Discovering Redis Master') sentinel_client.discover_master(REDIS_MASTER) master = sentinel_client.master_for( REDIS_MASTER, socket_timeout=0.1) logger.info('Discovering Redis Slave') slave = sentinel_client.slave_for(REDIS_MASTER, socket_timeout=0.1) for language in ('ru', 'en'): logger.info(f'Saving {language.upper()} covid_data into Slave') status = slave.set('covid_data_' + language, json.dumps(data[language], ensure_ascii=False)) if not status: return False return status except: logger.warning( 'Can not proceed with Redis. Saving JSON as a file.') try: for language in ('ru', 'en'): with open(language + '.json', 'w') as outfile: logger.info(f'Saving {language.upper()} as {language}.json') json.dump(data[language], outfile, ensure_ascii=False) except: logger.warning('Can not save data as a file.') return False
def wrapper(*args, **kwargs): if os.environ.get('PYBOSSA_REDIS_CACHE_DISABLED') is None: key = "%s::%s" % (settings.REDIS_KEYPREFIX, key_prefix) sentinel = Sentinel(settings.REDIS_SENTINEL, socket_timeout=0.1) master = sentinel.master_for('mymaster') slave = sentinel.slave_for('mymaster') output = slave.get(key) if output: return pickle.loads(output) else: output = f(*args, **kwargs) master.setex(key, timeout, pickle.dumps(output)) return output else: return f(*args, **kwargs)
def setUpClass(cls): cls.env_num = cls.a.get_num_name("环境") cls.apiName_num = cls.a.get_num_name("接口名称") cls.method_num = cls.a.get_num_name("请求方法") cls.headers_num = cls.a.get_num_name("请求头") cls.para_num = cls.a.get_num_name("请求参数") cls.desc_num = cls.a.get_num_name("用例描述") cls.data_num = cls.a.get_num_name("请求体") cls.expect_num = cls.a.get_num_name("预期结果") cls.isSkip_num = cls.a.get_num_name("是否跳过该用例") cls.relateData_num = cls.a.get_num_name("接口关联参数") # 线上楼盘ID sss["Restful_estateId2"] = json.dumps({"estateId": "1041394"}) sss["Restful_estateId3"] = json.dumps({"estateId": "22374"}) # 查看线上签约信息 sss["Restful_signId"] = json.dumps({ "estateId": "22374", "signId": "116" }) # 查看测试环境签约信息 sss["Restful_signId_test"] = json.dumps({ "estateId": "889", "signId": "66" }) # # SENTINEADDRESS = [('10.50.255.117', 26380), ('10.50.255.118', 26380), ('10.50.255.119', 26380)] # 尝试连接最长时间单位毫秒, 1000毫秒为1秒 MYSETINEL = Sentinel(SENTINEADDRESS, socket_timeout=3000) # 通过哨兵获取从数据库连接实例,用于数据读取 SLAVE = MYSETINEL.slave_for('redis.bp.fdd', socket_timeout=3000) # 通过哨兵获取主数据库连接实例,用于数据写入 MASTER = MYSETINEL.master_for('redis.bp.fdd', socket_timeout=3000) # 读取redis中的签约时间戳 sign_end_time = SLAVE.get('market_dev.ddxf.bp.fdd:sign_end_time') if sign_end_time is None: # 初始化一个时间戳 MASTER.setex('market_dev.ddxf.bp.fdd:sign_end_time', '86400', 1582819199999) sss["sign_end_time"] = 1582819199999 else: sss["sign_end_time"] = int(sign_end_time) new_end_time = int(sign_end_time) + 86400000 # 将新的时间戳写入redis中,有效期1天 MASTER.setex('market_dev.ddxf.bp.fdd:sign_end_time', '86400', new_end_time)
def get_client(self, mode, timeout=None): # def _curr_session_time_out(): # return timeout if timeout is not None else self.sock_timeout if timeout is not None: # 如果指定了超时,就临时生成一个符合这个超时的连接,使用短连接,这个场景主要应对DI轮训服务,不能频繁超时 if not self.__class__.instance_name: host = conf["host"] return Redis( host=host.split(':')[0], port=host.split(':')[1], socket_timeout=self.socket_timeout+2, socket_connect_timeout=self.connect_timeout, retry_on_timeout=1, socket_keepalive=False ) curr_sentinel = Sentinel( self.parse_config(), socket_timeout=self.socket_timeout, socket_connect_timeout=self.connect_timeout, ) if mode == self.MODE_WRITE: return curr_sentinel.master_for( self.instance_name, socket_timeout=timeout+2, socket_connect_timeout=self.connect_timeout, socket_keepalive=False, # 短连接 retry_on_timeout=True ) else: return curr_sentinel.slave_for( self.instance_name, socket_timeout=timeout+2, socket_connect_timeout=self.connect_timeout, socket_keepalive=False, retry_on_timeout=True ) else: return self.__class__.redis_objs[mode]
class RedisStorage(Storage): def __init__(self, env: dict): self._redis_service_name = env.get("YOCOTTO_URL_REDIS_SERVICE_NAME", "mymaster") self._redis_socket_timeout = float( env.get("YOCOTTO_URL_REDIS_SOCKET_TIMEOUT", 0.1)) self._redis_sentinel = Sentinel( json.loads( env.get("YOCOTTO_URL_REDIS_SENTINELS", '[["redis-sentinel", 26379]]'))) self._redis_master = self._redis_sentinel.master_for( self._redis_service_name, socket_timeout=self._redis_socket_timeout) self._redis_slave = self._redis_sentinel.slave_for( self._redis_service_name, socket_timeout=self._redis_socket_timeout) def get(self, url_id: str) -> ShortenedUrl: return ShortenedUrl(**json.loads(self._redis_slave.get(url_id))) def put(self, shortened_url: ShortenedUrl) -> ShortenedUrl: url_id = shortened_url.url_id res = self._redis_master.setnx(url_id, json.dumps(asdict(shortened_url))) if res == 1: return shortened_url else: raise UrlAlreadyExists(f"The url_id: {url_id} already taken.") def save(self, long_url: str, retry_count=5) -> ShortenedUrl: if retry_count == 0: raise StorageError("Failed to save URL.") try: shortened_url = ShortenedUrl(url_id=self._next_url_id(), long_url=long_url) self.put(shortened_url) return shortened_url except UrlAlreadyExists: return self.save(long_url, retry_count=retry_count - 1) def _next_url_id(self): url_id = base_encode(self._redis_master.incr("yocotto_url_counter")) return url_id
def connect_to_redis(read_only=False): """ Get a connection to the master redis service. """ logger.debug("Connecting to redis", server=redis_host, port=redis_sentinel_port) if config.REDIS_USE_SENTINEL: sentinel_service = Sentinel([(redis_host, redis_sentinel_port)], password=redis_pass, socket_timeout=5) sentinel_name = config.REDIS_SENTINEL_NAME if read_only: logger.debug("Looking up read only redis slave using sentinel protocol") r = sentinel_service.slave_for(sentinel_name) else: logger.debug("Looking up redis master using sentinel protocol") r = sentinel_service.master_for(sentinel_name) else: r = redis.StrictRedis(host=redis_host, password=redis_pass) return r
def create_app(config, enable_config_file=False): """ 创建flask应用 并 初始化各组件 :param config: 配置类 :param enable_config_file: 是否允许运行环境中的配置文件覆盖已加载的配置信息 :return: flask应用 """ app = create_flask_app(config, enable_config_file) # 添加自定义正则转换器 from utils.converters import register_converters register_converters(app) # TODO 创建redis哨兵 from redis.sentinel import Sentinel _sentinel = Sentinel(app.config['REDIS_SENTINELS']) # 获取redis主从连接对象 app.redis_master = _sentinel.master_for( app.config['REDIS_SENTINEL_SERVICE_NAME']) app.redis_slave = _sentinel.slave_for( app.config['REDIS_SENTINEL_SERVICE_NAME']) # 配置日志 from utils.logging import create_logger create_logger(app) # 限流器 from utils.limiter import limiter as lmt lmt.init_app(app) # 注册用户模块蓝图 from .resources.user import user_bp app.register_blueprint(user_bp) # 注册新闻模块蓝图 from .resources.news import news_bp app.register_blueprint(news_bp) # 注册搜索模块蓝图 from .resources.search import search_bp app.register_blueprint(search_bp) return app
def connect_to_redis(read_only=False): """ Get a connection to the master redis service. """ logger.debug("Connecting to redis") if config.REDIS_USE_SENTINEL: sentinel = Sentinel([(redis_host, 26379)], password=redis_pass, socket_timeout=5) if read_only: logger.debug( "Looking up read only redis slave using sentinel protocol") r = sentinel.slave_for('mymaster') else: logger.debug("Looking up redis master using sentinel protocol") r = sentinel.master_for('mymaster') else: r = redis.StrictRedis(host=redis_host, password=redis_pass) return r
def __init__(self, host="redis-master", port=6379, master_port=26379, db=0): super(Persister, self).__init__() self.handler.module = "Redis Persister" self.handler.log(message="Preparing redis connection") master_node = None sentinel = None slave_nodes = [(host, port)] try: self.handler.log( message="Checking if redis instance passed is a cluster") sentinel = Sentinel([(host, master_port)], socket_timeout=0.1) master_node = sentinel.discover_master('redis') self.handler.log(message="It is a cluster. Setting master node") except MasterNotFoundError: self.handler.log( message="No cluster found; using single redis instance only") except ResponseError: self.handler.log( message="No cluster found; using single redis instance only") except Exception: raise if master_node: self.handler.log(message="Setting redis master for writes") self._redis_master = sentinel.master_for("redis", socket_timeout=0.1) self._redis_connection = sentinel.slave_for("redis", socket_timeout=0.1) else: self._redis_connection = redis.StrictRedis(host=host, port=port, db=db) self.handler.log(message="Pointing redis master to connection") self._redis_master = self._redis_connection
def index(): try: stream = StringIO() sentinel = Sentinel([('redis-sentinel', 26379)], socket_timeout=0.1) dm = sentinel.discover_master('dpf') ds = sentinel.discover_slaves('dpf') print('master:', dm, file=stream) print('slaves:', ds, file=stream) master = sentinel.master_for('dpf', socket_timeout=0.1) master.set('foo', time.asctime()) slave = sentinel.slave_for('dpf', socket_timeout=0.1) print('get from redis:', slave.get('foo'), file=stream) except MasterNotFoundError as e: print('redis master is GONE, waiting for failover...', e, file=stream) except Exception as e: print('exception happened, connection lost?', e, file=stream) return '{}\n'.format(stream.getvalue())
def connection_examples(): # Connect to a standard Redis deployment. client = redis.Redis("localhost", port=6379, decode_responses=True, max_connections=20) # Read and write through the client. client.set("foo", "bar") client.get("foo") # Connect to a Redis Sentinel deployment. sentinel = Sentinel([('localhost', 26379)], socket_timeout=0.1) master = sentinel.master_for("cache", socket_timeout=0.1) slave = sentinel.slave_for("cache", socket_timeout=0.1) # Run write operations with the master. master.set("foo", "bar") # Run read operations with the slave. slave.get("foo")
def _create_sentinel_redis(self, config: dict) -> Tuple[Redis, Redis]: """ 创建一个Redis的主从链接 :param config: :return: """ def _parse_host(val: str) -> List[Tuple[str, int]]: """ @attention: 分解host,把10.1.113.158-26379分割为("10.1.113.158",26379) """ info = re.findall(r"([\d\.]+)-(\d+)", val) return [(item[0], int(item[1])) for item in info] host_port = _parse_host(config["host-port"]) sentinel = Sentinel(host_port, socket_timeout=2, password=config['pwd'], db=config['db']) master = sentinel.master_for('mymaster', socket_timeout=5) slave = sentinel.slave_for('mymaster', socket_timeout=2) return master, slave
def wrapper(*args, **kwargs): if os.environ.get('PYBOSSA_REDIS_CACHE_DISABLED') is None: key = "%s:%s_args:" % (settings.REDIS_KEYPREFIX, f.__name__) key_to_hash = "" for i in args: key_to_hash += ":%s" % i key = key + ":" + hashlib.md5(key_to_hash).hexdigest() #key += "_kwargs" #for i in frozenset(kwargs.items()): # key += ":%s" % i sentinel = Sentinel(settings.REDIS_SENTINEL, socket_timeout=0.1) master = sentinel.master_for(settings.REDIS_MASTER) slave = sentinel.slave_for(settings.REDIS_MASTER) output = slave.get(key) if output: return pickle.loads(output) else: output = f(*args, **kwargs) master.setex(key, timeout, pickle.dumps(output)) return output else: return f(*args, **kwargs)
class RedisInfo: def __init__(self, sentinel_use=1): cfg = Config() self.sentinel_use = sentinel_use self.rmq = {} self.grp = {} # Redis master, slave Context #self.redis_server = redis.StrictRedis(host=str(self.host), port=int(self.port), db=int(self.dbnm)) #, socket_timeout=-1) #self.rdm_slave = redis.StrictRedis(host=str(self.host_slave), port=int(self.port_slave), db=int(self.dbnm_slave) ) #, socket_timeout=1) #self.redis_server = self.redis_server ############################################################################# # Redis-Sentinel] # istSentinel = [('172.31.5.41', 26379)] # strServiceName = mymaster # strRedisPass = # nDB = 0 ############################################################################## if self.sentinel_use == 1: self.redis_sentinel = eval(cfg.get('Redis-Sentinel', 'istSentinel')) self.redis_servicename = cfg.get('Redis-Sentinel', 'strServiceName') self.redis_redispass = cfg.get('Redis-Sentinel', 'strRedisPass') self.redis_ndb = int(cfg.get('Redis-Sentinel', 'nDB')) self.sentinel = Sentinel(self.redis_sentinel) #socket_timeout=0.1) #master = self.sentinel.discover_master(self.redis_servicename) #slave = self.sentinel.discover_slaves(self.redis_servicename) ############################################################################### self.redis_list = eval(cfg.get('Redis-List', 'list')) self.redis_list_len = len(self.redis_list) self.rdm_server = {} self.redis_server = None self.redis_listup() self.redis_current() # Redis List Context def redis_listup(self): for x in range(self.redis_list_len): self.rdm_server[x] = redis.StrictRedis( host=self.redis_list[x]['hostname'], port=self.redis_list[x]['port'], db=self.redis_list[x]['db'] ) # Current Live Redis Check # 2015-09-25 add # => Redis Sentinel Master Context def redis_current(self): if self.sentinel_use == 1: master = self.sentinel.master_for(self.redis_servicename, password=self.redis_redispass, db=self.redis_ndb) #socket_timeout=0.1) slave = self.sentinel.slave_for(self.redis_servicename, password=self.redis_redispass, db=self.redis_ndb) #socket_timeout=0.1) self.redis_server = master return self.redis_server else: for x in self.rdm_server: try: if self.rdm_server[x].ping(): self.redis_server = self.rdm_server[x] break except: pass try: return self.redis_server except: self.redis_listup() self.redis_current() time.sleep(1) # Redis Real Time Check def redis_ping(self): l = {} for x in self.rdm_server: try: c = self.rdm_server[x].ping() except: c = False l[x] = {'hostname':self.redis_list[x]['hostname'], 'status':c} try: return l except: self.redis_listup() self.redis_ping() # Quant Default Meta Setting def set(self): #self.rdm = redis.StrictRedis(host=str(self.host), port=int(self.port), db=int(self.dbnm), socket_timeout=1) try: d = { 0:{'hostname':'172.31.21.219', 'port':5672, 'user':'******', 'pass':'******', 'vhost':'dev-quant', 'exchange_name':'quant'}, 1:{'hostname':'172.31.16.52' , 'port':5672, 'user':'******', 'pass':'******', 'vhost':'dev-quant', 'exchange_name':'quant'} } self.redis_server.set('rabbitmq', d) d = {'state':{'AA':'XKRX-CS-AA', 'KR':'XKRX-CS-KR'}} self.redis_server.set('group', d) d = {'172.31.7.219':'data', '172.31.7.217':'state', '172.31.7.218':'class', '172.31.7.216':'web'} self.redis_server.set('service', d) except Exception, e: print e
MasterNotFoundError, SlaveNotFoundError) # Redis 접속 기본 설정값 listSentinel = [('172.31.5.41', 26379)] strServiceName = 'mymaster' strRedisPass = '' nDB = 0 nMaxUser = 1000 sentinel = Sentinel(listSentinel, socket_timeout=0.1) try: #sentinel.discover_master(strServiceName) # No need for this #sentinel.discover_slaves(strServiceName) master = sentinel.master_for(strServiceName, password=strRedisPass, db=nDB, socket_timeout=0.1) slave = sentinel.slave_for(strServiceName, password=strRedisPass, db=nDB, socket_timeout=0.1) print master """ m = sentinel.discover_master(strServiceName) s = sentinel.discover_slaves(strServiceName) """ except MasterNotFoundError: print 'Master not found or Sentinel instances not runnung' sys.exit() except SlaveNotFoundError: print 'Slave not found or Sentinel instances not runnung' sys.exit() except ConnectionError: print 'Connection Error. Check if Sentinel instances are running' sys.exit()
#from Redis import RedisInfo #r = RedisInfo() from redis import Redis, exceptions, RedisError from redis.sentinel import (Sentinel, SentinelConnectionPool,ConnectionError, MasterNotFoundError, SlaveNotFoundError) # Redis 접속 기본 설정값 listSentinel = [('172.31.5.41', 26379)] strServiceName = 'mymaster' strRedisPass = '' nDB = 0 sentinel = Sentinel(listSentinel) master = sentinel.master_for(strServiceName, password=strRedisPass, db=nDB) slave = sentinel.slave_for(strServiceName, password=strRedisPass, db=nDB) rds = master rds_pub = rds.pubsub() rds_pub.subscribe('0_Text_0') """ print sys.argv[1] """ while True: try: for x in rds_pub.listen(): print "Text :", x['data'] except Exception, e: print "Exception :", e
# Redis Sentinel from redis.sentinel import Sentinel sentinel = Sentinel([('localhost', 26379)], socket_timeout=0.3, db=15, decode_responses=True) master = sentinel.master_for('madeira') slave = sentinel.slave_for('madeira') # redis 增删改查 master.set('foo', 'bar') slave.get('foo') master.delete('foo')
login_manager.login_view = 'account.signin' login_manager.login_message = u"Please sign in to access this page." # Configure theme try: # pragma: no cover # First with local settings import settings_local theme = settings_local.THEME except: # Otherwise try with default theme theme = settings.THEME # Create app app = create_app(theme=theme) sentinel = Sentinel(app.config['REDIS_SENTINEL'], socket_timeout=0.1) redis_master = sentinel.master_for('mymaster') redis_slave = sentinel.slave_for('mymaster') #toolbar = DebugToolbarExtension(app) db = SQLAlchemy(app) mail = Mail(app) signer = URLSafeTimedSerializer(app.config['ITSDANGEORUSKEY']) if app.config.get('SENTRY_DSN'): # pragma: no cover sentr = Sentry(app) babel = Babel(app) @babel.localeselector def get_locale(): if current_user.is_authenticated(): lang = current_user.locale
from bottle import route, run, static_file from math import ceil from redis.sentinel import Sentinel #Slave/Master distribution sentinel = Sentinel('TODO: Get some Redis instances in Amazons AWS') master = sentinel.master_for('.') slave = sentinel.slave_for('.') @route('/') def home(): return static_file('index.html', '..') @route('/get/stats') def getStats(): return generate_json() @route('/<filename>') def server_static(filename): return static_file(filename, root='..') def generate_json(): # Pipeline all the requests to the SLAVE (All read) pipe = slave.pipeline() #Pipeline returns all responses in a list
import time from redis.sentinel import Sentinel sentinel = Sentinel([('192.168.11.35', 26379),('192.168.11.37', 26379),('192.168.11.37', 26379)], socket_timeout=0.1) for i in range(1,1000): print "ciclo " + str(i) try: print "master: " + str(sentinel.discover_master('mymaster')) print "slave/s: " + str(sentinel.discover_slaves('mymaster')) master = sentinel.master_for('mymaster', socket_timeout=0.1) slave = sentinel.slave_for('mymaster', socket_timeout=0.1) except: print "sentinel eligiendo a un master" clave = "clave" + str(i) valor = "valor" + str(i) try: print "master.set " + clave + " " + valor master.set(clave, valor) print "master.get " + clave + " = " + str(master.get(clave)) print "master.set " + clave + " = " + str(slave.get(clave)) print "master.delete " + clave master.delete(clave) except: print "cluster en modo solo lectura"
from mickey.commonconf import SINGLE_MODE, REDIS_PASSWORD if SINGLE_MODE: import redis if REDIS_PASSWORD: _sentinel = redis.StrictRedis(host='localhost', port=6379, db=0, password=REDIS_PASSWORD, socket_timeout=5.0) else: _sentinel = redis.StrictRedis(host='localhost', port=6379, db=0, socket_timeout=5.0) _sentinel_salve = _sentinel _sentinel_master = _sentinel else: from redis.sentinel import Sentinel _sentinel = Sentinel([('localhost', 26379)], socket_timeout = 1) _sentinel_salve = _sentinel.slave_for('master', socket_timeout = 0.5) _sentinel_master = _sentinel.master_for('master', socket_timeout = 0.5) def write_to_redis(key, value, expire = None): if not key or not value: return try: _sentinel_master.set(key, value) if expire: _sentinel_master.expire(key, expire) except Exception as e: logging.error("save information to redis failed {0}".format(e)) def read_from_redis(key, defaultvalue = None):
class BaseCache(object): ''' This class uses redis sentinel to establish a connection with redis. It ensures that all write operations goes to master instance and all read operations served by slave instances. ''' # To support the *args, passed from ReConnectingCache def __init__(self, host='localhost', port=6379, db=0, password=None, socket_timeout=None): self.sentinel = None self.master = None self.selected_db = db self.slave = None self.socket_timeout = socket_timeout self.logger = logging.getLogger("django_sentinel") self.initialize() def initialize(self): try: self.sentinel = Sentinel(settings.LIST_OF_SENTINEL, self.socket_timeout) self.discover_master() self.discover_slave() except: self.logger.exception("Exception !! %s", traceback.format_exc()) raise self.logger.info("RedisHA initialized, db is %d", self.selected_db) def get_master(self): return self.master def get_slave(self): return self.slave def discover_instance(func): ''' It discover the new master/slave, if the current master or slave is down. ''' @wraps(func) def inner(self, *args, **kwargs): try: res = func(self, *args, **kwargs) except SlaveNotFoundError: self.logger.exception("Exception : SlaveNotFoundError") self.discover_slave() res = func(self, *args, **kwargs) except MasterNotFoundError: self.logger.exception("Exception : MasterNotFoundError") self.discover_master() res = func(self, *args, **kwargs) except: self.logger.exception("Exception !! %s", traceback.format_exc()) raise return res return inner @discover_instance def get(self, key): return self.slave.get(key) @discover_instance def set(self, key, value, expiry=7776000): return self.master.set(key, value, expiry) @discover_instance def delete(self, key): return self.master.delete(key) @discover_instance def keys(self): return self.slave.keys() @discover_instance def lpop(self, name): return self.master.lpop(name) @discover_instance def rpop(self, name): return self.master.rpop(name) @discover_instance def lpush(self, name, *values): return self.master.lpush(name, *values) @discover_instance def rpush(self, name, *values): return self.master.rpush(name, *values) @discover_instance def expire(self, name, time): return self.master.expire(name, time) @discover_instance def incr(self, name, amount=1): return self.master.incr(name, amount) @discover_instance def sadd(self, name, *values): return self.master.sadd(name, *values) @discover_instance def srem(self, name, *values): return self.master.srem(name, *values) @discover_instance def scard(self, name): return self.slave.scard(name) @discover_instance def smembers(self, name): return self.slave.smembers(name) @discover_instance def zadd(self, name, *args, **kwargs): return self.master.zadd(name, *args, **kwargs) @discover_instance def zrem(self, name, *values): return self.master.zrem(name, *values) @discover_instance def hset(self, name, key, value): return self.master.hset(name, key, value) @discover_instance def hget(self, name, key): return self.slave.hget(name, key) @discover_instance def hdel(self, name, *keys): return self.master.hdel(name, *keys) @discover_instance def hgetall(self, name): return self.slave.hgetall(name) @discover_instance def hmget(self, name, keys, *args): return self.slave.hmget(name, keys, *args) @discover_instance def flushdb(self): return self.master.flushdb() @discover_instance def dbsize(self): return self.slave.dbsize() @discover_instance def zrangebyscore(self, name, min, max, start=None, num=None, withscores=False, score_cast_func=float): return self.slave.zrangebyscore(name, min, max, start, num, withscores, score_cast_func) def discover_master(self): try: self.master = self.sentinel.master_for(settings.SENTINEL_MASTER, db=self.selected_db, socket_timeout=None) except: self.logger.exception("Exception !! %s", traceback.format_exc()) raise def discover_slave(self): try: self.slave = self.sentinel.slave_for(settings.SENTINEL_MASTER, db=self.selected_db, socket_timeout=None) except: self.logger.exception("Exception !! %s", traceback.format_exc()) raise def __getattr__(self, name): return getattr(self.get_slave(), name)