def get_db(): redis_store = getattr(g, '_database', None) if redis_store is None: if 'FAKE_DB' in app.config and app.config['FAKE_DB']: from mockredis import mock_redis_client redis_store = g._database = mock_redis_client( encoding='utf-8', decode_responses=True) else: if app.config['REDIS_URL'].startswith('redis://'): redis_store = g._database = Redis.from_url( app.config['REDIS_URL'], encoding='utf-8', decode_responses=True) elif app.config['REDIS_URL'].startswith('sentinel://'): parsed_url = urlparse(app.config['REDIS_URL']) service = parsed_url.path.lstrip('/') port = 26379 if ':' in parsed_url.netloc: host, str_port = parsed_url.netloc.split(':') port = int(str_port) else: host = parsed_url.netloc sentinel = Sentinel([(host, port)], socket_timeout=0.1) redis_store = sentinel.master_for(service, redis_class=Redis, socket_timeout=0.1) return redis_store
def __init__(self, name, namespace="queue", **redis_kwargs): import redis # pylint: disable=import-error from redis.sentinel import Sentinel # pylint: disable=import-error redis_kwargs["decode_responses"] = True if redis_kwargs.get("sentinel") and redis_kwargs.get( "sentinel_service"): sentinels = [ tuple(l.split(":")) for l in redis_kwargs.pop("sentinel").split(",") ] sentinel_service = redis_kwargs.pop("sentinel_service") kwargs = { k: v for k, v in redis_kwargs.items() if k in ["decode_responses", "password", "db", "socket_timeout"] } self._redis = Sentinel(sentinels, **kwargs).master_for(sentinel_service) else: kwargs = { k: v for k, v in redis_kwargs.items() if "sentinel" not in k } self._redis = redis.Redis(**kwargs) self._redis.ping() self.key = "{}:{}".format(namespace, name) logger.debug("Created redis queue with socket_timeout of {}s".format( redis_kwargs["socket_timeout"])) # clean up from previous implementations if self._redis.type(self.key) != "zset": self._redis.delete(self.key)
def __init__(self, servers, hash_method='crc32', sentinel=None, strict_redis=False): self.nodes = [] self.connections = {} self.pool = None servers = format_servers(servers) if sentinel: sentinel = Sentinel(sentinel['hosts'], socket_timeout=sentinel.get( 'socket_timeout', 1)) for server_config in servers: name = server_config.pop('name') server_config["max_connections"] = int( server_config.get("max_connections", 100)) if name in self.connections: raise ValueError("server's name config must be unique") if sentinel: self.connections[name] = SentinelRedis(sentinel, name) elif strict_redis: self.connections[name] = redis.StrictRedis(**server_config) else: self.connections[name] = redis.Redis(**server_config) server_config['name'] = name self.nodes.append(name) self.ring = HashRing(self.nodes, hash_method=hash_method)
class RedisCluster(object): def __init__(self, sentinel_hosts, service_name, **connection_kwargs): self.service_name = service_name self.sentinel = Sentinel(sentinel_hosts, **connection_kwargs) self.master = None self.master_host_port = None self.reconfigure() self.sentinel_threads = {} self.listen() def reconfigure(self): try: self.master_host_port = self.sentinel.discover_master(self.service_name) self.master = self.sentinel.master_for(self.service_name) log.info(f"Reconfigured master to {self.master_host_port}") except Exception as ex: log.error(f"Error while reconfiguring. {ex.args[0]}") def listen(self): def on_new_master(workerThread): self.reconfigure() for sentinel in self.sentinel.sentinels: sentinel_host = sentinel.connection_pool.connection_kwargs["host"] self.sentinel_threads[sentinel_host] = MyPubSubWorkerThread( sentinel, on_new_master, msg_sleep_time=0.001, daemon=True ) self.sentinel_threads[sentinel_host].start()
def __init__(self, namespace="lock", **redis_kwargs): import redis # pylint: disable=import-error from redis.sentinel import Sentinel # pylint: disable=import-error redis_kwargs["decode_responses"] = True if redis_kwargs.get("sentinel") and redis_kwargs.get( "sentinel_service"): sentinels = [ tuple(l.split(":")) for l in redis_kwargs.pop("sentinel").split(",") ] sentinel_service = redis_kwargs.pop("sentinel_service") kwargs = { k: v for k, v in redis_kwargs.items() if k in ["decode_responses", "password", "db", "socket_timeout"] } self._redis = Sentinel(sentinels, **kwargs).master_for(sentinel_service) else: kwargs = { k: v for k, v in redis_kwargs.items() if "sentinel" not in k } self._redis = redis.Redis(**kwargs) self._redis.ping() self._namespace = namespace logger.debug( "Created redis lock manager with socket_timeout of {}s".format( redis_kwargs["socket_timeout"]))
def get_connection(self, is_read_only=False) -> redis.StrictRedis: """ Gets a StrictRedis connection for normal redis or for redis sentinel based upon redis mode in configuration. :type is_read_only: bool :param is_read_only: In case of redis sentinel, it returns connection to slave :return: Returns a StrictRedis connection """ if self.connection is not None: return self.connection if self.is_sentinel: kwargs = dict() if self.password: kwargs["password"] = self.password sentinel = Sentinel([(self.host, self.port)], **kwargs) if is_read_only: connection = sentinel.slave_for(self.sentinel_service, decode_responses=True) else: connection = sentinel.master_for(self.sentinel_service, decode_responses=True) else: connection = redis.StrictRedis(host=self.host, port=self.port, decode_responses=True, password=self.password) self.connection = connection return connection
def get_master(self): print(self.redis_hosts) sentinel_hosts = [] for redis_host in self.redis_hosts: redis_address_array = redis_host.split(':') redis_ip = redis_address_array[0] redis_port = redis_address_array[1] address = (redis_ip, redis_port) sentinel_hosts.append(address) print(sentinel_hosts) sentinel = Sentinel([('192.168.206.41', 26379), ('192.168.206.42', 26379), ('192.168.206.43', 26379)], socket_timeout=0.1) print(sentinel.discover_master('mymaster')) master = sentinel.discover_master('mymaster') self.master_ip = master[0] self.master_port = master[1] print(str(master[0]) + ' is master') self.ip_master = { "addresses": [{ "ip": self.master_ip }], "ports": [{ "port": int(self.master_port), "protocol": "TCP" }] } return True
def _get_sentinel_connection(self, uri, kw): """ get sentinel connection details from _uri """ try: from redis.sentinel import Sentinel except ImportError: raise AnsibleError( "The 'redis' python module (version 2.9.0 or newer) is required to use redis sentinel." ) if ';' not in uri: raise AnsibleError('_uri does not have sentinel syntax.') # format: "localhost:26379;localhost2:26379;0:changeme" connections = uri.split(';') connection_args = connections.pop(-1) if len(connection_args) > 0: # hanle if no db nr is given connection_args = connection_args.split(':') kw['db'] = connection_args.pop(0) try: kw['password'] = connection_args.pop(0) except IndexError: pass # password is optional sentinels = [tuple(shost.split(':')) for shost in connections] display.vv('\nUsing redis sentinels: %s' % sentinels) scon = Sentinel(sentinels, **kw) try: return scon.master_for(self._sentinel_service_name, socket_timeout=0.2) except Exception as exc: raise AnsibleError('Could not connect to redis sentinel: %s' % to_native(exc))
def setup_rq_connection(): from redis.sentinel import Sentinel redis_url = current_app.config.get('RQ_SCHEDULER_DASHBOARD_REDIS_URL') redis_sentinels = current_app.config.get( 'RQ_SCHEDULER_DASHBOARD_REDIS_SENTINELS') if redis_url: current_app.redis_conn = from_url(redis_url) elif redis_sentinels: redis_master = current_app.config.get( 'RQ_SCHEDULER_DASHBOARD_REDIS_MASTER_NAME') password = current_app.config.get( 'RQ_SCHEDULER_DASHBOARD_REDIS_PASSWORD') db = current_app.config.get('RQ_SCHEDULER_DASHBOARD_REDIS_DB') sentinel_hosts = [ tuple(sentinel.split(':', 1)) for sentinel in redis_sentinels.split(',') ] sentinel = Sentinel(sentinel_hosts, db=db, password=password) current_app.redis_conn = sentinel.master_for(redis_master) else: current_app.redis_conn = Redis( host=current_app.config.get('REDIS_HOST'), port=current_app.config.get('REDIS_PORT'), password=current_app.config.get('REDIS_PASSWORD'), db=current_app.config.get('REDIS_DB'))
def __init__(self, connection_str, db=0, project=None, service=None, host=None, conf=cfg.CONF, **kwargs): """Redis driver for OSProfiler.""" super(RedisSentinel, self).__init__(connection_str, project=project, service=service, host=host, conf=conf, **kwargs) try: from redis.sentinel import Sentinel except ImportError: raise exc.CommandError("To use this command, you should install " "'redis' manually. Use command:\n " "'pip install redis'.") self.conf = conf socket_timeout = self.conf.profiler.socket_timeout parsed_url = parser.urlparse(self.connection_str) sentinel = Sentinel([(parsed_url.hostname, int(parsed_url.port))], password=parsed_url.password, socket_timeout=socket_timeout) self.db = sentinel.master_for(self.conf.profiler.sentinel_service_name, socket_timeout=socket_timeout)
def get_redis_connection(config, use_strict_redis=False): """ Returns a redis connection from a connection config """ redis_cls = redis.StrictRedis if use_strict_redis else redis.Redis if 'URL' in config: return redis_cls.from_url(config['URL'], db=config.get('DB')) if 'USE_REDIS_CACHE' in config.keys(): try: from django.core.cache import caches cache = caches[config['USE_REDIS_CACHE']] except ImportError: from django.core.cache import get_cache cache = get_cache(config['USE_REDIS_CACHE']) if hasattr(cache, 'client'): # We're using django-redis. The cache's `client` attribute # is a pluggable backend that return its Redis connection as # its `client` try: # To get Redis connection on django-redis >= 3.4.0 # we need to use cache.client.get_client() instead of # cache.client.client used in older versions try: return cache.client.get_client() except AttributeError: return cache.client.client except NotImplementedError: pass else: # We're using django-redis-cache try: return cache._client except AttributeError: # For django-redis-cache > 0.13.1 return cache.get_master_client() if 'UNIX_SOCKET_PATH' in config: return redis_cls(unix_socket_path=config['UNIX_SOCKET_PATH'], db=config['DB']) if 'SENTINELS' in config: sentinel_kwargs = { 'db': config.get('DB'), 'password': config.get('PASSWORD'), 'socket_timeout': config.get('SOCKET_TIMEOUT'), } sentinel_kwargs.update(config.get('CONNECTION_KWARGS', {})) sentinel = Sentinel(config['SENTINELS'], **sentinel_kwargs) return sentinel.master_for( service_name=config['MASTER_NAME'], redis_class=redis_cls, ) return redis_cls(host=config['HOST'], port=config['PORT'], db=config['DB'], password=config.get('PASSWORD'))
def post(self, location): if location not in self.locations: return 'Unknown Location' if app.config['DEVEL']: pool = redis.ConnectionPool(host='localhost', port=6379) r = redis.Redis(connection_pool=pool) else: sentinel = Sentinel([('localhost', 26379)], socket_timeout=1) r = sentinel.master_for('mymaster', socket_timeout=1) beers = {key: r.hgetall(key) for key in r.keys('beer_{0}_*'.format(location))} for beername, beer in beers.items(): r.hset(beername, 'active', 'True' if beername in request.form.getlist('checks') else 'False') for beer in request.form.getlist('delete'): r.delete(beer) r.save() beers = [] for key in r.keys('beer_{0}_*'.format(location)): beer = convert(r.hgetall(key)) beer['beername'] = key beers.append(beer) beers.sort(key=operator.itemgetter('brewery', 'name', 'type')) return redirect(location)
def get(self, location): if location not in self.locations: return 'Unknown Location' colors = get_colors(location, self.config) priceinfo = get_priceinfo(location, self.config) if app.config['DEVEL']: pool = redis.ConnectionPool(host='localhost', port=6379) r = redis.Redis(connection_pool=pool) else: sentinel = Sentinel([('localhost', 26379)], socket_timeout=1) r = sentinel.master_for('mymaster', socket_timeout=1) beers = [] for key in r.keys('beer_{0}_*'.format(location)): beer = convert(r.hgetall(key)) beer['beername'] = key beers.append(beer) if 'items' in priceinfo: for beer in beers: beer['price'] = '$ {0}'.format(' /'.join([beer[p].replace('.0', '') for p in priceinfo['items'] if p in beer])) beers.sort(key=operator.itemgetter('brewery', 'name', 'type')) return render_template( 'edit.html', title='Beer List', beers=beers, location=location, colors=colors, roles=self.groups, locations=self.locations, priceinfo=priceinfo, scroll=False )
class TestApplication(Application): def __init__(self): # Call to the superclass to bootstrap. super(TestApplication, self).__init__(name='krux-redis-sentinel') self.logger.debug('Parsing sentinels from args: %r', self.args.sentinel) sentinels = hostports_from_args(self.args) self.logger.debug('Parsed sentinel host, port pairs: %r', sentinels) self.logger.debug('Initializing Sentinel instance...') self.sentinel = Sentinel(sentinels, socket_timeout=SOCKET_TIMEOUT) self.logger.debug('Initialized Sentinel instance: %r', self.sentinel) def add_cli_arguments(self, parser): add_sentinel_cli_arguments(parser) def run(self): master = self.sentinel.master_for(self.name) slave = self.sentinel.slave_for(self.name) try: self.logger.info('Ping master: %s', master.ping()) except ConnectionError as err: self.logger.warning('Could not connect to master!') self.logger.error(err) try: self.logger.info('Ping slave: %s', slave.ping()) except ConnectionError as err: self.logger.warning('Could not connect to slave!') self.logger.error(err)
def redis_client(): if settings.CACHEOPS_REDIS and settings.CACHEOPS_SENTINEL: raise ImproperlyConfigured("CACHEOPS_REDIS and CACHEOPS_SENTINEL are mutually exclusive") client_class = CacheopsRedis if settings.CACHEOPS_CLIENT_CLASS: client_class = import_string(settings.CACHEOPS_CLIENT_CLASS) if settings.CACHEOPS_SENTINEL: if not {'locations', 'service_name'} <= set(settings.CACHEOPS_SENTINEL): raise ImproperlyConfigured("Specify locations and service_name for CACHEOPS_SENTINEL") sentinel = Sentinel( settings.CACHEOPS_SENTINEL['locations'], **omit(settings.CACHEOPS_SENTINEL, ('locations', 'service_name', 'db'))) return sentinel.master_for( settings.CACHEOPS_SENTINEL['service_name'], redis_class=client_class, db=settings.CACHEOPS_SENTINEL.get('db', 0) ) # Allow client connection settings to be specified by a URL. if isinstance(settings.CACHEOPS_REDIS, six.string_types): return client_class.from_url(settings.CACHEOPS_REDIS) else: return client_class(**settings.CACHEOPS_REDIS)
def setup_rq_connection(): # It's the only place where we can safely define default value for web background # since It is used in template current_app.config.setdefault('RQ_DASHBOARD_WEB_BACKGROUND', 'black') # we need to do It here instead of cli, since It may be embeded upgrade_config(current_app) # Getting Redis connection parameters for RQ redis_url = current_app.config.get('RQ_DASHBOARD_REDIS_URL') redis_sentinels = current_app.config.get('RQ_DASHBOARD_REDIS_SENTINELS') if isinstance(redis_url, list): current_app.redis_conn = from_url(redis_url[0]) elif isinstance(redis_url, string_types): current_app.redis_conn = from_url(redis_url) elif redis_sentinels: redis_master = current_app.config.get('RQ_DASHBOARD_REDIS_MASTER_NAME') password = current_app.config.get('RQ_DASHBOARD_REDIS_PASSWORD') db = current_app.config.get('RQ_DASHBOARD_REDIS_DB') sentinel_hosts = [tuple(sentinel.split(':', 1)) for sentinel in redis_sentinels.split(',')] sentinel = Sentinel(sentinel_hosts, db=db, password=password) current_app.redis_conn = sentinel.master_for(redis_master) else: current_app.redis_conn = Redis( host=current_app.config.get('RQ_DASHBOARD_REDIS_HOST', 'localhost'), port=current_app.config.get('RQ_DASHBOARD_REDIS_PORT', 6379), password=current_app.config.get('RQ_DASHBOARD_REDIS_PASSWORD'), db=current_app.config.get('RQ_DASHBOARD_REDIS_DB', 0), )
def _init(self, server, params): super(CacheClass, self).__init__(params) self._server = server self._params = params self.service_name = self.options.get("SERVICE_NAME", 'mymaster') self.slave_read = self.options.get("READ_SLAVE", False) if self.server: def _get_hosttupe(conn): host_lst = conn.rsplit(':', 1) return host_lst[0], int(host_lst[1]) conn_list = [_get_hosttupe(conn) for conn in self.server] else: raise ImproperlyConfigured("sentinel server config error.") kwargs = self.connection_pool_class_kwargs max_connections = kwargs.pop('max_connections') sentinel_manager = Sentinel(conn_list, sentinel_kwargs=kwargs) kwargs.update({ 'db': self.db, 'password': self.password, 'max_connections': max_connections }) self._client = sentinel_manager.master_for(self.service_name, connection_pool_class=self.connection_pool_class, **kwargs) if self.slave_read: self._slave_client = sentinel_manager.slave_for(self.service_name, connection_pool_class=self.connection_pool_class, **kwargs)
def get_redis(app=None): app = app_or_default(app) conf = ensure_conf(app) if not hasattr(app, 'redbeat_redis') or app.redbeat_redis is None: redis_options = conf.app.conf.get( 'REDBEAT_REDIS_OPTIONS', conf.app.conf.get('BROKER_TRANSPORT_OPTIONS', {})) retry_period = redis_options.get('retry_period') if conf.redis_url.startswith( 'redis-sentinel') and 'sentinels' in redis_options: from redis.sentinel import Sentinel sentinel = Sentinel( redis_options['sentinels'], socket_timeout=redis_options.get('socket_timeout'), password=redis_options.get('password'), decode_responses=True) connection = sentinel.master_for( redis_options.get('service_name', 'master')) else: connection = StrictRedis.from_url(conf.redis_url, decode_responses=True) if retry_period is None: app.redbeat_redis = connection else: app.redbeat_redis = RetryingConnection(retry_period, connection) return app.redbeat_redis
def create_app(): '''工厂方法''' # 创建app对象 app = Flask(__name__) # 加载配置 config = config_dict.get(app.config.get('ENV')) app.config.from_object(config) # 加载数据库配置 from common.models import db db.init_app(app) # 记录日志信息 log_file(config.LOG_LV) # redis集群加载 from rediscluster import RedisCluster app.redis_cluster = RedisCluster( startup_nodes=app.config.get('REDIS_CLUSTER')) # redis哨兵,主从配置 from redis.sentinel import Sentinel sentinel = Sentinel(app.config.get('REDIS_SENTINELS')) app.redis_master = sentinel.master_for( app.config.get('REDIS_SENTINEL_SERVICES_NAME')) app.redis_slave = sentinel.slave_for( app.config.get('REDIS_SENTINEL_SERVICES_NAME')) # 注册用户蓝图到app中 from .resources.users import user_blue app.register_blueprint(user_blue) return app
def __get_master(self, update=False): master_ip = self.node.get_ip() master_port = self.node.get_port() master_node_id = self.node.get_node_id() hosts = self.__get_hosts(refresh=update) if len(hosts) > 0: master_ip = hosts[0]['ip'] master_node_id = hosts[0]['node_id'] if os.path.isfile(Constants.MASTER_FILE): fd = open(Constants.MASTER_FILE, "r") temp_ip, temp_port = fd.readline().split(' ') fd.close() if temp_ip in hosts: master_ip, master_port = temp_ip, temp_port else: os.remove(Constants.MASTER_FILE) for host in hosts: sentinel = Sentinel([(host["ip"], Constants.SENTINEL_PORT)], socket_timeout=0.1) try: ip, port = sentinel.discover_master(Constants.REDIS_SENTINEL_NAME) for ht in hosts: if ip == ht['ip']: master_ip, master_port = ip, port break except: pass return master_ip, master_port
def __init__(self, name, namespace='queue', **redis_kwargs): import redis from redis.sentinel import Sentinel redis_kwargs['decode_responses'] = True if redis_kwargs.get('sentinel') and redis_kwargs.get( 'sentinel_service'): sentinels = [ tuple(l.split(':')) for l in redis_kwargs.pop('sentinel').split(',') ] sentinel_service = redis_kwargs.pop('sentinel_service') kwargs = { k: v for k, v in redis_kwargs.items() if k in ["decode_responses", "password", "db", "socket_timeout"] } self._redis = Sentinel(sentinels, **kwargs).master_for(sentinel_service) else: kwargs = { k: v for k, v in redis_kwargs.items() if "sentinel" not in k } self._redis = redis.Redis(**kwargs) self._redis.ping() self.key = "{}:{}".format(namespace, name) info("Created redis queue with socket_timeout of {}s".format( redis_kwargs['socket_timeout'])) # clean up from previous implementations if self._redis.type(self.key) != 'zset': self._redis.delete(self.key)
def redis_populate(): """Function to populate keys in Redis Server""" # client = redis.StrictRedis(host=configs["redis_host"], port=configs["redis_port"]) # sentinel = Sentinel( # [ # ("redis-cloudflare-node-0.redis-cloudflare-headless.dns-proxy.svc.cluster.local", 26379), # ("redis-cloudflare-node-1.redis-cloudflare-headless.dns-proxy.svc.cluster.local", 26379), # ("redis-cloudflare-node-2.redis-cloudflare-headless.dns-proxy.svc.cluster.local", 26379) # ], # socket_timeout=0.1) from redis.sentinel import Sentinel sentinel = Sentinel([ ('redis-cloudflare', 26379), ], sentinel_kwargs={'password': '******'}, password='******') sentinel.discover_master('mymaster') client = sentinel.master_for('mymaster', socket_timeout=0.5) i = 93218 key = 'key' + str(i) value = 'value' + str(i) result = client.get(key) print(result)
def redis_populate(): """Function to populate keys in Redis Server""" # client = redis.StrictRedis(host=configs["redis_host"], port=configs["redis_port"]) # sentinel = Sentinel( # [ # ("redis-cloudflare-node-0.redis-cloudflare-headless.dns-proxy.svc.cluster.local", 26379), # ("redis-cloudflare-node-1.redis-cloudflare-headless.dns-proxy.svc.cluster.local", 26379), # ("redis-cloudflare-node-2.redis-cloudflare-headless.dns-proxy.svc.cluster.local", 26379) # ], # socket_timeout=0.1) from redis.sentinel import Sentinel sentinel = Sentinel([ ('redis-cloudflare', 26379), ], sentinel_kwargs={'password': '******'}, password='******') sentinel.discover_master('mymaster') client = sentinel.master_for('mymaster', socket_timeout=0.5) for i in range(1000000): key = 'key' + str(i) # value='value'+str(i) value = str( i ) + '::: https://www.google.co.uk/#sclient=psy-ab&hl=en&source=hp&q=ASUSTeK+Computer+INC.+Model+M4A78LT-M+manual&pbx=1&oq=ASUSTeK+Computer+INC.+Model+M4A78LT-M+manual&aq=f&aqi=&aql=&gs_sm=3&gs_upl=52765l57528l0l57848l8l8l0l0l0l0l2413l3989l8-1.1l2l0&bav=on.2,or.r_gc.r_pw.,cf.osb&fp=3d6c1d1d0a5ea45f&biw=1262&bih=879' client.set(key, value) print(key, value)
def get_redis_connection(config, is_master=True): """ Returns redis connection basing on configuration dict Args: config: Dict with configuration, if key `SENTINELS` is present sentinel connection pool will be used. is_master: Whether connection should be master or slave, applies only if sentinels are used Returns: StrictRedis connection """ if 'SENTINELS' in config: sentinel = Sentinel( config['SENTINELS'], db=config.get('DB'), password=config.get('PASSWORD'), socket_timeout=config.get('TIMEOUT'), socket_connect_timeout=config.get('CONNECT_TIMEOUT'), ) if is_master: return sentinel.master_for(settings.REDIS_CLUSTER_NAME) else: return sentinel.slave_for(settings.REDIS_CLUSTER_NAME) else: return StrictRedis( host=config['HOST'], port=config['PORT'], db=config.get('DB'), password=config.get('PASSWORD'), socket_timeout=config.get('TIMEOUT'), socket_connect_timeout=config.get('CONNECT_TIMEOUT'), )
def sentinel_connect(self, master_name): url = urlparse.urlparse(self.schedule_url) def parse_host(s): if ':' in s: host, port = s.split(':', 1) port = int(port) else: host = s port = 26379 return host, port if '@' in url.netloc: auth, hostspec = url.netloc.split('@', 1) else: auth = None hostspec = url.netloc if auth and ':' in auth: _, password = auth.split(':', 1) else: password = None path = url.path if path.startswith('/'): path = path[1:] hosts = [parse_host(s) for s in hostspec.split(',')] sentinel = Sentinel(hosts, password=password, db=path) master = sentinel.master_for(master_name) return master
def connect(self, index=0, write=True): """ Creates a redis connection with connection pool. """ master_name, sentinel_hosts, db = self.parse_connection_string(self._connection_string) sentinel_timeout = self._options.get('SENTINEL_TIMEOUT', 1) sentinel = Sentinel(sentinel_hosts, socket_timeout=sentinel_timeout) if write: host, port = sentinel.discover_master(master_name) else: host, port = random.choice([sentinel.discover_master(master_name)] + sentinel.discover_slaves(master_name)) kwargs = { "db": db, "parser_class": self.parser_class, "password": self._options.get('PASSWORD', None), } kwargs.update({'host': host, 'port': port, 'connection_class': Connection}) if 'SOCKET_TIMEOUT' in self._options: kwargs.update({'socket_timeout': int(self._options['SOCKET_TIMEOUT'])}) kwargs.update(self._pool_cls_kwargs) connection_pool = get_or_create_connection_pool(self._pool_cls, **kwargs) connection = Redis(connection_pool=connection_pool) return connection
def __redis_client__(self, instance): try: LOG.debug('Connecting to redis databaseinfra %s', self.databaseinfra) # redis uses timeout in seconds connection_timeout_in_seconds = Configuration.get_by_name_as_int('redis_connect_timeout', default=REDIS_CONNECTION_DEFAULT_TIMEOUT) if (instance and instance.instance_type == Instance.REDIS) or (not self.databaseinfra.plan.is_ha and not instance): connection_address, connection_port = self.__get_admin_single_connection(instance) client = redis.Redis(host = connection_address, port = int(connection_port), password = self.databaseinfra.password, socket_connect_timeout = connection_timeout_in_seconds) else: sentinels = self.__get_admin_sentinel_connection(instance) sentinel = Sentinel(sentinels, socket_timeout=connection_timeout_in_seconds) client = sentinel.master_for(self.databaseinfra.name, socket_timeout=connection_timeout_in_seconds, password = self.databaseinfra.password) LOG.debug('Successfully connected to redis databaseinfra %s' % (self.databaseinfra)) return client except Exception, e: raise e
def _connection(self): sentinel = Sentinel(self._sentinels, socket_timeout=self._socket_timeout) slave = sentinel.slave_for(self._sentinel_master, socket_timeout=self._socket_timeout, db=self._db) return slave
def connect(self, write=True): """ Create a redis connection with connection pool. """ self.log.debug("connect called: write=%s", write) sentinel_timeout = self._options.get("SENTINEL_TIMEOUT", 1) password = self._options.get("PASSWORD", None) sentinel = Sentinel( self._sentinel_hosts, socket_timeout=sentinel_timeout, password=password ) if write: host, port = sentinel.discover_master(self._master_name) else: try: host, port = random.choice( # nosec sentinel.discover_slaves(self._master_name) ) except IndexError: self.log.debug("no slaves are available. using master for read.") host, port = sentinel.discover_master(self._master_name) if password: connection_url = f"redis://:{password}@{host}:{port}/{self._database_name}" else: connection_url = f"redis://{host}:{port}/{self._database_name}" self.log.debug("Connecting to: %s", connection_url) return self.connection_factory.connect(connection_url)
def get_redis_master(redis_sentinel_nodes): nodes_list = redis_sentinel_nodes.split(',') sentinel = Sentinel([tuple(nodes_list[0].split(':'))], socket_timeout=0.1) master = sentinel.discover_master('mymaster') if not master: sys.exit(1) return list(master)
def __init__(self, namespace='lock', **redis_kwargs): import redis from redis.sentinel import Sentinel redis_kwargs['decode_responses'] = True if redis_kwargs.get('sentinel') and redis_kwargs.get( 'sentinel_service'): sentinels = [ tuple(l.split(':')) for l in redis_kwargs.pop('sentinel').split(',') ] sentinel_service = redis_kwargs.pop('sentinel_service') kwargs = { k: v for k, v in redis_kwargs.items() if k in ["decode_responses", "password", "db", "socket_timeout"] } self._redis = Sentinel(sentinels, **kwargs).master_for(sentinel_service) else: kwargs = { k: v for k, v in redis_kwargs.items() if "sentinel" not in k } self._redis = redis.Redis(**kwargs) self._redis.ping() self._namespace = namespace info("Created redis lock manager with socket_timeout of {}s".format( redis_kwargs['socket_timeout']))
class RedisLock(Lock): def __init__(self, namespace='lock', **redis_kwargs): import redis from redis.sentinel import Sentinel redis_kwargs['decode_responses'] = True if redis_kwargs.get('sentinel') and redis_kwargs.get('sentinel_service'): sentinels = [tuple(l.split(':')) for l in redis_kwargs.pop('sentinel').split(',')] sentinel_service = redis_kwargs.pop('sentinel_service') kwargs = {k: v for k, v in redis_kwargs.items() if k in ["decode_responses", "password", "db", "socket_timeout"]} self._redis = Sentinel(sentinels, **kwargs).master_for(sentinel_service) else: kwargs = {k: v for k, v in redis_kwargs.items() if "sentinel" not in k} self._redis = redis.Redis(**kwargs) self._redis.ping() self._namespace = namespace info("Created redis lock manager with socket_timeout of {}s".format(redis_kwargs['socket_timeout'])) def __key(self, name): return "{}:{}".format(self._namespace, name) def lock(self, name, owner, expiration=1, allow_owner_relock=False): """ Obtain the named lock. :param allow_owner_relock: bool :param name: str the name of the lock :param owner: str a unique name for the locking node :param expiration: int in seconds, 0 expiration means forever """ import redis try: if int(expiration) < 1: expiration = 1 key = self.__key(name) non_existing = not (allow_owner_relock and self._redis.get(key) == owner) return self._redis.set(key, owner, ex=int(expiration), nx=non_existing) except redis.exceptions.ResponseError as e: exception("Unable to obtain lock, local state: name: %s, owner: %s, expiration: %s, allow_owner_relock: %s", name, owner, expiration, allow_owner_relock) def unlock(self, name, owner): """ Release the named lock. :param name: str the name of the lock :param owner: str a unique name for the locking node """ key = self.__key(name) if self._redis.get(key) == owner: self._redis.delete(key) return True return False def check_lock(self, name): return self._redis.get(self.__key(name)) is not None def print_locks(self): keys = self._redis.keys(self.__key('*')) for key in keys: print("{} locked by {}, expires in {} seconds".format(key, self._redis.get(key), self._redis.ttl(key)))
def get_redis_from_config(settings, connection_class=Redis): """Returns a StrictRedis instance from a dictionary of settings. To use redis sentinel, you must specify a dictionary in the configuration file. Example of a dictionary with keys without values: SENTINEL: {'INSTANCES':, 'SOCKET_TIMEOUT':, 'PASSWORD':,'DB':, 'MASTER_NAME':} """ if settings.get('REDIS_URL') is not None: return connection_class.from_url(settings['REDIS_URL']) elif settings.get('SENTINEL') is not None: instances = settings['SENTINEL'].get('INSTANCES', [('localhost', 26379)]) socket_timeout = settings['SENTINEL'].get('SOCKET_TIMEOUT', None) password = settings['SENTINEL'].get('PASSWORD', None) db = settings['SENTINEL'].get('DB', 0) master_name = settings['SENTINEL'].get('MASTER_NAME', 'mymaster') sn = Sentinel(instances, socket_timeout=socket_timeout, password=password, db=db) return sn.master_for(master_name) kwargs = { 'host': settings.get('REDIS_HOST', 'localhost'), 'port': settings.get('REDIS_PORT', 6379), 'db': settings.get('REDIS_DB', 0), 'password': settings.get('REDIS_PASSWORD', None), 'ssl': settings.get('REDIS_SSL', False), } return connection_class(**kwargs)
def get_connection(self, is_read_only=False) -> redis.Redis: """ Gets a Redis connection for normal redis or for redis sentinel based upon redis mode in configuration. :type is_read_only: bool :param is_read_only: In case of redis sentinel, it returns connection to slave :return: Returns a Redis connection """ if self.connection is not None: return self.connection if self.is_sentinel: kwargs = dict() if self.password: kwargs["password"] = self.password sentinel = Sentinel([(self.host, self.port)], **kwargs) if is_read_only: connection = sentinel.slave_for(self.sentinel_service, decode_responses=True) else: connection = sentinel.master_for(self.sentinel_service, decode_responses=True) else: connection = redis.Redis(host=self.host, port=self.port, decode_responses=True, password=self.password) self.connection = connection return connection
def __init__(self, **config): try: redisIpConf = config['_channel_redis_sentinel'] if not redisIpConf: log.error('Can not find valid ip config.') else: redisConfigure = [] redisIpSplit = redisIpConf.split(',') for redisInfo in redisIpSplit: redisInfoSplit = redisInfo.split(':') redisConfigure.append( (redisInfoSplit[0], redisInfoSplit[1])) self.sentinel = Sentinel(redisConfigure) self.redisInstance = self.sentinel.master_for( 'redis-master', password=config['_channel_redis_password']) ##self._pool = redis.ConnectionPool(host=config['_channel_redis_host'], port=config['_channel_redis_port'], db=1, password=config['_channel_redis_password']) ##self.redisInstance = redis.StrictRedis(connection_pool=self._pool) self._master_pub_topic = config['_master_pub_topic'] self._none_match_ip = 'no_ip_matched' self._sub_node = config['_sub_node'] except: #log.error("redis init failed.") log.error(traceback.format_exc())
def redis_client(): if settings.CACHEOPS_REDIS and settings.CACHEOPS_SENTINEL: raise ImproperlyConfigured("CACHEOPS_REDIS and CACHEOPS_SENTINEL are mutually exclusive") client_class = CacheopsRedis if settings.CACHEOPS_CLIENT_CLASS: client_class = import_string(settings.CACHEOPS_CLIENT_CLASS) if settings.CACHEOPS_SENTINEL: if not {'locations', 'service_name'} <= set(settings.CACHEOPS_SENTINEL): raise ImproperlyConfigured("Specify locations and service_name for CACHEOPS_SENTINEL") sentinel = Sentinel( settings.CACHEOPS_SENTINEL['locations'], **omit(settings.CACHEOPS_SENTINEL, ('locations', 'service_name', 'db'))) return sentinel.master_for( settings.CACHEOPS_SENTINEL['service_name'], redis_class=client_class, db=settings.CACHEOPS_SENTINEL.get('db', 0) ) # Allow client connection settings to be specified by a URL. if isinstance(settings.CACHEOPS_REDIS, str): return client_class.from_url(settings.CACHEOPS_REDIS) else: return client_class(**settings.CACHEOPS_REDIS)
def connect2redis(host, port=6379, db=0, connectionPool=True, logging=None, sentinel_service=prms.REDIS_SENTINEL_SERVICE_NAME, master=True): if sentinel_service: retry_count = 0 while retry_count < 10: sentinel = Sentinel([(host, 26379)]) if master: client = sentinel.master_for(sentinel_service) else: client = sentinel.slave_for(sentinel_service) if is_available(client, logging=logging): logging.info("Connected to Redis succesfully.") return client else: print(f"Waiting for {pow(2, retry_count)} secs.") time.sleep(pow(2, retry_count)) retry_count += 1 else: retry_count = 0 while retry_count < 10: if connectionPool: pool = redis.ConnectionPool(host=host, port=port, db=db) client = redis.StrictRedis(connection_pool=pool) else: client = redis.StrictRedis(host=host, port=port, db=db) if is_available(client, logging=logging): logging.info("Connected to Redis succesfully.") return client else: print(f"Waiting for {pow(2, retry_count)} secs.") time.sleep(pow(2, retry_count)) retry_count += 1 logging.error("Exiting...") exit(3)
def run(): try: logging.info('Calculating training booking reminders') if SENTINEL_HOST is None: client = redis.StrictRedis(host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD) else: sentinel = Sentinel([(SENTINEL_HOST, SENTINEL_PORT)]) client = sentinel.master_for(SENTINEL_MASTER) # When was the last time we notified for? Default to now last_not_str = client.get(WEEK_LAST_CHECK) now = datetime.datetime.now() last_notified = now if last_not_str is not None: last_notified_date = dateutil.parser.parse(last_not_str) # Check that this is after the current date if last_notified_date > now: last_notified = last_notified_date # Now work out the date that is midnight one week and a day from now one_week = datetime.datetime.combine(datetime.date.today() + datetime.timedelta(8), datetime.time()) logging.info('Getting sessions from %s until %s' % (str(last_notified), str(one_week))) if one_week <= last_notified: return # Find all the sessions that fall bewtween the last notified date and a weeks time sessions = TrainingSession.query({ 'find': { 'start': { '$gte': last_notified, '$lt': one_week } } }) # Notify that these need reminding for session in sessions: data = { 'name': 'reminder', 'data': { 'type': 'trainingSession', 'data': session._id } } client.rpush(EVENT_Q, json.dumps(data)) if (len(sessions) > 0): client.publish(EVENT_CHAN, 'event') logging.info('Published training booking events') # Store that we've notified client.set(WEEK_LAST_CHECK, one_week.isoformat()) logging.info('Finished training booking') except Exception as ex: logging.exception(ex)
def check_sso_auth(user_id, brand_id, trans_type ): sentinel = Sentinel([('120.76.154.208', 26379)], socket_timeout=0.5) log.d(sentinel.discover_master('mymaster')) redis_obj = sentinel.master_for('mymaster', socket_timeout=0.1) db_obj = PostgreSQLConnector() sso_cls = SSOAuth(redis_obj,db_obj) return sso_cls.check_auth(user_id, brand_id, trans_type)
def sso_log(user_id, brand_id, trans_type, data_type,data_content,log_time): sentinel = Sentinel([('112.74.198.224', 26379)], socket_timeout=0.5) log.d(sentinel.discover_master('mymaster')) redis_obj = sentinel.master_for('mymaster', socket_timeout=0.1) db_obj = PostgreSQLConnector() sso_cls = SSOAuth(redis_obj,db_obj) return sso_cls.send_log(user_id, brand_id, trans_type, data_type,data_content,log_time)
def redis_conn(readonly=True): global sentinel, master_client, slave_client if not sentinel: sentinel = Sentinel(settings.REDIS_SENTINEL, socket_timeout=1) if not master_client: master_client = sentinel.master_for(settings.REDIS_CLUSTER) if not slave_client: slave_client = sentinel.slave_for(settings.REDIS_CLUSTER) return slave_client if readonly else master_client
def setUp(self): sentinel = Sentinel(settings_test.REDIS_SENTINEL) db = getattr(settings_test, 'REDIS_DB', 0) self.connection = sentinel.master_for('mymaster', db=db) self.connection.flushall() self.guard = ContributionsGuard(self.connection) self.anon_user = {'user_id': None, 'user_ip': '127.0.0.1'} self.auth_user = {'user_id': 33, 'user_ip': None} self.task = Task(id=22)
def client(self): sentinel = Sentinel( self.sentinel_conf.get('sentinels'), min_other_sentinels=self.sentinel_conf.get("min_other_sentinels", 0), password=self.sentinel_conf.get("password"), sentinel_kwargs={"socket_timeout": self.sentinel_conf.get("sentinel_timeout")}, ) return sentinel.master_for(self.sentinel_conf.get("service_name"), Redis, socket_timeout=self.sentinel_conf.get("socket_timeout"))
def __init__(self, **redis_conf): self.prefix = 'q' sentinels = redis_conf.pop('sentinels', '()') sentinels_service_name = redis_conf.pop('sentinels_service_name', 'mymaster') min_other_sentinels = redis_conf.pop('min_other_sentinels', 0) s = Sentinel(sentinels, **redis_conf) self.db = s.master_for(sentinels_service_name) #self.db = StrictRedis(**redis_conf) self.key = '%s:%s' % (self.prefix, '%s')
def _sentinel_managed_pool(self): sentinel = Sentinel( self.sentinels, min_other_sentinels=getattr(self, "min_other_sentinels", 0), password=getattr(self, "password", None), sentinel_kwargs={"socket_timeout": getattr(self, "sentinel_timeout", None)}, ) return sentinel.master_for(self.service_name, self.Client, socket_timeout=self.socket_timeout).connection_pool
def get_redis_url(): if hasattr(config, 'ETCD_HOST'): return config.HERA_REDIS_SENTINEL sentinel_list = [tuple(i.split(':')) for i in get_etcd_setting('HERA_REDIS_SENTINEL').split(',')] redis_db = get_etcd_setting('HERA_REDIS_DB') sentinel = Sentinel(sentinel_list, socket_timeout=0.1) master_name = get_etcd_setting('HERA_REDIS_MASTERNAME') return 'redis://{}/{}'.format( ':'.join(map(str, sentinel.discover_master(master_name))), redis_db )
def new_new(_, cls, broker_url, *args, **kwargs): scheme = urlparse(broker_url).scheme if scheme == "redis-sentinel": from rpaas.tasks import app opts = app.conf.BROKER_TRANSPORT_OPTIONS s = Sentinel(opts["sentinels"], password=opts["password"]) host, port = s.discover_master(opts["service_name"]) return RedisBroker("redis://:{}@{}:{}".format(opts["password"], host, port)) else: old_new(cls, broker_url, *args, **kwargs)
def _redis(self): if getattr(settings, 'EXPERIMENTS_REDIS_SENTINELS', None): sentinel = Sentinel(settings.EXPERIMENTS_REDIS_SENTINELS, socket_timeout=settings.EXPERIMENTS_REDIS_SENTINELS_TIMEOUT) host, port = sentinel.discover_master(settings.EXPERIMENTS_REDIS_MASTER_NAME) else: host = getattr(settings, 'EXPERIMENTS_REDIS_HOST', 'localhost') port = getattr(settings, 'EXPERIMENTS_REDIS_PORT', 6379) password = getattr(settings, 'EXPERIMENTS_REDIS_PASSWORD', None) db = getattr(settings, 'EXPERIMENTS_REDIS_DB', 0) return redis.Redis(host=host, port=port, password=password, db=db)
def init_shards_from_sentinel(self): sentinel_client = Sentinel( [(h, 8422) for h in self._sentinels], socket_timeout=self._timeout) # Connect to all the shards with the names specified per # shard_name_format. The names are important since it's getting # the instances from Redis Sentinel. for shard_num in range(0, self.num_shards()): shard_name = self.get_shard_name(shard_num) self._shards[shard_num] = sentinel_client.master_for( shard_name, socket_timeout=self._timeout) # Just in case we need it later. self._sentinel_client = sentinel_client
def client(self): if not self._redis_connection: sentinel = Sentinel( self.sentinel_conf.get("sentinels"), min_other_sentinels=self.sentinel_conf.get("min_other_sentinels", 0), password=self.sentinel_conf.get("password"), socket_timeout=self.sentinel_conf.get("sentinel_timeout", None), ) redis_connection = sentinel.master_for( self.sentinel_conf.get("service_name"), Redis, socket_timeout=self.sentinel_conf.get("socket_timeout") ) self._redis_connection.append(redis_connection) return self._redis_connection[0]
def init(self): """ Called by Scheduler to say 'let's prepare yourself guy' """ logger.info("[RedisRetention] Initialization of the redis module") #self.return_queue = self.properties['from_queue'] if self.sentinel_servers: logger.info("[RedisRetention] sentinel_servers: %s" % str(self.sentinel_servers)) logger.info("[RedisRetention] redis_instance: %s" % str(self.redis_instance)) sentinel = Sentinel(self.sentinel_servers) self.mc = sentinel.master_for(self.redis_instance, socket_timeout=0.1) else: self.mc = redis.Redis(self.server)
def get(self): if self._config.sentinel_id: sentinel = Sentinel([(self._config.host, self._config.port)], socket_timeout=self._config.timeout) return sentinel.master_for(self._config.sentinel_id, socket_timeout=self._config.timeout, db=self._config.db, retry_on_timeout=True) else: conn = StrictRedis(host=self._config.host, port=self._config.port, db=self._config.db, retry_on_timeout=True, socket_timeout=self._config.timeout) return conn
def get_redis_connection(config, use_strict_redis=False): """ Returns a redis connection from a connection config """ redis_cls = redis.StrictRedis if use_strict_redis else redis.Redis if 'URL' in config: return redis_cls.from_url(config['URL'], db=config.get('DB')) if 'USE_REDIS_CACHE' in config.keys(): try: from django.core.cache import caches cache = caches[config['USE_REDIS_CACHE']] except ImportError: from django.core.cache import get_cache cache = get_cache(config['USE_REDIS_CACHE']) if hasattr(cache, 'client'): # We're using django-redis. The cache's `client` attribute # is a pluggable backend that return its Redis connection as # its `client` try: # To get Redis connection on django-redis >= 3.4.0 # we need to use cache.client.get_client() instead of # cache.client.client used in older versions try: return cache.client.get_client() except AttributeError: return cache.client.client except NotImplementedError: pass else: # We're using django-redis-cache try: return cache._client except AttributeError: # For django-redis-cache > 0.13.1 return cache.get_master_client() if 'UNIX_SOCKET_PATH' in config: return redis_cls(unix_socket_path=config['UNIX_SOCKET_PATH'], db=config['DB']) if 'SENTINELS' in config: sentinel = Sentinel(config['SENTINELS']) return sentinel.master_for( service_name=config['MASTER_NAME'], redis_class=redis_cls, db=config.get('DB'), password=config.get('PASSWORD'), socket_timeout=config.get('SOCKET_TIMEOUT') ) return redis_cls(host=config['HOST'], port=config['PORT'], db=config['DB'], password=config.get('PASSWORD'))
def __init__(self, sentinels, service, **kwargs): sentinel = Sentinel(sentinels, **kwargs) try: master = sentinel.discover_master(service) except MasterNotFoundError: raise try: slaves = sentinel.discover_slaves(service) except SlaveNotFoundError: self._slaves = None if version_info[0] < 3: super(AutoRedisSentinel, self).__init__(master, slaves=slaves, **kwargs) else: super().__init__(master, slaves=slaves, **kwargs)
def client(self): kwargs = { 'retry_on_timeout': False, 'socket_keepalive': None, 'socket_connect_timeout': 1, 'socket_timeout': self.socket_timeout or 1, # redis连接池要小于进程允许的最大连接数 'max_connections': 10240, } sentinel = Sentinel(self.sentinels, min_other_sentinels=self.min_other_sentinels, **kwargs) kwargs.update({ 'db': self.db, }) return sentinel.master_for(self.service_name, redis_class=Redis, **kwargs)
def run(self, terms, inject=None, **kwargs): values = [] terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject) for term in terms: params = self.parse_params(term) try: conn = Sentinel(params['server'], socket_timeout=1) res = conn.discover_master(params['name']) if res is None: res = "" values.append(":".join((str(v) for v in res))) except Exception, e: print(e)
def connect(self, index=0, write=True): """ Creates a redis connection with connection pool. """ master_name, sentinel_hosts, db = self.parse_connection_string(self._connection_string) sentinel_timeout = self._options.get("SENTINEL_TIMEOUT", 1) sentinel = Sentinel(sentinel_hosts, socket_timeout=sentinel_timeout) if write: host, port = sentinel.discover_master(master_name) else: host, port = random.choice([sentinel.discover_master(master_name)] + sentinel.discover_slaves(master_name)) return self.connection_factory.connect(host, port, db)