class RedisDriver: def __init__(self, config): self.config = config sentinels = [(c['ip'], c['port']) for c in self.config['sentinels']] self.sentinel = Sentinel(sentinels, socket_timeout=0.1, db=self.config['db'], decode_responses=True) self.__master = None self.__slave = None @property def master(self): if False: if not self.__master: self.__master = self.sentinel.master_for( self.config['name'], password=self.config.get('auth', '')) return self.__master else: return self.sentinel.master_for(self.config['name'], password=self.config.get( 'auth', '')) @property def slave(self): if False: if not self.__slave: self.__slave = self.sentinel.slave_for( self.config['name'], password=self.config.get('auth', '')) return self.__slave else: return self.sentinel.slave_for(self.config['name'], password=self.config.get( 'auth', ''))
class redisSentinelHelper(): def __init__(self,sentinel_list,service_name,db): self.sentinel = Sentinel(sentinel_list,socket_timeout=0.5) self.service_name = service_name self.db = db def get_master_redis(self): return self.sentinel.discover_master(self.service_name) def get_slave_redis(self): return self.sentinel.discover_slaves(self.service_name) def set_key(self,key,value): master = self.sentinel.master_for( service_name=self.service_name, socket_timeout=0.5, db=self.db ) return master.set(key,value) def set_hash_key(self,hkey,key,value): master = self.sentinel.master_for( service_name=self.service_name, socket_timeout=0.5, db=self.db ) return master.hset(hkey,key,value) def get_key(self,key): slave = self.sentinel.slave_for( service_name=self.service_name, socket_timeout=0.5, db=self.db ) return slave.get(key) def get_hash_key(self,hkey,hey): slave = self.sentinel.slave_for( service_name=self.service_name, socket_timeout=0.5, db=self.db ) return slave.hget(hkey,hey) def delete(self,key): master = self.sentinel.master_for( service_name=self.service_name, socket_timeout=0.5, db=self.db ) return master.delete(key)
class SentinelProxy: """Proxy for Redis sentinel.""" def __init__(self, sentinel_host, master_name, socket_timeout=0.1, **kwargs): """Initialize Redis sentinel connection. :params: sentinel_host: (host, port) """ self.sentinel = Sentinel([sentinel_host], socket_timeout=socket_timeout) self.master = self.sentinel.master_for(master_name, socket_timeout=socket_timeout, **kwargs) self.slave = self.sentinel.slave_for(master_name, socket_timeout=socket_timeout, **kwargs) def __getattr__(self, name): """Get attribute from Redis master or slave.""" master_key = ('set', 'hset', 'hmset', 'lset', 'lpush', 'blpop', 'brpop', 'rpush', 'expire', 'delete', 'incr') if name not in master_key: target = self.slave else: target = self.master return getattr(target, name)
def init_redis_client(cls): # 防止多线程竞争,获取到锁后依然可能已经初始化了 if cls.redis_objs: return cls.instance_name = conf.get('sentinel', "") cls.socket_timeout = conf.get("socket_timeout", 5) cls.connect_timeout = conf.get("connect_timeout", 0.1) if not cls.instance_name: # 单例模式 host = conf["host"] cls.redis_objs[cls.MODE_READ] = Redis( host=host.split(':')[0], port=host.split(':')[1], socket_timeout=cls.socket_timeout, socket_connect_timeout=cls.connect_timeout, retry_on_timeout=1 ) cls.redis_objs[cls.MODE_WRITE] = cls.redis_objs[cls.MODE_READ] else: # 哨兵模式 sentinel = Sentinel( cls.parse_config(), socket_timeout=cls.socket_timeout, socket_connect_timeout=cls.connect_timeout, retry_on_timeout=1 ) cls.redis_objs[cls.MODE_READ] = sentinel.slave_for(cls.instance_name) cls.redis_objs[cls.MODE_WRITE] = sentinel.master_for(cls.instance_name)
class RedisCluster(object): def __init__(self, sentinel_hosts, service_name, **connection_kwargs): self.service_name = service_name self.sentinel = Sentinel(sentinel_hosts, **connection_kwargs) self.master = None self.master_host_port = None self.reconfigure() self.sentinel_threads = {} self.listen() def reconfigure(self): try: self.master_host_port = self.sentinel.discover_master(self.service_name) self.master = self.sentinel.master_for(self.service_name) log.info(f"Reconfigured master to {self.master_host_port}") except Exception as ex: log.error(f"Error while reconfiguring. {ex.args[0]}") def listen(self): def on_new_master(workerThread): self.reconfigure() for sentinel in self.sentinel.sentinels: sentinel_host = sentinel.connection_pool.connection_kwargs["host"] self.sentinel_threads[sentinel_host] = MyPubSubWorkerThread( sentinel, on_new_master, msg_sleep_time=0.001, daemon=True ) self.sentinel_threads[sentinel_host].start()
def setup_rq_connection(): from redis.sentinel import Sentinel redis_url = current_app.config.get('RQ_SCHEDULER_DASHBOARD_REDIS_URL') redis_sentinels = current_app.config.get( 'RQ_SCHEDULER_DASHBOARD_REDIS_SENTINELS') if redis_url: current_app.redis_conn = from_url(redis_url) elif redis_sentinels: redis_master = current_app.config.get( 'RQ_SCHEDULER_DASHBOARD_REDIS_MASTER_NAME') password = current_app.config.get( 'RQ_SCHEDULER_DASHBOARD_REDIS_PASSWORD') db = current_app.config.get('RQ_SCHEDULER_DASHBOARD_REDIS_DB') sentinel_hosts = [ tuple(sentinel.split(':', 1)) for sentinel in redis_sentinels.split(',') ] sentinel = Sentinel(sentinel_hosts, db=db, password=password) current_app.redis_conn = sentinel.master_for(redis_master) else: current_app.redis_conn = Redis( host=current_app.config.get('REDIS_HOST'), port=current_app.config.get('REDIS_PORT'), password=current_app.config.get('REDIS_PASSWORD'), db=current_app.config.get('REDIS_DB'))
def redis_populate(): """Function to populate keys in Redis Server""" # client = redis.StrictRedis(host=configs["redis_host"], port=configs["redis_port"]) # sentinel = Sentinel( # [ # ("redis-cloudflare-node-0.redis-cloudflare-headless.dns-proxy.svc.cluster.local", 26379), # ("redis-cloudflare-node-1.redis-cloudflare-headless.dns-proxy.svc.cluster.local", 26379), # ("redis-cloudflare-node-2.redis-cloudflare-headless.dns-proxy.svc.cluster.local", 26379) # ], # socket_timeout=0.1) from redis.sentinel import Sentinel sentinel = Sentinel([ ('redis-cloudflare', 26379), ], sentinel_kwargs={'password': '******'}, password='******') sentinel.discover_master('mymaster') client = sentinel.master_for('mymaster', socket_timeout=0.5) i = 93218 key = 'key' + str(i) value = 'value' + str(i) result = client.get(key) print(result)
def run(name): sentinel = Sentinel([ ('10.5.5.194', 26379), ('10.5.5.155', 26379), ('10.5.5.128', 26379), ], socket_timeout=0.5) master = sentinel.master_for('mymaster', socket_timeout=0.5, password='******', db=0) slave = sentinel.slave_for('mymaster', socket_timeout=0.5, password='******', db=0) # 判断是否所传参数对应的value值是否为1 while True: time.sleep(0.05) if slave.get(name) == "0" or slave.get(name) is None: continue else: break master.set(name, "0") f_do = sys.argv[1] + '.do' f_dat = sys.argv[1] + '.dat' f = open(f_dat, 'w') f.write('0' + '\n') f.write('success' + '\n') f.write('set ' + name + '为0 \n') f.close() f = open(f_do, 'w') f.write('') f.close()
def _get_sentinel_connection(self, uri, kw): """ get sentinel connection details from _uri """ try: from redis.sentinel import Sentinel except ImportError: raise AnsibleError( "The 'redis' python module (version 2.9.0 or newer) is required to use redis sentinel." ) if ';' not in uri: raise AnsibleError('_uri does not have sentinel syntax.') # format: "localhost:26379;localhost2:26379;0:changeme" connections = uri.split(';') connection_args = connections.pop(-1) if len(connection_args) > 0: # hanle if no db nr is given connection_args = connection_args.split(':') kw['db'] = connection_args.pop(0) try: kw['password'] = connection_args.pop(0) except IndexError: pass # password is optional sentinels = [tuple(shost.split(':')) for shost in connections] display.vv('\nUsing redis sentinels: %s' % sentinels) scon = Sentinel(sentinels, **kw) try: return scon.master_for(self._sentinel_service_name, socket_timeout=0.2) except Exception as exc: raise AnsibleError('Could not connect to redis sentinel: %s' % to_native(exc))
def post(self, location): if location not in self.locations: return 'Unknown Location' if app.config['DEVEL']: pool = redis.ConnectionPool(host='localhost', port=6379) r = redis.Redis(connection_pool=pool) else: sentinel = Sentinel([('localhost', 26379)], socket_timeout=1) r = sentinel.master_for('mymaster', socket_timeout=1) beers = {key: r.hgetall(key) for key in r.keys('beer_{0}_*'.format(location))} for beername, beer in beers.items(): r.hset(beername, 'active', 'True' if beername in request.form.getlist('checks') else 'False') for beer in request.form.getlist('delete'): r.delete(beer) r.save() beers = [] for key in r.keys('beer_{0}_*'.format(location)): beer = convert(r.hgetall(key)) beer['beername'] = key beers.append(beer) beers.sort(key=operator.itemgetter('brewery', 'name', 'type')) return redirect(location)
def get(self, location): if location not in self.locations: return 'Unknown Location' colors = get_colors(location, self.config) priceinfo = get_priceinfo(location, self.config) if app.config['DEVEL']: pool = redis.ConnectionPool(host='localhost', port=6379) r = redis.Redis(connection_pool=pool) else: sentinel = Sentinel([('localhost', 26379)], socket_timeout=1) r = sentinel.master_for('mymaster', socket_timeout=1) beers = [] for key in r.keys('beer_{0}_*'.format(location)): beer = convert(r.hgetall(key)) beer['beername'] = key beers.append(beer) if 'items' in priceinfo: for beer in beers: beer['price'] = '$ {0}'.format(' /'.join([beer[p].replace('.0', '') for p in priceinfo['items'] if p in beer])) beers.sort(key=operator.itemgetter('brewery', 'name', 'type')) return render_template( 'edit.html', title='Beer List', beers=beers, location=location, colors=colors, roles=self.groups, locations=self.locations, priceinfo=priceinfo, scroll=False )
def add_queues_length_stats(self): if len(self.queues) == 0: return sentinel_list = [(node.get('host', 'localhost'), node.get('sentinel_port', 26379)) for node in self.node_list] sentinel = Sentinel(sentinel_list, socket_timeout=5) master_host = '' queues_length_map = {} for queue in self.queues: queues_length_map[queue] = -1 try: master_host, master_port = sentinel.discover_master( self.master_name) master_conn = sentinel.master_for(self.master_name, db=self.db, password=self.password) for queue in self.queues: try: queues_length_map[queue] = master_conn.llen(queue) except Exception: continue except Exception: pass for queue in self.queues: self.add_gauge_value('Redis_Queues/%s' % queue, None, queues_length_map[queue], count=1)
def get_redis_connection(config, use_strict_redis=False): """ Returns a redis connection from a connection config """ redis_cls = redis.StrictRedis if use_strict_redis else redis.Redis if 'URL' in config: return redis_cls.from_url(config['URL'], db=config.get('DB')) if 'USE_REDIS_CACHE' in config.keys(): try: from django.core.cache import caches cache = caches[config['USE_REDIS_CACHE']] except ImportError: from django.core.cache import get_cache cache = get_cache(config['USE_REDIS_CACHE']) if hasattr(cache, 'client'): # We're using django-redis. The cache's `client` attribute # is a pluggable backend that return its Redis connection as # its `client` try: # To get Redis connection on django-redis >= 3.4.0 # we need to use cache.client.get_client() instead of # cache.client.client used in older versions try: return cache.client.get_client() except AttributeError: return cache.client.client except NotImplementedError: pass else: # We're using django-redis-cache try: return cache._client except AttributeError: # For django-redis-cache > 0.13.1 return cache.get_master_client() if 'UNIX_SOCKET_PATH' in config: return redis_cls(unix_socket_path=config['UNIX_SOCKET_PATH'], db=config['DB']) if 'SENTINELS' in config: sentinel_kwargs = { 'db': config.get('DB'), 'password': config.get('PASSWORD'), 'socket_timeout': config.get('SOCKET_TIMEOUT'), } sentinel_kwargs.update(config.get('CONNECTION_KWARGS', {})) sentinel = Sentinel(config['SENTINELS'], **sentinel_kwargs) return sentinel.master_for( service_name=config['MASTER_NAME'], redis_class=redis_cls, ) return redis_cls(host=config['HOST'], port=config['PORT'], db=config['DB'], password=config.get('PASSWORD'))
class TestApplication(Application): def __init__(self): # Call to the superclass to bootstrap. super(TestApplication, self).__init__(name='krux-redis-sentinel') self.logger.debug('Parsing sentinels from args: %r', self.args.sentinel) sentinels = hostports_from_args(self.args) self.logger.debug('Parsed sentinel host, port pairs: %r', sentinels) self.logger.debug('Initializing Sentinel instance...') self.sentinel = Sentinel(sentinels, socket_timeout=SOCKET_TIMEOUT) self.logger.debug('Initialized Sentinel instance: %r', self.sentinel) def add_cli_arguments(self, parser): add_sentinel_cli_arguments(parser) def run(self): master = self.sentinel.master_for(self.name) slave = self.sentinel.slave_for(self.name) try: self.logger.info('Ping master: %s', master.ping()) except ConnectionError as err: self.logger.warning('Could not connect to master!') self.logger.error(err) try: self.logger.info('Ping slave: %s', slave.ping()) except ConnectionError as err: self.logger.warning('Could not connect to slave!') self.logger.error(err)
def __init__(self, connection_str, db=0, project=None, service=None, host=None, conf=cfg.CONF, **kwargs): """Redis driver for OSProfiler.""" super(RedisSentinel, self).__init__(connection_str, project=project, service=service, host=host, conf=conf, **kwargs) try: from redis.sentinel import Sentinel except ImportError: raise exc.CommandError("To use this command, you should install " "'redis' manually. Use command:\n " "'pip install redis'.") self.conf = conf socket_timeout = self.conf.profiler.socket_timeout parsed_url = parser.urlparse(self.connection_str) sentinel = Sentinel([(parsed_url.hostname, int(parsed_url.port))], password=parsed_url.password, socket_timeout=socket_timeout) self.db = sentinel.master_for(self.conf.profiler.sentinel_service_name, socket_timeout=socket_timeout)
def get_connection(self, is_read_only=False) -> redis.StrictRedis: """ Gets a StrictRedis connection for normal redis or for redis sentinel based upon redis mode in configuration. :type is_read_only: bool :param is_read_only: In case of redis sentinel, it returns connection to slave :return: Returns a StrictRedis connection """ if self.connection is not None: return self.connection if self.is_sentinel: kwargs = dict() if self.password: kwargs["password"] = self.password sentinel = Sentinel([(self.host, self.port)], **kwargs) if is_read_only: connection = sentinel.slave_for(self.sentinel_service, decode_responses=True) else: connection = sentinel.master_for(self.sentinel_service, decode_responses=True) else: connection = redis.StrictRedis(host=self.host, port=self.port, decode_responses=True, password=self.password) self.connection = connection return connection
def get_redis(**kwargs): """Returns a redis client instance. Parameters ---------- redis_cls : class, optional Defaults to ``redis.StrictRedis``. url : str, optional If given, ``redis_cls.from_url`` is used to instantiate the class. **kwargs Extra parameters to be passed to the ``redis_cls`` class. Returns ------- server Redis client instance. """ sentiels = kwargs.pop('sentinels', None) if sentiels: from redis.sentinel import Sentinel master_name = kwargs.pop('master_name') sentiel = Sentinel(sentiels, **kwargs) master = sentiel.master_for(master_name) return master else: redis_cls = kwargs.pop('redis_cls', defaults.REDIS_CLS) url = kwargs.pop('url', None) if url: return redis_cls.from_url(url, **kwargs) else: return redis_cls(**kwargs)
class RedisSentinelMessageBroker(RedisMessageBroker): """ Redis Sentinel Message broker. """ def __init__(self, **kwargs): self.init_sentinel(**kwargs) RedisMessageBroker.__init__(self, **kwargs) def init_sentinel(self, **kwargs): if not kwargs.has_key('sentinel_servers'): raise ConfigurationError('Kewyword sentinel_servers is missing.') if not kwargs.has_key('sentinel_master'): raise ConfigurationError('Kewyword sentinel_master is missing.') # Enable sentinel support self.__sentinel_servers = kwargs['sentinel_servers'] self.__sentinel_master = kwargs['sentinel_master'] del kwargs['sentinel_servers'] del kwargs['sentinel_master'] self.__sentinel = Sentinel(self.__sentinel_servers, **kwargs) def configure_redis(self, **kwargs): return self.__sentinel.master_for(self.__sentinel_master) def get_connection_port(self): return self.__sentinel.discover_master(self.__sentinel_master)[1] def get_connection_host(self): return self.__sentinel.discover_master(self.__sentinel_master)[0]
def __create_redis_client(self, cfg_params, addr): new_sentinel = None new_redis = None if cfg_params.db_sentinel_port is None: new_redis = Redis(host=addr, port=cfg_params.db_port, db=0, max_connections=20) else: sentinel_node = (addr, cfg_params.db_sentinel_port) master_name = cfg_params.db_sentinel_master_name new_sentinel = Sentinel([sentinel_node]) new_redis = new_sentinel.master_for(master_name) new_redis.set_response_callback( 'SETIE', lambda r: r and nativestr(r) == 'OK' or False) new_redis.set_response_callback('DELIE', lambda r: r and int(r) == 1 or False) redis_pubsub = PubSub(self.event_separator, new_redis.connection_pool, ignore_subscribe_messages=True) pubsub_thread = threading.Thread(target=None) run_in_thread = False return _RedisConn(new_redis, redis_pubsub, pubsub_thread, run_in_thread)
def __init__(self): self.kafka_consumer = Consumer({ 'bootstrap.servers': KAFKA_SERVERS, 'group.id': KAFKA_MESSAGE_CONSUMER_GROUP, 'session.timeout.ms': 6000, 'auto.offset.reset': 'latest' }) self.kafka_consumer.subscribe([KAFKA_MESSAGE_TOPIC]) LOG.info('Connected with Kafka Consumer') self.kafka_producer = Producer({'bootstrap.servers': KAFKA_SERVERS}) LOG.info('Connected with Kafka Producer') hosts_sentinel = [] endpoint = DBAAS_SENTINEL_ENDPOINT_SIMPLE.replace("sentinel://", "") for host in endpoint.split(','): hosts_sentinel.append(tuple(host.split(':'))) sentinel = Sentinel(hosts_sentinel, socket_timeout=REDIS_SOCKET_TIMEOUT) self.rediscon = sentinel.master_for( DBAAS_SENTINEL_SERVICE_NAME, socket_timeout=REDIS_SOCKET_TIMEOUT, password=DBAAS_SENTINEL_PASSWORD) LOG.info('Connected with Redis Database')
def sentinel_connect(self, master_name): url = urlparse.urlparse(self.schedule_url) def parse_host(s): if ':' in s: host, port = s.split(':', 1) port = int(port) else: host = s port = 26379 return host, port if '@' in url.netloc: auth, hostspec = url.netloc.split('@', 1) else: auth = None hostspec = url.netloc if auth and ':' in auth: _, password = auth.split(':', 1) else: password = None path = url.path if path.startswith('/'): path = path[1:] hosts = [parse_host(s) for s in hostspec.split(',')] sentinel = Sentinel(hosts, password=password, db=path) master = sentinel.master_for(master_name) return master
def get_redis_from_config(settings, connection_class=Redis): """Returns a StrictRedis instance from a dictionary of settings. To use redis sentinel, you must specify a dictionary in the configuration file. Example of a dictionary with keys without values: SENTINEL: {'INSTANCES':, 'SOCKET_TIMEOUT':, 'PASSWORD':,'DB':, 'MASTER_NAME':} """ if settings.get('REDIS_URL') is not None: return connection_class.from_url(settings['REDIS_URL']) elif settings.get('SENTINEL') is not None: instances = settings['SENTINEL'].get('INSTANCES', [('localhost', 26379)]) socket_timeout = settings['SENTINEL'].get('SOCKET_TIMEOUT', None) password = settings['SENTINEL'].get('PASSWORD', None) db = settings['SENTINEL'].get('DB', 0) master_name = settings['SENTINEL'].get('MASTER_NAME', 'mymaster') sn = Sentinel(instances, socket_timeout=socket_timeout, password=password, db=db) return sn.master_for(master_name) kwargs = { 'host': settings.get('REDIS_HOST', 'localhost'), 'port': settings.get('REDIS_PORT', 6379), 'db': settings.get('REDIS_DB', 0), 'password': settings.get('REDIS_PASSWORD', None), 'ssl': settings.get('REDIS_SSL', False), } return connection_class(**kwargs)
def redis_client(): if settings.CACHEOPS_REDIS and settings.CACHEOPS_SENTINEL: raise ImproperlyConfigured("CACHEOPS_REDIS and CACHEOPS_SENTINEL are mutually exclusive") client_class = CacheopsRedis if settings.CACHEOPS_CLIENT_CLASS: client_class = import_string(settings.CACHEOPS_CLIENT_CLASS) if settings.CACHEOPS_SENTINEL: if not {'locations', 'service_name'} <= set(settings.CACHEOPS_SENTINEL): raise ImproperlyConfigured("Specify locations and service_name for CACHEOPS_SENTINEL") sentinel = Sentinel( settings.CACHEOPS_SENTINEL['locations'], **omit(settings.CACHEOPS_SENTINEL, ('locations', 'service_name', 'db'))) return sentinel.master_for( settings.CACHEOPS_SENTINEL['service_name'], redis_class=client_class, db=settings.CACHEOPS_SENTINEL.get('db', 0) ) # Allow client connection settings to be specified by a URL. if isinstance(settings.CACHEOPS_REDIS, str): return client_class.from_url(settings.CACHEOPS_REDIS) else: return client_class(**settings.CACHEOPS_REDIS)
def get_redis(app=None): app = app_or_default(app) conf = ensure_conf(app) if not hasattr(app, 'redbeat_redis') or app.redbeat_redis is None: redis_options = conf.app.conf.get( 'REDBEAT_REDIS_OPTIONS', conf.app.conf.get('BROKER_TRANSPORT_OPTIONS', {})) retry_period = redis_options.get('retry_period') if conf.redis_url.startswith( 'redis-sentinel') and 'sentinels' in redis_options: from redis.sentinel import Sentinel sentinel = Sentinel( redis_options['sentinels'], socket_timeout=redis_options.get('socket_timeout'), password=redis_options.get('password'), decode_responses=True) connection = sentinel.master_for( redis_options.get('service_name', 'master')) else: connection = StrictRedis.from_url(conf.redis_url, decode_responses=True) if retry_period is None: app.redbeat_redis = connection else: app.redbeat_redis = RetryingConnection(retry_period, connection) return app.redbeat_redis
def get_redis_connection(config, is_master=True): """ Returns redis connection basing on configuration dict Args: config: Dict with configuration, if key `SENTINELS` is present sentinel connection pool will be used. is_master: Whether connection should be master or slave, applies only if sentinels are used Returns: StrictRedis connection """ if 'SENTINELS' in config: sentinel = Sentinel( config['SENTINELS'], db=config.get('DB'), password=config.get('PASSWORD'), socket_timeout=config.get('TIMEOUT'), socket_connect_timeout=config.get('CONNECT_TIMEOUT'), ) if is_master: return sentinel.master_for(settings.REDIS_CLUSTER_NAME) else: return sentinel.slave_for(settings.REDIS_CLUSTER_NAME) else: return StrictRedis( host=config['HOST'], port=config['PORT'], db=config.get('DB'), password=config.get('PASSWORD'), socket_timeout=config.get('TIMEOUT'), socket_connect_timeout=config.get('CONNECT_TIMEOUT'), )
def get_connection(self, is_read_only=False) -> redis.Redis: """ Gets a Redis connection for normal redis or for redis sentinel based upon redis mode in configuration. :type is_read_only: bool :param is_read_only: In case of redis sentinel, it returns connection to slave :return: Returns a Redis connection """ if self.connection is not None: return self.connection if self.is_sentinel: kwargs = dict() if self.password: kwargs["password"] = self.password sentinel = Sentinel([(self.host, self.port)], **kwargs) if is_read_only: connection = sentinel.slave_for(self.sentinel_service, decode_responses=True) else: connection = sentinel.master_for(self.sentinel_service, decode_responses=True) else: connection = redis.Redis(host=self.host, port=self.port, decode_responses=True, password=self.password) self.connection = connection return connection
def __redis_client__(self, instance): try: LOG.debug('Connecting to redis databaseinfra %s', self.databaseinfra) # redis uses timeout in seconds connection_timeout_in_seconds = Configuration.get_by_name_as_int('redis_connect_timeout', default=REDIS_CONNECTION_DEFAULT_TIMEOUT) if (instance and instance.instance_type == Instance.REDIS) or (not self.databaseinfra.plan.is_ha and not instance): connection_address, connection_port = self.__get_admin_single_connection(instance) client = redis.Redis(host = connection_address, port = int(connection_port), password = self.databaseinfra.password, socket_connect_timeout = connection_timeout_in_seconds) else: sentinels = self.__get_admin_sentinel_connection(instance) sentinel = Sentinel(sentinels, socket_timeout=connection_timeout_in_seconds) client = sentinel.master_for(self.databaseinfra.name, socket_timeout=connection_timeout_in_seconds, password = self.databaseinfra.password) LOG.debug('Successfully connected to redis databaseinfra %s' % (self.databaseinfra)) return client except Exception, e: raise e
def __init__(self): """ init setting and create redis instance """ settings = Settings.get("redis") self.prefix = settings.get("prefix") self.queue_max_len = settings.get("queue_max_len") self.sentinel = settings.get("sentinel") if self.sentinel: sentinel = Sentinel(sentinels=map(lambda x: x.split(":"), settings.get("sentinel_hosts"))) kwargs = dict( service_name=settings.get("sentinel_master"), password=settings.get("password"), decode_responses=True, ) self.master = sentinel.master_for(**kwargs) self.slave = sentinel.slave_for(**kwargs) else: pool = redis.ConnectionPool( host=settings.get("host"), port=settings.get("port"), db=settings.get("db"), password=settings.get("password"), decode_responses=True, ) self.master = self.slave = redis.StrictRedis(connection_pool=pool)
def redis_populate(): """Function to populate keys in Redis Server""" # client = redis.StrictRedis(host=configs["redis_host"], port=configs["redis_port"]) # sentinel = Sentinel( # [ # ("redis-cloudflare-node-0.redis-cloudflare-headless.dns-proxy.svc.cluster.local", 26379), # ("redis-cloudflare-node-1.redis-cloudflare-headless.dns-proxy.svc.cluster.local", 26379), # ("redis-cloudflare-node-2.redis-cloudflare-headless.dns-proxy.svc.cluster.local", 26379) # ], # socket_timeout=0.1) from redis.sentinel import Sentinel sentinel = Sentinel([ ('redis-cloudflare', 26379), ], sentinel_kwargs={'password': '******'}, password='******') sentinel.discover_master('mymaster') client = sentinel.master_for('mymaster', socket_timeout=0.5) for i in range(1000000): key = 'key' + str(i) # value='value'+str(i) value = str( i ) + '::: https://www.google.co.uk/#sclient=psy-ab&hl=en&source=hp&q=ASUSTeK+Computer+INC.+Model+M4A78LT-M+manual&pbx=1&oq=ASUSTeK+Computer+INC.+Model+M4A78LT-M+manual&aq=f&aqi=&aql=&gs_sm=3&gs_upl=52765l57528l0l57848l8l8l0l0l0l0l2413l3989l8-1.1l2l0&bav=on.2,or.r_gc.r_pw.,cf.osb&fp=3d6c1d1d0a5ea45f&biw=1262&bih=879' client.set(key, value) print(key, value)
def setup_rq_connection(): # It's the only place where we can safely define default value for web background # since It is used in template current_app.config.setdefault('RQ_DASHBOARD_WEB_BACKGROUND', 'black') # we need to do It here instead of cli, since It may be embeded upgrade_config(current_app) # Getting Redis connection parameters for RQ redis_url = current_app.config.get('RQ_DASHBOARD_REDIS_URL') redis_sentinels = current_app.config.get('RQ_DASHBOARD_REDIS_SENTINELS') if isinstance(redis_url, list): current_app.redis_conn = from_url(redis_url[0]) elif isinstance(redis_url, string_types): current_app.redis_conn = from_url(redis_url) elif redis_sentinels: redis_master = current_app.config.get('RQ_DASHBOARD_REDIS_MASTER_NAME') password = current_app.config.get('RQ_DASHBOARD_REDIS_PASSWORD') db = current_app.config.get('RQ_DASHBOARD_REDIS_DB') sentinel_hosts = [tuple(sentinel.split(':', 1)) for sentinel in redis_sentinels.split(',')] sentinel = Sentinel(sentinel_hosts, db=db, password=password) current_app.redis_conn = sentinel.master_for(redis_master) else: current_app.redis_conn = Redis( host=current_app.config.get('RQ_DASHBOARD_REDIS_HOST', 'localhost'), port=current_app.config.get('RQ_DASHBOARD_REDIS_PORT', 6379), password=current_app.config.get('RQ_DASHBOARD_REDIS_PASSWORD'), db=current_app.config.get('RQ_DASHBOARD_REDIS_DB', 0), )
def get_db(): redis_store = getattr(g, '_database', None) if redis_store is None: if 'FAKE_DB' in app.config and app.config['FAKE_DB']: from mockredis import mock_redis_client redis_store = g._database = mock_redis_client( encoding='utf-8', decode_responses=True) else: if app.config['REDIS_URL'].startswith('redis://'): redis_store = g._database = Redis.from_url( app.config['REDIS_URL'], encoding='utf-8', decode_responses=True) elif app.config['REDIS_URL'].startswith('sentinel://'): parsed_url = urlparse(app.config['REDIS_URL']) service = parsed_url.path.lstrip('/') port = 26379 if ':' in parsed_url.netloc: host, str_port = parsed_url.netloc.split(':') port = int(str_port) else: host = parsed_url.netloc sentinel = Sentinel([(host, port)], socket_timeout=0.1) redis_store = sentinel.master_for(service, redis_class=Redis, socket_timeout=0.1) return redis_store
def redis_client(): if settings.CACHEOPS_REDIS and settings.CACHEOPS_SENTINEL: raise ImproperlyConfigured("CACHEOPS_REDIS and CACHEOPS_SENTINEL are mutually exclusive") client_class = CacheopsRedis if settings.CACHEOPS_CLIENT_CLASS: client_class = import_string(settings.CACHEOPS_CLIENT_CLASS) if settings.CACHEOPS_SENTINEL: if not {'locations', 'service_name'} <= set(settings.CACHEOPS_SENTINEL): raise ImproperlyConfigured("Specify locations and service_name for CACHEOPS_SENTINEL") sentinel = Sentinel( settings.CACHEOPS_SENTINEL['locations'], **omit(settings.CACHEOPS_SENTINEL, ('locations', 'service_name', 'db'))) return sentinel.master_for( settings.CACHEOPS_SENTINEL['service_name'], redis_class=client_class, db=settings.CACHEOPS_SENTINEL.get('db', 0) ) # Allow client connection settings to be specified by a URL. if isinstance(settings.CACHEOPS_REDIS, six.string_types): return client_class.from_url(settings.CACHEOPS_REDIS) else: return client_class(**settings.CACHEOPS_REDIS)
def connect2redis(host, port=6379, db=0, connectionPool=True, logging=None, sentinel_service=prms.REDIS_SENTINEL_SERVICE_NAME, master=True): if sentinel_service: retry_count = 0 while retry_count < 10: sentinel = Sentinel([(host, 26379)]) if master: client = sentinel.master_for(sentinel_service) else: client = sentinel.slave_for(sentinel_service) if is_available(client, logging=logging): logging.info("Connected to Redis succesfully.") return client else: print(f"Waiting for {pow(2, retry_count)} secs.") time.sleep(pow(2, retry_count)) retry_count += 1 else: retry_count = 0 while retry_count < 10: if connectionPool: pool = redis.ConnectionPool(host=host, port=port, db=db) client = redis.StrictRedis(connection_pool=pool) else: client = redis.StrictRedis(host=host, port=port, db=db) if is_available(client, logging=logging): logging.info("Connected to Redis succesfully.") return client else: print(f"Waiting for {pow(2, retry_count)} secs.") time.sleep(pow(2, retry_count)) retry_count += 1 logging.error("Exiting...") exit(3)
def run(skip=0, stop=9999999): cfg = yaml.load(open('config.yaml', 'r')) sentinels = [(c['ip'], c['port']) for c in cfg['cache']] sentinel = Sentinel(sentinels, socket_timeout=0.1, db=1, decode_responses=True) master = sentinel.master_for('madeira') stream = open('clean.csv', 'r') n = 0 for l in stream: if n < skip: print('SKIP %d' % n) n += 1 continue if n >= stop: print('STOP AT %d' % n) break # Q2014120613471910003652,13413352219,10,cmcc order_id, mobile, price, routing = l.strip().split(',') print(order_id, mobile, price, routing) partner = cfg['upstream'][routing] order_id = 'T' + order_id[1:] call_cmcc(master, mobile, price, order_id, partner) n += 1
def create_app(): '''工厂方法''' # 创建app对象 app = Flask(__name__) # 加载配置 config = config_dict.get(app.config.get('ENV')) app.config.from_object(config) # 加载数据库配置 from common.models import db db.init_app(app) # 记录日志信息 log_file(config.LOG_LV) # redis集群加载 from rediscluster import RedisCluster app.redis_cluster = RedisCluster( startup_nodes=app.config.get('REDIS_CLUSTER')) # redis哨兵,主从配置 from redis.sentinel import Sentinel sentinel = Sentinel(app.config.get('REDIS_SENTINELS')) app.redis_master = sentinel.master_for( app.config.get('REDIS_SENTINEL_SERVICES_NAME')) app.redis_slave = sentinel.slave_for( app.config.get('REDIS_SENTINEL_SERVICES_NAME')) # 注册用户蓝图到app中 from .resources.users import user_blue app.register_blueprint(user_blue) return app
def _init(self, server, params): super(CacheClass, self).__init__(params) self._server = server self._params = params self.service_name = self.options.get("SERVICE_NAME", 'mymaster') self.slave_read = self.options.get("READ_SLAVE", False) if self.server: def _get_hosttupe(conn): host_lst = conn.rsplit(':', 1) return host_lst[0], int(host_lst[1]) conn_list = [_get_hosttupe(conn) for conn in self.server] else: raise ImproperlyConfigured("sentinel server config error.") kwargs = self.connection_pool_class_kwargs max_connections = kwargs.pop('max_connections') sentinel_manager = Sentinel(conn_list, sentinel_kwargs=kwargs) kwargs.update({ 'db': self.db, 'password': self.password, 'max_connections': max_connections }) self._client = sentinel_manager.master_for(self.service_name, connection_pool_class=self.connection_pool_class, **kwargs) if self.slave_read: self._slave_client = sentinel_manager.slave_for(self.service_name, connection_pool_class=self.connection_pool_class, **kwargs)
def run(): try: logging.info('Calculating training booking reminders') if SENTINEL_HOST is None: client = redis.StrictRedis(host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD) else: sentinel = Sentinel([(SENTINEL_HOST, SENTINEL_PORT)]) client = sentinel.master_for(SENTINEL_MASTER) # When was the last time we notified for? Default to now last_not_str = client.get(WEEK_LAST_CHECK) now = datetime.datetime.now() last_notified = now if last_not_str is not None: last_notified_date = dateutil.parser.parse(last_not_str) # Check that this is after the current date if last_notified_date > now: last_notified = last_notified_date # Now work out the date that is midnight one week and a day from now one_week = datetime.datetime.combine(datetime.date.today() + datetime.timedelta(8), datetime.time()) logging.info('Getting sessions from %s until %s' % (str(last_notified), str(one_week))) if one_week <= last_notified: return # Find all the sessions that fall bewtween the last notified date and a weeks time sessions = TrainingSession.query({ 'find': { 'start': { '$gte': last_notified, '$lt': one_week } } }) # Notify that these need reminding for session in sessions: data = { 'name': 'reminder', 'data': { 'type': 'trainingSession', 'data': session._id } } client.rpush(EVENT_Q, json.dumps(data)) if (len(sessions) > 0): client.publish(EVENT_CHAN, 'event') logging.info('Published training booking events') # Store that we've notified client.set(WEEK_LAST_CHECK, one_week.isoformat()) logging.info('Finished training booking') except Exception as ex: logging.exception(ex)
def check_sso_auth(user_id, brand_id, trans_type ): sentinel = Sentinel([('120.76.154.208', 26379)], socket_timeout=0.5) log.d(sentinel.discover_master('mymaster')) redis_obj = sentinel.master_for('mymaster', socket_timeout=0.1) db_obj = PostgreSQLConnector() sso_cls = SSOAuth(redis_obj,db_obj) return sso_cls.check_auth(user_id, brand_id, trans_type)
def sso_log(user_id, brand_id, trans_type, data_type,data_content,log_time): sentinel = Sentinel([('112.74.198.224', 26379)], socket_timeout=0.5) log.d(sentinel.discover_master('mymaster')) redis_obj = sentinel.master_for('mymaster', socket_timeout=0.1) db_obj = PostgreSQLConnector() sso_cls = SSOAuth(redis_obj,db_obj) return sso_cls.send_log(user_id, brand_id, trans_type, data_type,data_content,log_time)
def redis_conn(readonly=True): global sentinel, master_client, slave_client if not sentinel: sentinel = Sentinel(settings.REDIS_SENTINEL, socket_timeout=1) if not master_client: master_client = sentinel.master_for(settings.REDIS_CLUSTER) if not slave_client: slave_client = sentinel.slave_for(settings.REDIS_CLUSTER) return slave_client if readonly else master_client
def setUp(self): sentinel = Sentinel(settings_test.REDIS_SENTINEL) db = getattr(settings_test, 'REDIS_DB', 0) self.connection = sentinel.master_for('mymaster', db=db) self.connection.flushall() self.guard = ContributionsGuard(self.connection) self.anon_user = {'user_id': None, 'user_ip': '127.0.0.1'} self.auth_user = {'user_id': 33, 'user_ip': None} self.task = Task(id=22)
def client(self): sentinel = Sentinel( self.sentinel_conf.get('sentinels'), min_other_sentinels=self.sentinel_conf.get("min_other_sentinels", 0), password=self.sentinel_conf.get("password"), sentinel_kwargs={"socket_timeout": self.sentinel_conf.get("sentinel_timeout")}, ) return sentinel.master_for(self.sentinel_conf.get("service_name"), Redis, socket_timeout=self.sentinel_conf.get("socket_timeout"))
def __init__(self, **redis_conf): self.prefix = 'q' sentinels = redis_conf.pop('sentinels', '()') sentinels_service_name = redis_conf.pop('sentinels_service_name', 'mymaster') min_other_sentinels = redis_conf.pop('min_other_sentinels', 0) s = Sentinel(sentinels, **redis_conf) self.db = s.master_for(sentinels_service_name) #self.db = StrictRedis(**redis_conf) self.key = '%s:%s' % (self.prefix, '%s')
def _sentinel_managed_pool(self): sentinel = Sentinel( self.sentinels, min_other_sentinels=getattr(self, "min_other_sentinels", 0), password=getattr(self, "password", None), sentinel_kwargs={"socket_timeout": getattr(self, "sentinel_timeout", None)}, ) return sentinel.master_for(self.service_name, self.Client, socket_timeout=self.socket_timeout).connection_pool
class SentinelBackedRingClient(RingClient): def __init__(self, hosts, options): sentinel_kwargs = self._get_sentinel_kwargs(options) node_kwargs = self._get_node_kwargs(options) masters = None # Try to fetch a list of all masters from any sentinel. hosts = list(hosts) shuffle(hosts) # Randomly sort sentinels before trying to bootstrap. for host, port in hosts: client = StrictRedis(host=host, port=port, **sentinel_kwargs) try: masters = client.sentinel_masters().keys() break except RedisError: pass if masters is None: # No Sentinel responded successfully? raise errors.MastersListUnavailable if not len(masters): # The masters list was empty? raise errors.NoMastersConfigured sentinel_kwargs.update({ # Sentinels connected to fewer sentinels than `MIN_SENTINELS` will # be ignored. 'min_other_sentinels': options.get('MIN_SENTINELS', len(hosts) / 2), }) self.sentinel = Sentinel(hosts, **sentinel_kwargs) masters = [self.sentinel.master_for(name, **node_kwargs) for name in masters] super(SentinelBackedRingClient, self).__init__(masters, options) def _get_sentinel_kwargs(self, options): password = options.get('SENTINEL_PASSWORD') try: socket_timeout = float(options.get('SOCKET_TIMEOUT', 0.2)) except ValueError: raise ImproperlyConfigured('`SOCKET_TIMEOUT` must be a valid number type.') return { 'password': password, 'socket_timeout': socket_timeout } def disconnect(self): for node in self.sentinel.sentinels: try: node.connection_pool.disconnect() except AttributeError: # TODO(usmanm): Figure this shit out. pass super(SentinelBackedRingClient, self).disconnect()
def init_shards_from_sentinel(self): sentinel_client = Sentinel( [(h, 8422) for h in self._sentinels], socket_timeout=self._timeout) # Connect to all the shards with the names specified per # shard_name_format. The names are important since it's getting # the instances from Redis Sentinel. for shard_num in range(0, self.num_shards()): shard_name = self.get_shard_name(shard_num) self._shards[shard_num] = sentinel_client.master_for( shard_name, socket_timeout=self._timeout) # Just in case we need it later. self._sentinel_client = sentinel_client
def client(self): if not self._redis_connection: sentinel = Sentinel( self.sentinel_conf.get("sentinels"), min_other_sentinels=self.sentinel_conf.get("min_other_sentinels", 0), password=self.sentinel_conf.get("password"), socket_timeout=self.sentinel_conf.get("sentinel_timeout", None), ) redis_connection = sentinel.master_for( self.sentinel_conf.get("service_name"), Redis, socket_timeout=self.sentinel_conf.get("socket_timeout") ) self._redis_connection.append(redis_connection) return self._redis_connection[0]
def init(self): """ Called by Scheduler to say 'let's prepare yourself guy' """ logger.info("[RedisRetention] Initialization of the redis module") #self.return_queue = self.properties['from_queue'] if self.sentinel_servers: logger.info("[RedisRetention] sentinel_servers: %s" % str(self.sentinel_servers)) logger.info("[RedisRetention] redis_instance: %s" % str(self.redis_instance)) sentinel = Sentinel(self.sentinel_servers) self.mc = sentinel.master_for(self.redis_instance, socket_timeout=0.1) else: self.mc = redis.Redis(self.server)
def client(self): kwargs = { 'retry_on_timeout': False, 'socket_keepalive': None, 'socket_connect_timeout': 1, 'socket_timeout': self.socket_timeout or 1, # redis连接池要小于进程允许的最大连接数 'max_connections': 10240, } sentinel = Sentinel(self.sentinels, min_other_sentinels=self.min_other_sentinels, **kwargs) kwargs.update({ 'db': self.db, }) return sentinel.master_for(self.service_name, redis_class=Redis, **kwargs)
def get_redis_connection(config, use_strict_redis=False): """ Returns a redis connection from a connection config """ redis_cls = redis.StrictRedis if use_strict_redis else redis.Redis if 'URL' in config: return redis_cls.from_url(config['URL'], db=config.get('DB')) if 'USE_REDIS_CACHE' in config.keys(): try: from django.core.cache import caches cache = caches[config['USE_REDIS_CACHE']] except ImportError: from django.core.cache import get_cache cache = get_cache(config['USE_REDIS_CACHE']) if hasattr(cache, 'client'): # We're using django-redis. The cache's `client` attribute # is a pluggable backend that return its Redis connection as # its `client` try: # To get Redis connection on django-redis >= 3.4.0 # we need to use cache.client.get_client() instead of # cache.client.client used in older versions try: return cache.client.get_client() except AttributeError: return cache.client.client except NotImplementedError: pass else: # We're using django-redis-cache try: return cache._client except AttributeError: # For django-redis-cache > 0.13.1 return cache.get_master_client() if 'UNIX_SOCKET_PATH' in config: return redis_cls(unix_socket_path=config['UNIX_SOCKET_PATH'], db=config['DB']) if 'SENTINELS' in config: sentinel = Sentinel(config['SENTINELS']) return sentinel.master_for( service_name=config['MASTER_NAME'], redis_class=redis_cls, db=config.get('DB'), password=config.get('PASSWORD'), socket_timeout=config.get('SOCKET_TIMEOUT') ) return redis_cls(host=config['HOST'], port=config['PORT'], db=config['DB'], password=config.get('PASSWORD'))
def get(self): if self._config.sentinel_id: sentinel = Sentinel([(self._config.host, self._config.port)], socket_timeout=self._config.timeout) return sentinel.master_for(self._config.sentinel_id, socket_timeout=self._config.timeout, db=self._config.db, retry_on_timeout=True) else: conn = StrictRedis(host=self._config.host, port=self._config.port, db=self._config.db, retry_on_timeout=True, socket_timeout=self._config.timeout) return conn
def put(self, location): if location not in self.locations: return 'Unknown Location' form = BeerForm() beer = self._beer(form, location) if app.config['DEVEL']: pool = redis.ConnectionPool(host='localhost', port=6379) r = redis.Redis(connection_pool=pool) else: sentinel = Sentinel([('localhost', 26379)], socket_timeout=1) r = sentinel.master_for('mymaster', socket_timeout=1) r.hmset(request.args.get('name'), beer) r.save() return redirect('/{0}/entry'.format(location))
def wrapper(*args, **kwargs): if os.environ.get('PYBOSSA_REDIS_CACHE_DISABLED') is None: key = "%s::%s" % (settings.REDIS_KEYPREFIX, key_prefix) sentinel = Sentinel(settings.REDIS_SENTINEL, socket_timeout=0.1) master = sentinel.master_for('mymaster') slave = sentinel.slave_for('mymaster') output = slave.get(key) if output: return pickle.loads(output) else: output = f(*args, **kwargs) master.setex(key, timeout, pickle.dumps(output)) return output else: return f(*args, **kwargs)
def get_redis_cli(): global _redis_cli if not _redis_cli and is_redis_sentinel(): sentinel = Sentinel([(get_redis_host(), get_redis_port())], socket_timeout=0.1) _redis_cli = sentinel.master_for(get_redis_sentinel_master(), socket_timeout=0.1) elif not _redis_cli: _redis_cli = redis.StrictRedis( host=get_redis_host(), port=get_redis_port(), db=get_redis_db(), password=get_redis_password() ) return _redis_cli
def delete_memoized(function, arg=None): if os.environ.get('PYBOSSA_REDIS_CACHE_DISABLED') is None: sentinel = Sentinel(settings.REDIS_SENTINEL, socket_timeout=0.1) master = sentinel.master_for(settings.REDIS_MASTER) keys = [] if arg: key_to_hash = ":%s" % arg key = "%s:%s_args::%s" % (settings.REDIS_KEYPREFIX, function.__name__, hashlib.md5(key_to_hash).hexdigest()) keys.append(key) else: key = "%s:%s_args::*" % (settings.REDIS_KEYPREFIX, function.__name__) keys = master.keys(key) for k in keys: master.delete(k) return True
def get_atomic_connection(self) -> StrictPipeline: """ Gets a StrictPipeline for normal redis or for redis sentinel based upon redis mode in configuration :return: Returns a StrictPipeline object """ if self.is_sentinel: kwargs = dict() if self.password: kwargs["password"] = self.password sentinel = Sentinel([(self.host, self.port)], **kwargs) pipeline = sentinel.master_for(self.sentinel_service, decode_responses=True).pipeline(True) else: pipeline = redis.StrictRedis(host=self.host, port=self.port, decode_responses=True, password=self.password).pipeline(True) return pipeline
def main(): last_send = None sentinel = Sentinel(REDIS_SENTINEL_LIST) redis_master = sentinel.master_for("mymaster", socket_timeout=120) while True: r = redis_master.blpop('alert:message', 60) if not r: continue current_timestamp = int(time.time()) if last_send and current_timestamp - last_send < 60: continue last_send = current_timestamp message = r[1] for email in ALERT_EMAILS: send_email(email, "viabtc server error", message)
def create_redis_connection(self): if self.mode == 'Redis': self.redis = stoneredis.StoneRedis(options.redisIP, db=options.redisDB, port=options.redisPort, socket_connect_timeout=5, socket_timeout=5) self.redis.connect() else: SENTINEL_POOL = Sentinel( options.sentinelServers, socket_timeout=0.1, max_connections=1000, ) self.redis = SENTINEL_POOL.master_for( options.sentinelName, redis_class=stoneredis.StoneRedis, socket_timeout=5, socket_connect_timeout=5, )