示例#1
0
def get_db():
    redis_store = getattr(g, '_database', None)
    if redis_store is None:
        if 'FAKE_DB' in app.config and app.config['FAKE_DB']:
            from mockredis import mock_redis_client
            redis_store = g._database = mock_redis_client(
                encoding='utf-8', decode_responses=True)
        else:
            if app.config['REDIS_URL'].startswith('redis://'):
                redis_store = g._database = Redis.from_url(
                    app.config['REDIS_URL'],
                    encoding='utf-8',
                    decode_responses=True)
            elif app.config['REDIS_URL'].startswith('sentinel://'):
                parsed_url = urlparse(app.config['REDIS_URL'])
                service = parsed_url.path.lstrip('/')
                port = 26379
                if ':' in parsed_url.netloc:
                    host, str_port = parsed_url.netloc.split(':')
                    port = int(str_port)
                else:
                    host = parsed_url.netloc
                sentinel = Sentinel([(host, port)], socket_timeout=0.1)
                redis_store = sentinel.master_for(service,
                                                  redis_class=Redis,
                                                  socket_timeout=0.1)
    return redis_store
示例#2
0
    def __init__(self, name, namespace="queue", **redis_kwargs):
        import redis  # pylint: disable=import-error
        from redis.sentinel import Sentinel  # pylint: disable=import-error

        redis_kwargs["decode_responses"] = True
        if redis_kwargs.get("sentinel") and redis_kwargs.get(
                "sentinel_service"):
            sentinels = [
                tuple(l.split(":"))
                for l in redis_kwargs.pop("sentinel").split(",")
            ]
            sentinel_service = redis_kwargs.pop("sentinel_service")
            kwargs = {
                k: v
                for k, v in redis_kwargs.items() if k in
                ["decode_responses", "password", "db", "socket_timeout"]
            }
            self._redis = Sentinel(sentinels,
                                   **kwargs).master_for(sentinel_service)
        else:
            kwargs = {
                k: v
                for k, v in redis_kwargs.items() if "sentinel" not in k
            }
            self._redis = redis.Redis(**kwargs)
        self._redis.ping()
        self.key = "{}:{}".format(namespace, name)
        logger.debug("Created redis queue with socket_timeout of {}s".format(
            redis_kwargs["socket_timeout"]))

        # clean up from previous implementations
        if self._redis.type(self.key) != "zset":
            self._redis.delete(self.key)
示例#3
0
 def __init__(self,
              servers,
              hash_method='crc32',
              sentinel=None,
              strict_redis=False):
     self.nodes = []
     self.connections = {}
     self.pool = None
     servers = format_servers(servers)
     if sentinel:
         sentinel = Sentinel(sentinel['hosts'],
                             socket_timeout=sentinel.get(
                                 'socket_timeout', 1))
     for server_config in servers:
         name = server_config.pop('name')
         server_config["max_connections"] = int(
             server_config.get("max_connections", 100))
         if name in self.connections:
             raise ValueError("server's name config must be unique")
         if sentinel:
             self.connections[name] = SentinelRedis(sentinel, name)
         elif strict_redis:
             self.connections[name] = redis.StrictRedis(**server_config)
         else:
             self.connections[name] = redis.Redis(**server_config)
         server_config['name'] = name
         self.nodes.append(name)
     self.ring = HashRing(self.nodes, hash_method=hash_method)
示例#4
0
class RedisCluster(object):
    def __init__(self, sentinel_hosts, service_name, **connection_kwargs):
        self.service_name = service_name
        self.sentinel = Sentinel(sentinel_hosts, **connection_kwargs)
        self.master = None
        self.master_host_port = None
        self.reconfigure()
        self.sentinel_threads = {}
        self.listen()

    def reconfigure(self):
        try:
            self.master_host_port = self.sentinel.discover_master(self.service_name)
            self.master = self.sentinel.master_for(self.service_name)
            log.info(f"Reconfigured master to {self.master_host_port}")
        except Exception as ex:
            log.error(f"Error while reconfiguring. {ex.args[0]}")

    def listen(self):
        def on_new_master(workerThread):
            self.reconfigure()

        for sentinel in self.sentinel.sentinels:
            sentinel_host = sentinel.connection_pool.connection_kwargs["host"]
            self.sentinel_threads[sentinel_host] = MyPubSubWorkerThread(
                sentinel, on_new_master, msg_sleep_time=0.001, daemon=True
            )
            self.sentinel_threads[sentinel_host].start()
示例#5
0
    def __init__(self, namespace="lock", **redis_kwargs):
        import redis  # pylint: disable=import-error
        from redis.sentinel import Sentinel  # pylint: disable=import-error

        redis_kwargs["decode_responses"] = True
        if redis_kwargs.get("sentinel") and redis_kwargs.get(
                "sentinel_service"):
            sentinels = [
                tuple(l.split(":"))
                for l in redis_kwargs.pop("sentinel").split(",")
            ]
            sentinel_service = redis_kwargs.pop("sentinel_service")
            kwargs = {
                k: v
                for k, v in redis_kwargs.items() if k in
                ["decode_responses", "password", "db", "socket_timeout"]
            }
            self._redis = Sentinel(sentinels,
                                   **kwargs).master_for(sentinel_service)
        else:
            kwargs = {
                k: v
                for k, v in redis_kwargs.items() if "sentinel" not in k
            }
            self._redis = redis.Redis(**kwargs)
        self._redis.ping()
        self._namespace = namespace
        logger.debug(
            "Created redis lock manager with socket_timeout of {}s".format(
                redis_kwargs["socket_timeout"]))
示例#6
0
    def get_connection(self, is_read_only=False) -> redis.StrictRedis:
        """
        Gets a StrictRedis connection for normal redis or for redis sentinel based upon redis mode in configuration.

        :type is_read_only: bool
        :param is_read_only: In case of redis sentinel, it returns connection to slave

        :return: Returns a StrictRedis connection
        """
        if self.connection is not None:
            return self.connection

        if self.is_sentinel:
            kwargs = dict()
            if self.password:
                kwargs["password"] = self.password
            sentinel = Sentinel([(self.host, self.port)], **kwargs)
            if is_read_only:
                connection = sentinel.slave_for(self.sentinel_service, decode_responses=True)
            else:
                connection = sentinel.master_for(self.sentinel_service, decode_responses=True)
        else:
            connection = redis.StrictRedis(host=self.host, port=self.port, decode_responses=True,
                                           password=self.password)
        self.connection = connection
        return connection
示例#7
0
    def get_master(self):
        print(self.redis_hosts)
        sentinel_hosts = []
        for redis_host in self.redis_hosts:
            redis_address_array = redis_host.split(':')
            redis_ip = redis_address_array[0]
            redis_port = redis_address_array[1]
            address = (redis_ip, redis_port)
            sentinel_hosts.append(address)

        print(sentinel_hosts)

        sentinel = Sentinel([('192.168.206.41', 26379),
                             ('192.168.206.42', 26379),
                             ('192.168.206.43', 26379)],
                            socket_timeout=0.1)
        print(sentinel.discover_master('mymaster'))
        master = sentinel.discover_master('mymaster')
        self.master_ip = master[0]
        self.master_port = master[1]
        print(str(master[0]) + ' is master')
        self.ip_master = {
            "addresses": [{
                "ip": self.master_ip
            }],
            "ports": [{
                "port": int(self.master_port),
                "protocol": "TCP"
            }]
        }
        return True
示例#8
0
    def _get_sentinel_connection(self, uri, kw):
        """
        get sentinel connection details from _uri
        """
        try:
            from redis.sentinel import Sentinel
        except ImportError:
            raise AnsibleError(
                "The 'redis' python module (version 2.9.0 or newer) is required to use redis sentinel."
            )

        if ';' not in uri:
            raise AnsibleError('_uri does not have sentinel syntax.')

        # format: "localhost:26379;localhost2:26379;0:changeme"
        connections = uri.split(';')
        connection_args = connections.pop(-1)
        if len(connection_args) > 0:  # hanle if no db nr is given
            connection_args = connection_args.split(':')
            kw['db'] = connection_args.pop(0)
            try:
                kw['password'] = connection_args.pop(0)
            except IndexError:
                pass  # password is optional

        sentinels = [tuple(shost.split(':')) for shost in connections]
        display.vv('\nUsing redis sentinels: %s' % sentinels)
        scon = Sentinel(sentinels, **kw)
        try:
            return scon.master_for(self._sentinel_service_name,
                                   socket_timeout=0.2)
        except Exception as exc:
            raise AnsibleError('Could not connect to redis sentinel: %s' %
                               to_native(exc))
示例#9
0
def setup_rq_connection():
    from redis.sentinel import Sentinel
    redis_url = current_app.config.get('RQ_SCHEDULER_DASHBOARD_REDIS_URL')
    redis_sentinels = current_app.config.get(
        'RQ_SCHEDULER_DASHBOARD_REDIS_SENTINELS')
    if redis_url:
        current_app.redis_conn = from_url(redis_url)
    elif redis_sentinels:
        redis_master = current_app.config.get(
            'RQ_SCHEDULER_DASHBOARD_REDIS_MASTER_NAME')
        password = current_app.config.get(
            'RQ_SCHEDULER_DASHBOARD_REDIS_PASSWORD')
        db = current_app.config.get('RQ_SCHEDULER_DASHBOARD_REDIS_DB')
        sentinel_hosts = [
            tuple(sentinel.split(':', 1))
            for sentinel in redis_sentinels.split(',')
        ]

        sentinel = Sentinel(sentinel_hosts, db=db, password=password)
        current_app.redis_conn = sentinel.master_for(redis_master)
    else:
        current_app.redis_conn = Redis(
            host=current_app.config.get('REDIS_HOST'),
            port=current_app.config.get('REDIS_PORT'),
            password=current_app.config.get('REDIS_PASSWORD'),
            db=current_app.config.get('REDIS_DB'))
示例#10
0
    def __init__(self,
                 connection_str,
                 db=0,
                 project=None,
                 service=None,
                 host=None,
                 conf=cfg.CONF,
                 **kwargs):
        """Redis driver for OSProfiler."""

        super(RedisSentinel, self).__init__(connection_str,
                                            project=project,
                                            service=service,
                                            host=host,
                                            conf=conf,
                                            **kwargs)
        try:
            from redis.sentinel import Sentinel
        except ImportError:
            raise exc.CommandError("To use this command, you should install "
                                   "'redis' manually. Use command:\n "
                                   "'pip install redis'.")

        self.conf = conf
        socket_timeout = self.conf.profiler.socket_timeout
        parsed_url = parser.urlparse(self.connection_str)
        sentinel = Sentinel([(parsed_url.hostname, int(parsed_url.port))],
                            password=parsed_url.password,
                            socket_timeout=socket_timeout)
        self.db = sentinel.master_for(self.conf.profiler.sentinel_service_name,
                                      socket_timeout=socket_timeout)
示例#11
0
def get_redis_connection(config, use_strict_redis=False):
    """
    Returns a redis connection from a connection config
    """
    redis_cls = redis.StrictRedis if use_strict_redis else redis.Redis

    if 'URL' in config:
        return redis_cls.from_url(config['URL'], db=config.get('DB'))
    if 'USE_REDIS_CACHE' in config.keys():

        try:
            from django.core.cache import caches
            cache = caches[config['USE_REDIS_CACHE']]
        except ImportError:
            from django.core.cache import get_cache
            cache = get_cache(config['USE_REDIS_CACHE'])

        if hasattr(cache, 'client'):
            # We're using django-redis. The cache's `client` attribute
            # is a pluggable backend that return its Redis connection as
            # its `client`
            try:
                # To get Redis connection on django-redis >= 3.4.0
                # we need to use cache.client.get_client() instead of
                # cache.client.client used in older versions
                try:
                    return cache.client.get_client()
                except AttributeError:
                    return cache.client.client
            except NotImplementedError:
                pass
        else:
            # We're using django-redis-cache
            try:
                return cache._client
            except AttributeError:
                # For django-redis-cache > 0.13.1
                return cache.get_master_client()

    if 'UNIX_SOCKET_PATH' in config:
        return redis_cls(unix_socket_path=config['UNIX_SOCKET_PATH'],
                         db=config['DB'])

    if 'SENTINELS' in config:
        sentinel_kwargs = {
            'db': config.get('DB'),
            'password': config.get('PASSWORD'),
            'socket_timeout': config.get('SOCKET_TIMEOUT'),
        }
        sentinel_kwargs.update(config.get('CONNECTION_KWARGS', {}))
        sentinel = Sentinel(config['SENTINELS'], **sentinel_kwargs)
        return sentinel.master_for(
            service_name=config['MASTER_NAME'],
            redis_class=redis_cls,
        )

    return redis_cls(host=config['HOST'],
                     port=config['PORT'],
                     db=config['DB'],
                     password=config.get('PASSWORD'))
示例#12
0
文件: views.py 项目: taplists/taplist
    def post(self, location):
        if location not in self.locations:
            return 'Unknown Location'
        if app.config['DEVEL']:
            pool = redis.ConnectionPool(host='localhost', port=6379)
            r = redis.Redis(connection_pool=pool)
        else:
            sentinel = Sentinel([('localhost', 26379)], socket_timeout=1)
            r = sentinel.master_for('mymaster', socket_timeout=1)
        beers = {key: r.hgetall(key) for key in r.keys('beer_{0}_*'.format(location))}

        for beername, beer in beers.items():
            r.hset(beername, 'active', 'True' if beername in request.form.getlist('checks') else 'False')

        for beer in request.form.getlist('delete'):
            r.delete(beer)

        r.save()
        beers = []
        for key in r.keys('beer_{0}_*'.format(location)):
            beer = convert(r.hgetall(key))
            beer['beername'] = key
            beers.append(beer)

        beers.sort(key=operator.itemgetter('brewery', 'name', 'type'))

        return redirect(location)
示例#13
0
文件: views.py 项目: taplists/taplist
 def get(self, location):
     if location not in self.locations:
         return 'Unknown Location'
     colors = get_colors(location, self.config)
     priceinfo = get_priceinfo(location, self.config)
     if app.config['DEVEL']:
         pool = redis.ConnectionPool(host='localhost', port=6379)
         r = redis.Redis(connection_pool=pool)
     else:
         sentinel = Sentinel([('localhost', 26379)], socket_timeout=1)
         r = sentinel.master_for('mymaster', socket_timeout=1)
     beers = []
     for key in r.keys('beer_{0}_*'.format(location)):
         beer = convert(r.hgetall(key))
         beer['beername'] = key
         beers.append(beer)
     if 'items' in priceinfo:
         for beer in beers:
             beer['price'] = '$ {0}'.format(' /'.join([beer[p].replace('.0', '') for p in priceinfo['items'] if p in beer]))
     beers.sort(key=operator.itemgetter('brewery', 'name', 'type'))
     return render_template(
         'edit.html',
         title='Beer List',
         beers=beers,
         location=location,
         colors=colors,
         roles=self.groups,
         locations=self.locations,
         priceinfo=priceinfo,
         scroll=False
     )
示例#14
0
class TestApplication(Application):
    def __init__(self):
        # Call to the superclass to bootstrap.
        super(TestApplication, self).__init__(name='krux-redis-sentinel')

        self.logger.debug('Parsing sentinels from args: %r', self.args.sentinel)
        sentinels = hostports_from_args(self.args)
        self.logger.debug('Parsed sentinel host, port pairs: %r', sentinels)

        self.logger.debug('Initializing Sentinel instance...')
        self.sentinel = Sentinel(sentinels, socket_timeout=SOCKET_TIMEOUT)
        self.logger.debug('Initialized Sentinel instance: %r', self.sentinel)

    def add_cli_arguments(self, parser):
        add_sentinel_cli_arguments(parser)

    def run(self):
        master = self.sentinel.master_for(self.name)
        slave = self.sentinel.slave_for(self.name)

        try:
            self.logger.info('Ping master: %s', master.ping())
        except ConnectionError as err:
            self.logger.warning('Could not connect to master!')
            self.logger.error(err)

        try:
            self.logger.info('Ping slave: %s', slave.ping())
        except ConnectionError as err:
            self.logger.warning('Could not connect to slave!')
            self.logger.error(err)
示例#15
0
def redis_client():
    if settings.CACHEOPS_REDIS and settings.CACHEOPS_SENTINEL:
        raise ImproperlyConfigured("CACHEOPS_REDIS and CACHEOPS_SENTINEL are mutually exclusive")

    client_class = CacheopsRedis
    if settings.CACHEOPS_CLIENT_CLASS:
        client_class = import_string(settings.CACHEOPS_CLIENT_CLASS)

    if settings.CACHEOPS_SENTINEL:
        if not {'locations', 'service_name'} <= set(settings.CACHEOPS_SENTINEL):
            raise ImproperlyConfigured("Specify locations and service_name for CACHEOPS_SENTINEL")

        sentinel = Sentinel(
            settings.CACHEOPS_SENTINEL['locations'],
            **omit(settings.CACHEOPS_SENTINEL, ('locations', 'service_name', 'db')))
        return sentinel.master_for(
            settings.CACHEOPS_SENTINEL['service_name'],
            redis_class=client_class,
            db=settings.CACHEOPS_SENTINEL.get('db', 0)
        )

    # Allow client connection settings to be specified by a URL.
    if isinstance(settings.CACHEOPS_REDIS, six.string_types):
        return client_class.from_url(settings.CACHEOPS_REDIS)
    else:
        return client_class(**settings.CACHEOPS_REDIS)
示例#16
0
def setup_rq_connection():
    # It's the only place where we can safely define default value for web background
    # since It is used in template
    current_app.config.setdefault('RQ_DASHBOARD_WEB_BACKGROUND', 'black')
    # we need to do It here instead of cli, since It may be embeded
    upgrade_config(current_app)
    # Getting Redis connection parameters for RQ
    redis_url = current_app.config.get('RQ_DASHBOARD_REDIS_URL')
    redis_sentinels = current_app.config.get('RQ_DASHBOARD_REDIS_SENTINELS')
    if isinstance(redis_url, list):
        current_app.redis_conn = from_url(redis_url[0])
    elif isinstance(redis_url, string_types):
        current_app.redis_conn = from_url(redis_url)
    elif redis_sentinels:
        redis_master = current_app.config.get('RQ_DASHBOARD_REDIS_MASTER_NAME')
        password = current_app.config.get('RQ_DASHBOARD_REDIS_PASSWORD')
        db = current_app.config.get('RQ_DASHBOARD_REDIS_DB')
        sentinel_hosts = [tuple(sentinel.split(':', 1))
                          for sentinel in redis_sentinels.split(',')]

        sentinel = Sentinel(sentinel_hosts, db=db, password=password)
        current_app.redis_conn = sentinel.master_for(redis_master)
    else:
        current_app.redis_conn = Redis(
            host=current_app.config.get('RQ_DASHBOARD_REDIS_HOST', 'localhost'),
            port=current_app.config.get('RQ_DASHBOARD_REDIS_PORT', 6379),
            password=current_app.config.get('RQ_DASHBOARD_REDIS_PASSWORD'),
            db=current_app.config.get('RQ_DASHBOARD_REDIS_DB', 0),
        )
示例#17
0
    def _init(self, server, params):
        super(CacheClass, self).__init__(params)
        self._server = server
        self._params = params

        self.service_name = self.options.get("SERVICE_NAME", 'mymaster')
        self.slave_read = self.options.get("READ_SLAVE", False)
        if self.server:
            def _get_hosttupe(conn):
                host_lst = conn.rsplit(':', 1)
                return host_lst[0], int(host_lst[1])

            conn_list = [_get_hosttupe(conn) for conn in self.server]

        else:
            raise ImproperlyConfigured("sentinel server config error.")

        kwargs = self.connection_pool_class_kwargs
        max_connections = kwargs.pop('max_connections')
        sentinel_manager = Sentinel(conn_list, sentinel_kwargs=kwargs)

        kwargs.update({
            'db': self.db,
            'password': self.password,
            'max_connections': max_connections
        })
        self._client = sentinel_manager.master_for(self.service_name,
                                                   connection_pool_class=self.connection_pool_class,
                                                   **kwargs)

        if self.slave_read:
            self._slave_client = sentinel_manager.slave_for(self.service_name,
                                                            connection_pool_class=self.connection_pool_class,
                                                            **kwargs)
示例#18
0
def get_redis(app=None):
    app = app_or_default(app)
    conf = ensure_conf(app)
    if not hasattr(app, 'redbeat_redis') or app.redbeat_redis is None:
        redis_options = conf.app.conf.get(
            'REDBEAT_REDIS_OPTIONS',
            conf.app.conf.get('BROKER_TRANSPORT_OPTIONS', {}))
        retry_period = redis_options.get('retry_period')
        if conf.redis_url.startswith(
                'redis-sentinel') and 'sentinels' in redis_options:
            from redis.sentinel import Sentinel
            sentinel = Sentinel(
                redis_options['sentinels'],
                socket_timeout=redis_options.get('socket_timeout'),
                password=redis_options.get('password'),
                decode_responses=True)
            connection = sentinel.master_for(
                redis_options.get('service_name', 'master'))
        else:
            connection = StrictRedis.from_url(conf.redis_url,
                                              decode_responses=True)

        if retry_period is None:
            app.redbeat_redis = connection
        else:
            app.redbeat_redis = RetryingConnection(retry_period, connection)

    return app.redbeat_redis
示例#19
0
def create_app():
    '''工厂方法'''
    # 创建app对象
    app = Flask(__name__)
    # 加载配置
    config = config_dict.get(app.config.get('ENV'))
    app.config.from_object(config)
    # 加载数据库配置
    from common.models import db
    db.init_app(app)
    # 记录日志信息
    log_file(config.LOG_LV)
    # redis集群加载
    from rediscluster import RedisCluster
    app.redis_cluster = RedisCluster(
        startup_nodes=app.config.get('REDIS_CLUSTER'))
    # redis哨兵,主从配置
    from redis.sentinel import Sentinel
    sentinel = Sentinel(app.config.get('REDIS_SENTINELS'))
    app.redis_master = sentinel.master_for(
        app.config.get('REDIS_SENTINEL_SERVICES_NAME'))
    app.redis_slave = sentinel.slave_for(
        app.config.get('REDIS_SENTINEL_SERVICES_NAME'))
    # 注册用户蓝图到app中
    from .resources.users import user_blue
    app.register_blueprint(user_blue)
    return app
示例#20
0
    def __get_master(self, update=False):
        master_ip = self.node.get_ip()
        master_port = self.node.get_port()
        master_node_id = self.node.get_node_id()

        hosts = self.__get_hosts(refresh=update)

        if len(hosts) > 0:
            master_ip = hosts[0]['ip']
            master_node_id = hosts[0]['node_id']
        
        if os.path.isfile(Constants.MASTER_FILE):
            fd = open(Constants.MASTER_FILE, "r")
            temp_ip, temp_port = fd.readline().split(' ')
            fd.close()
            if temp_ip in hosts:
                master_ip, master_port = temp_ip, temp_port
            else:
                os.remove(Constants.MASTER_FILE)

        for host in hosts:
            sentinel = Sentinel([(host["ip"], Constants.SENTINEL_PORT)], socket_timeout=0.1)
            try:
                ip, port = sentinel.discover_master(Constants.REDIS_SENTINEL_NAME)
                for ht in hosts:
                    if ip == ht['ip']:
                        master_ip, master_port = ip, port
                        break
            except:
                pass
        return master_ip, master_port
示例#21
0
    def __init__(self, name, namespace='queue', **redis_kwargs):
        import redis
        from redis.sentinel import Sentinel
        redis_kwargs['decode_responses'] = True
        if redis_kwargs.get('sentinel') and redis_kwargs.get(
                'sentinel_service'):
            sentinels = [
                tuple(l.split(':'))
                for l in redis_kwargs.pop('sentinel').split(',')
            ]
            sentinel_service = redis_kwargs.pop('sentinel_service')
            kwargs = {
                k: v
                for k, v in redis_kwargs.items() if k in
                ["decode_responses", "password", "db", "socket_timeout"]
            }
            self._redis = Sentinel(sentinels,
                                   **kwargs).master_for(sentinel_service)
        else:
            kwargs = {
                k: v
                for k, v in redis_kwargs.items() if "sentinel" not in k
            }
            self._redis = redis.Redis(**kwargs)
        self._redis.ping()
        self.key = "{}:{}".format(namespace, name)
        info("Created redis queue with socket_timeout of {}s".format(
            redis_kwargs['socket_timeout']))

        # clean up from previous implementations
        if self._redis.type(self.key) != 'zset':
            self._redis.delete(self.key)
示例#22
0
def redis_populate():
    """Function to populate keys in Redis Server"""
    # client = redis.StrictRedis(host=configs["redis_host"], port=configs["redis_port"])
    # sentinel = Sentinel(
    #   [
    #     ("redis-cloudflare-node-0.redis-cloudflare-headless.dns-proxy.svc.cluster.local", 26379),
    #     ("redis-cloudflare-node-1.redis-cloudflare-headless.dns-proxy.svc.cluster.local", 26379),
    #     ("redis-cloudflare-node-2.redis-cloudflare-headless.dns-proxy.svc.cluster.local", 26379)
    #   ],
    # socket_timeout=0.1)

    from redis.sentinel import Sentinel

    sentinel = Sentinel([
        ('redis-cloudflare', 26379),
    ],
                        sentinel_kwargs={'password': '******'},
                        password='******')

    sentinel.discover_master('mymaster')

    client = sentinel.master_for('mymaster', socket_timeout=0.5)
    i = 93218
    key = 'key' + str(i)
    value = 'value' + str(i)
    result = client.get(key)
    print(result)
示例#23
0
def redis_populate():
    """Function to populate keys in Redis Server"""
    # client = redis.StrictRedis(host=configs["redis_host"], port=configs["redis_port"])
    # sentinel = Sentinel(
    #   [
    #     ("redis-cloudflare-node-0.redis-cloudflare-headless.dns-proxy.svc.cluster.local", 26379),
    #     ("redis-cloudflare-node-1.redis-cloudflare-headless.dns-proxy.svc.cluster.local", 26379),
    #     ("redis-cloudflare-node-2.redis-cloudflare-headless.dns-proxy.svc.cluster.local", 26379)
    #   ],
    # socket_timeout=0.1)

    from redis.sentinel import Sentinel

    sentinel = Sentinel([
        ('redis-cloudflare', 26379),
    ],
                        sentinel_kwargs={'password': '******'},
                        password='******')

    sentinel.discover_master('mymaster')

    client = sentinel.master_for('mymaster', socket_timeout=0.5)
    for i in range(1000000):
        key = 'key' + str(i)
        # value='value'+str(i)
        value = str(
            i
        ) + '::: https://www.google.co.uk/#sclient=psy-ab&hl=en&source=hp&q=ASUSTeK+Computer+INC.+Model+M4A78LT-M+manual&pbx=1&oq=ASUSTeK+Computer+INC.+Model+M4A78LT-M+manual&aq=f&aqi=&aql=&gs_sm=3&gs_upl=52765l57528l0l57848l8l8l0l0l0l0l2413l3989l8-1.1l2l0&bav=on.2,or.r_gc.r_pw.,cf.osb&fp=3d6c1d1d0a5ea45f&biw=1262&bih=879'
        client.set(key, value)
        print(key, value)
示例#24
0
def get_redis_connection(config, is_master=True):
    """
    Returns redis connection basing on configuration dict
    Args:
        config: Dict with configuration, if key `SENTINELS` is present
                sentinel connection pool will be used.
        is_master: Whether connection should be master or slave,
                   applies only if sentinels are used

    Returns: StrictRedis connection

    """
    if 'SENTINELS' in config:
        sentinel = Sentinel(
            config['SENTINELS'],
            db=config.get('DB'),
            password=config.get('PASSWORD'),
            socket_timeout=config.get('TIMEOUT'),
            socket_connect_timeout=config.get('CONNECT_TIMEOUT'),
        )
        if is_master:
            return sentinel.master_for(settings.REDIS_CLUSTER_NAME)
        else:
            return sentinel.slave_for(settings.REDIS_CLUSTER_NAME)
    else:
        return StrictRedis(
            host=config['HOST'],
            port=config['PORT'],
            db=config.get('DB'),
            password=config.get('PASSWORD'),
            socket_timeout=config.get('TIMEOUT'),
            socket_connect_timeout=config.get('CONNECT_TIMEOUT'),
        )
示例#25
0
    def sentinel_connect(self, master_name):
        url = urlparse.urlparse(self.schedule_url)

        def parse_host(s):
            if ':' in s:
                host, port = s.split(':', 1)
                port = int(port)
            else:
                host = s
                port = 26379

            return host, port

        if '@' in url.netloc:
            auth, hostspec = url.netloc.split('@', 1)
        else:
            auth = None
            hostspec = url.netloc

        if auth and ':' in auth:
            _, password = auth.split(':', 1)
        else:
            password = None
        path = url.path
        if path.startswith('/'):
            path = path[1:]
        hosts = [parse_host(s) for s in hostspec.split(',')]
        sentinel = Sentinel(hosts, password=password, db=path)
        master = sentinel.master_for(master_name)
        return master
示例#26
0
    def connect(self, index=0, write=True):
        """
        Creates a redis connection with connection pool.
        """
        master_name, sentinel_hosts, db = self.parse_connection_string(self._connection_string)

        sentinel_timeout = self._options.get('SENTINEL_TIMEOUT', 1)
        sentinel = Sentinel(sentinel_hosts, socket_timeout=sentinel_timeout)

        if write:
            host, port = sentinel.discover_master(master_name)
        else:
            host, port = random.choice([sentinel.discover_master(master_name)] + sentinel.discover_slaves(master_name))

        kwargs = {
            "db": db,
            "parser_class": self.parser_class,
            "password": self._options.get('PASSWORD', None),
        }

        kwargs.update({'host': host, 'port': port, 'connection_class': Connection})

        if 'SOCKET_TIMEOUT' in self._options:
            kwargs.update({'socket_timeout': int(self._options['SOCKET_TIMEOUT'])})

        kwargs.update(self._pool_cls_kwargs)

        connection_pool = get_or_create_connection_pool(self._pool_cls, **kwargs)
        connection = Redis(connection_pool=connection_pool)
        return connection
示例#27
0
    def __redis_client__(self, instance):

        try:
            LOG.debug('Connecting to redis databaseinfra %s', self.databaseinfra)
            # redis uses timeout in seconds
            connection_timeout_in_seconds = Configuration.get_by_name_as_int('redis_connect_timeout', default=REDIS_CONNECTION_DEFAULT_TIMEOUT)

            if (instance and instance.instance_type == Instance.REDIS) or (not self.databaseinfra.plan.is_ha and not instance):
                connection_address, connection_port = self.__get_admin_single_connection(instance)
                client = redis.Redis(host = connection_address,
                                    port = int(connection_port),
                                    password = self.databaseinfra.password,
                                    socket_connect_timeout = connection_timeout_in_seconds)

            else:
                sentinels = self.__get_admin_sentinel_connection(instance)
                sentinel = Sentinel(sentinels, socket_timeout=connection_timeout_in_seconds)
                client = sentinel.master_for(self.databaseinfra.name,
                                             socket_timeout=connection_timeout_in_seconds,
                                             password = self.databaseinfra.password)

            LOG.debug('Successfully connected to redis databaseinfra %s' % (self.databaseinfra))
            return client
        except Exception, e:
            raise e
示例#28
0
 def _connection(self):
     sentinel = Sentinel(self._sentinels,
                         socket_timeout=self._socket_timeout)
     slave = sentinel.slave_for(self._sentinel_master,
                                socket_timeout=self._socket_timeout,
                                db=self._db)
     return slave
示例#29
0
    def connect(self, write=True):
        """
        Create a redis connection with connection pool.
        """
        self.log.debug("connect called: write=%s", write)

        sentinel_timeout = self._options.get("SENTINEL_TIMEOUT", 1)
        password = self._options.get("PASSWORD", None)
        sentinel = Sentinel(
            self._sentinel_hosts, socket_timeout=sentinel_timeout, password=password
        )

        if write:
            host, port = sentinel.discover_master(self._master_name)
        else:
            try:
                host, port = random.choice(  # nosec
                    sentinel.discover_slaves(self._master_name)
                )
            except IndexError:
                self.log.debug("no slaves are available. using master for read.")
                host, port = sentinel.discover_master(self._master_name)

        if password:
            connection_url = f"redis://:{password}@{host}:{port}/{self._database_name}"
        else:
            connection_url = f"redis://{host}:{port}/{self._database_name}"
        self.log.debug("Connecting to: %s", connection_url)
        return self.connection_factory.connect(connection_url)
示例#30
0
def get_redis_master(redis_sentinel_nodes):
    nodes_list = redis_sentinel_nodes.split(',')
    sentinel = Sentinel([tuple(nodes_list[0].split(':'))], socket_timeout=0.1)
    master = sentinel.discover_master('mymaster')
    if not master:
        sys.exit(1)
    return list(master)
示例#31
0
 def __init__(self, namespace='lock', **redis_kwargs):
     import redis
     from redis.sentinel import Sentinel
     redis_kwargs['decode_responses'] = True
     if redis_kwargs.get('sentinel') and redis_kwargs.get(
             'sentinel_service'):
         sentinels = [
             tuple(l.split(':'))
             for l in redis_kwargs.pop('sentinel').split(',')
         ]
         sentinel_service = redis_kwargs.pop('sentinel_service')
         kwargs = {
             k: v
             for k, v in redis_kwargs.items() if k in
             ["decode_responses", "password", "db", "socket_timeout"]
         }
         self._redis = Sentinel(sentinels,
                                **kwargs).master_for(sentinel_service)
     else:
         kwargs = {
             k: v
             for k, v in redis_kwargs.items() if "sentinel" not in k
         }
         self._redis = redis.Redis(**kwargs)
     self._redis.ping()
     self._namespace = namespace
     info("Created redis lock manager with socket_timeout of {}s".format(
         redis_kwargs['socket_timeout']))
示例#32
0
class RedisLock(Lock):
    def __init__(self, namespace='lock', **redis_kwargs):
        import redis
        from redis.sentinel import Sentinel
        redis_kwargs['decode_responses'] = True
        if redis_kwargs.get('sentinel') and redis_kwargs.get('sentinel_service'):
            sentinels = [tuple(l.split(':')) for l in redis_kwargs.pop('sentinel').split(',')]
            sentinel_service = redis_kwargs.pop('sentinel_service')
            kwargs = {k: v for k, v in redis_kwargs.items() if k in ["decode_responses", "password", "db", "socket_timeout"]}
            self._redis = Sentinel(sentinels, **kwargs).master_for(sentinel_service)
        else:
            kwargs = {k: v for k, v in redis_kwargs.items() if "sentinel" not in k}
            self._redis = redis.Redis(**kwargs)
        self._redis.ping()
        self._namespace = namespace
        info("Created redis lock manager with socket_timeout of {}s".format(redis_kwargs['socket_timeout']))

    def __key(self, name):
        return "{}:{}".format(self._namespace, name)

    def lock(self, name, owner, expiration=1, allow_owner_relock=False):
        """
        Obtain the named lock.
        :param allow_owner_relock: bool
        :param name: str the name of the lock
        :param owner: str a unique name for the locking node
        :param expiration: int in seconds, 0 expiration means forever
        """
        import redis

        try:
            if int(expiration) < 1:
                expiration = 1

            key = self.__key(name)
            non_existing = not (allow_owner_relock and self._redis.get(key) == owner)
            return self._redis.set(key, owner, ex=int(expiration), nx=non_existing)
        except redis.exceptions.ResponseError as e:
            exception("Unable to obtain lock, local state: name: %s, owner: %s, expiration: %s, allow_owner_relock: %s",
                      name, owner, expiration, allow_owner_relock)

    def unlock(self, name, owner):
        """
        Release the named lock.
        :param name: str the name of the lock
        :param owner: str a unique name for the locking node
        """
        key = self.__key(name)
        if self._redis.get(key) == owner:
            self._redis.delete(key)
            return True
        return False

    def check_lock(self, name):
        return self._redis.get(self.__key(name)) is not None

    def print_locks(self):
        keys = self._redis.keys(self.__key('*'))
        for key in keys:
            print("{} locked by {}, expires in {} seconds".format(key, self._redis.get(key), self._redis.ttl(key)))
示例#33
0
def get_redis_from_config(settings, connection_class=Redis):
    """Returns a StrictRedis instance from a dictionary of settings.
       To use redis sentinel, you must specify a dictionary in the configuration file.
       Example of a dictionary with keys without values:
       SENTINEL: {'INSTANCES':, 'SOCKET_TIMEOUT':, 'PASSWORD':,'DB':, 'MASTER_NAME':}
    """
    if settings.get('REDIS_URL') is not None:
        return connection_class.from_url(settings['REDIS_URL'])

    elif settings.get('SENTINEL') is not None:
        instances = settings['SENTINEL'].get('INSTANCES', [('localhost', 26379)])
        socket_timeout = settings['SENTINEL'].get('SOCKET_TIMEOUT', None)
        password = settings['SENTINEL'].get('PASSWORD', None)
        db = settings['SENTINEL'].get('DB', 0)
        master_name = settings['SENTINEL'].get('MASTER_NAME', 'mymaster')
        sn = Sentinel(instances, socket_timeout=socket_timeout, password=password, db=db)
        return sn.master_for(master_name)

    kwargs = {
        'host': settings.get('REDIS_HOST', 'localhost'),
        'port': settings.get('REDIS_PORT', 6379),
        'db': settings.get('REDIS_DB', 0),
        'password': settings.get('REDIS_PASSWORD', None),
        'ssl': settings.get('REDIS_SSL', False),
    }

    return connection_class(**kwargs)
示例#34
0
    def get_connection(self, is_read_only=False) -> redis.Redis:
        """
        Gets a Redis connection for normal redis or for redis sentinel based upon redis mode in configuration.

        :type is_read_only: bool
        :param is_read_only: In case of redis sentinel, it returns connection to slave

        :return: Returns a Redis connection
        """
        if self.connection is not None:
            return self.connection

        if self.is_sentinel:
            kwargs = dict()
            if self.password:
                kwargs["password"] = self.password
            sentinel = Sentinel([(self.host, self.port)], **kwargs)
            if is_read_only:
                connection = sentinel.slave_for(self.sentinel_service,
                                                decode_responses=True)
            else:
                connection = sentinel.master_for(self.sentinel_service,
                                                 decode_responses=True)
        else:
            connection = redis.Redis(host=self.host,
                                     port=self.port,
                                     decode_responses=True,
                                     password=self.password)
        self.connection = connection
        return connection
示例#35
0
    def __init__(self, **config):
        try:
            redisIpConf = config['_channel_redis_sentinel']
            if not redisIpConf:
                log.error('Can not find valid ip config.')
            else:
                redisConfigure = []
                redisIpSplit = redisIpConf.split(',')
                for redisInfo in redisIpSplit:
                    redisInfoSplit = redisInfo.split(':')
                    redisConfigure.append(
                        (redisInfoSplit[0], redisInfoSplit[1]))

                self.sentinel = Sentinel(redisConfigure)
                self.redisInstance = self.sentinel.master_for(
                    'redis-master', password=config['_channel_redis_password'])

                ##self._pool = redis.ConnectionPool(host=config['_channel_redis_host'], port=config['_channel_redis_port'], db=1, password=config['_channel_redis_password'])
                ##self.redisInstance = redis.StrictRedis(connection_pool=self._pool)
                self._master_pub_topic = config['_master_pub_topic']
                self._none_match_ip = 'no_ip_matched'
                self._sub_node = config['_sub_node']

        except:
            #log.error("redis init failed.")
            log.error(traceback.format_exc())
示例#36
0
def redis_client():
    if settings.CACHEOPS_REDIS and settings.CACHEOPS_SENTINEL:
        raise ImproperlyConfigured("CACHEOPS_REDIS and CACHEOPS_SENTINEL are mutually exclusive")

    client_class = CacheopsRedis
    if settings.CACHEOPS_CLIENT_CLASS:
        client_class = import_string(settings.CACHEOPS_CLIENT_CLASS)

    if settings.CACHEOPS_SENTINEL:
        if not {'locations', 'service_name'} <= set(settings.CACHEOPS_SENTINEL):
            raise ImproperlyConfigured("Specify locations and service_name for CACHEOPS_SENTINEL")

        sentinel = Sentinel(
            settings.CACHEOPS_SENTINEL['locations'],
            **omit(settings.CACHEOPS_SENTINEL, ('locations', 'service_name', 'db')))
        return sentinel.master_for(
            settings.CACHEOPS_SENTINEL['service_name'],
            redis_class=client_class,
            db=settings.CACHEOPS_SENTINEL.get('db', 0)
        )

    # Allow client connection settings to be specified by a URL.
    if isinstance(settings.CACHEOPS_REDIS, str):
        return client_class.from_url(settings.CACHEOPS_REDIS)
    else:
        return client_class(**settings.CACHEOPS_REDIS)
示例#37
0
文件: redis.py 项目: gokerguner/ecomm
def connect2redis(host, port=6379, db=0, connectionPool=True, logging=None, sentinel_service=prms.REDIS_SENTINEL_SERVICE_NAME, master=True):
    if sentinel_service:
        retry_count = 0
        while retry_count < 10:
            sentinel = Sentinel([(host, 26379)])
            if master:
                client = sentinel.master_for(sentinel_service)
            else:
                client = sentinel.slave_for(sentinel_service)
            if is_available(client, logging=logging):
                logging.info("Connected to Redis succesfully.")
                return client
            else:
                print(f"Waiting for {pow(2, retry_count)} secs.")
                time.sleep(pow(2, retry_count))
                retry_count += 1
    else:
        retry_count = 0
        while retry_count < 10:
            if connectionPool:
                pool = redis.ConnectionPool(host=host, port=port, db=db)
                client = redis.StrictRedis(connection_pool=pool)
            else:
                client = redis.StrictRedis(host=host, port=port, db=db)
            if is_available(client, logging=logging):
                logging.info("Connected to Redis succesfully.")
                return client
            else:
                print(f"Waiting for {pow(2, retry_count)} secs.")
                time.sleep(pow(2, retry_count))
                retry_count += 1
    logging.error("Exiting...")
    exit(3)
示例#38
0
文件: helpers.py 项目: nvie/rq
def get_redis_from_config(settings, connection_class=Redis):
    """Returns a StrictRedis instance from a dictionary of settings.
       To use redis sentinel, you must specify a dictionary in the configuration file.
       Example of a dictionary with keys without values:
       SENTINEL: {'INSTANCES':, 'SOCKET_TIMEOUT':, 'PASSWORD':,'DB':, 'MASTER_NAME':}
    """
    if settings.get('REDIS_URL') is not None:
        return connection_class.from_url(settings['REDIS_URL'])

    elif settings.get('SENTINEL') is not None:
        instances = settings['SENTINEL'].get('INSTANCES', [('localhost', 26379)])
        socket_timeout = settings['SENTINEL'].get('SOCKET_TIMEOUT', None)
        password = settings['SENTINEL'].get('PASSWORD', None)
        db = settings['SENTINEL'].get('DB', 0)
        master_name = settings['SENTINEL'].get('MASTER_NAME', 'mymaster')
        sn = Sentinel(instances, socket_timeout=socket_timeout, password=password, db=db)
        return sn.master_for(master_name)

    kwargs = {
        'host': settings.get('REDIS_HOST', 'localhost'),
        'port': settings.get('REDIS_PORT', 6379),
        'db': settings.get('REDIS_DB', 0),
        'password': settings.get('REDIS_PASSWORD', None),
        'ssl': settings.get('REDIS_SSL', False),
    }

    return connection_class(**kwargs)
示例#39
0
def run():
  try:
    logging.info('Calculating training booking reminders')
    if SENTINEL_HOST is None:
        client = redis.StrictRedis(host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD)
    else:
        sentinel = Sentinel([(SENTINEL_HOST, SENTINEL_PORT)])
        client = sentinel.master_for(SENTINEL_MASTER)

    # When was the last time we notified for? Default to now
    last_not_str = client.get(WEEK_LAST_CHECK)
    now = datetime.datetime.now()
    last_notified = now
    if last_not_str is not None:
      last_notified_date = dateutil.parser.parse(last_not_str)

      # Check that this is after the current date
      if last_notified_date > now:
        last_notified = last_notified_date

    # Now work out the date that is midnight one week and a day from now
    one_week = datetime.datetime.combine(datetime.date.today() + datetime.timedelta(8), datetime.time())

    logging.info('Getting sessions from %s until %s' % (str(last_notified), str(one_week)))

    if one_week <= last_notified:
      return

    # Find all the sessions that fall bewtween the last notified date and a weeks time
    sessions = TrainingSession.query({
      'find': {
        'start': {
          '$gte': last_notified,
          '$lt': one_week
        }
      }
    })

    # Notify that these need reminding
    for session in sessions:
      data = {
        'name': 'reminder',
        'data': {
          'type': 'trainingSession',
          'data': session._id
        }
      }
      client.rpush(EVENT_Q, json.dumps(data))

    if (len(sessions) > 0):
      client.publish(EVENT_CHAN, 'event')

    logging.info('Published training booking events')

    # Store that we've notified
    client.set(WEEK_LAST_CHECK, one_week.isoformat())
    logging.info('Finished training booking')
  except Exception as ex:
    logging.exception(ex)
示例#40
0
def check_sso_auth(user_id, brand_id, trans_type ):
    sentinel = Sentinel([('120.76.154.208', 26379)], socket_timeout=0.5)
    log.d(sentinel.discover_master('mymaster'))
    redis_obj = sentinel.master_for('mymaster', socket_timeout=0.1)

    db_obj = PostgreSQLConnector()
    sso_cls = SSOAuth(redis_obj,db_obj)
    return sso_cls.check_auth(user_id, brand_id, trans_type)
示例#41
0
def sso_log(user_id, brand_id, trans_type, data_type,data_content,log_time):
    sentinel = Sentinel([('112.74.198.224', 26379)], socket_timeout=0.5)
    log.d(sentinel.discover_master('mymaster'))
    redis_obj = sentinel.master_for('mymaster', socket_timeout=0.1)

    db_obj = PostgreSQLConnector()
    sso_cls = SSOAuth(redis_obj,db_obj)
    return sso_cls.send_log(user_id, brand_id, trans_type, data_type,data_content,log_time)
示例#42
0
def redis_conn(readonly=True):
    global sentinel, master_client, slave_client
    if not sentinel:
        sentinel = Sentinel(settings.REDIS_SENTINEL, socket_timeout=1)
    if not master_client:
        master_client = sentinel.master_for(settings.REDIS_CLUSTER)
    if not slave_client:
        slave_client = sentinel.slave_for(settings.REDIS_CLUSTER)
    return slave_client if readonly else master_client
示例#43
0
 def setUp(self):
     sentinel = Sentinel(settings_test.REDIS_SENTINEL)
     db = getattr(settings_test, 'REDIS_DB', 0)
     self.connection = sentinel.master_for('mymaster', db=db)
     self.connection.flushall()
     self.guard = ContributionsGuard(self.connection)
     self.anon_user = {'user_id': None, 'user_ip': '127.0.0.1'}
     self.auth_user = {'user_id': 33, 'user_ip': None}
     self.task = Task(id=22)
示例#44
0
 def client(self):
     sentinel = Sentinel(
         self.sentinel_conf.get('sentinels'),
         min_other_sentinels=self.sentinel_conf.get("min_other_sentinels", 0),
         password=self.sentinel_conf.get("password"),
         sentinel_kwargs={"socket_timeout": self.sentinel_conf.get("sentinel_timeout")},
     )
     return sentinel.master_for(self.sentinel_conf.get("service_name"), Redis,
                                socket_timeout=self.sentinel_conf.get("socket_timeout"))
示例#45
0
	def __init__(self, **redis_conf):
		self.prefix = 'q'
		sentinels = redis_conf.pop('sentinels', '()')
		sentinels_service_name = redis_conf.pop('sentinels_service_name', 
				'mymaster')
		min_other_sentinels = redis_conf.pop('min_other_sentinels', 0)
		s = Sentinel(sentinels, **redis_conf)
		self.db = s.master_for(sentinels_service_name)
		#self.db = StrictRedis(**redis_conf)
		self.key = '%s:%s' % (self.prefix, '%s')
示例#46
0
    def _sentinel_managed_pool(self):

        sentinel = Sentinel(
            self.sentinels,
            min_other_sentinels=getattr(self, "min_other_sentinels", 0),
            password=getattr(self, "password", None),
            sentinel_kwargs={"socket_timeout": getattr(self, "sentinel_timeout", None)},
        )
        return sentinel.master_for(self.service_name, self.Client,
                                   socket_timeout=self.socket_timeout).connection_pool
示例#47
0
文件: util.py 项目: liutaihua/hera
def get_redis_url():
    if hasattr(config, 'ETCD_HOST'):
        return config.HERA_REDIS_SENTINEL
    sentinel_list = [tuple(i.split(':')) for i in get_etcd_setting('HERA_REDIS_SENTINEL').split(',')]
    redis_db = get_etcd_setting('HERA_REDIS_DB')
    sentinel = Sentinel(sentinel_list, socket_timeout=0.1)
    master_name = get_etcd_setting('HERA_REDIS_MASTERNAME')
    return 'redis://{}/{}'.format(
        ':'.join(map(str, sentinel.discover_master(master_name))),
        redis_db
    )
示例#48
0
    def new_new(_, cls, broker_url, *args, **kwargs):
        scheme = urlparse(broker_url).scheme
        if scheme == "redis-sentinel":
            from rpaas.tasks import app

            opts = app.conf.BROKER_TRANSPORT_OPTIONS
            s = Sentinel(opts["sentinels"], password=opts["password"])
            host, port = s.discover_master(opts["service_name"])
            return RedisBroker("redis://:{}@{}:{}".format(opts["password"], host, port))
        else:
            old_new(cls, broker_url, *args, **kwargs)
示例#49
0
    def _redis(self):
        if getattr(settings, 'EXPERIMENTS_REDIS_SENTINELS', None):
            sentinel = Sentinel(settings.EXPERIMENTS_REDIS_SENTINELS, socket_timeout=settings.EXPERIMENTS_REDIS_SENTINELS_TIMEOUT)
            host, port = sentinel.discover_master(settings.EXPERIMENTS_REDIS_MASTER_NAME)
        else:
            host = getattr(settings, 'EXPERIMENTS_REDIS_HOST', 'localhost')
            port = getattr(settings, 'EXPERIMENTS_REDIS_PORT', 6379)

        password = getattr(settings, 'EXPERIMENTS_REDIS_PASSWORD', None)
        db = getattr(settings, 'EXPERIMENTS_REDIS_DB', 0)

        return redis.Redis(host=host, port=port, password=password, db=db)
示例#50
0
 def init_shards_from_sentinel(self):
     sentinel_client = Sentinel(
         [(h, 8422) for h in self._sentinels], socket_timeout=self._timeout)
     # Connect to all the shards with the names specified per
     # shard_name_format. The names are important since it's getting
     # the instances from Redis Sentinel.
     for shard_num in range(0, self.num_shards()):
         shard_name = self.get_shard_name(shard_num)
         self._shards[shard_num] = sentinel_client.master_for(
             shard_name, socket_timeout=self._timeout)
     # Just in case we need it later.
     self._sentinel_client = sentinel_client
示例#51
0
 def client(self):
     if not self._redis_connection:
         sentinel = Sentinel(
             self.sentinel_conf.get("sentinels"),
             min_other_sentinels=self.sentinel_conf.get("min_other_sentinels", 0),
             password=self.sentinel_conf.get("password"),
             socket_timeout=self.sentinel_conf.get("sentinel_timeout", None),
         )
         redis_connection = sentinel.master_for(
             self.sentinel_conf.get("service_name"), Redis, socket_timeout=self.sentinel_conf.get("socket_timeout")
         )
         self._redis_connection.append(redis_connection)
     return self._redis_connection[0]
示例#52
0
    def init(self):
        """
        Called by Scheduler to say 'let's prepare yourself guy'
        """
        logger.info("[RedisRetention] Initialization of the redis module")
        #self.return_queue = self.properties['from_queue']
        if self.sentinel_servers:
            logger.info("[RedisRetention] sentinel_servers: %s" % str(self.sentinel_servers))
            logger.info("[RedisRetention] redis_instance: %s" % str(self.redis_instance))

            sentinel = Sentinel(self.sentinel_servers)
            self.mc = sentinel.master_for(self.redis_instance, socket_timeout=0.1)
        else:
            self.mc = redis.Redis(self.server)
 def get(self):
     if self._config.sentinel_id:
         sentinel = Sentinel([(self._config.host,
                               self._config.port)],
                             socket_timeout=self._config.timeout)
         return sentinel.master_for(self._config.sentinel_id,
                                    socket_timeout=self._config.timeout,
                                    db=self._config.db,
                                    retry_on_timeout=True)
     else:
         conn = StrictRedis(host=self._config.host, port=self._config.port,
                            db=self._config.db, retry_on_timeout=True,
                            socket_timeout=self._config.timeout)
         return conn
示例#54
0
def get_redis_connection(config, use_strict_redis=False):
    """
    Returns a redis connection from a connection config
    """
    redis_cls = redis.StrictRedis if use_strict_redis else redis.Redis

    if 'URL' in config:
        return redis_cls.from_url(config['URL'], db=config.get('DB'))
    if 'USE_REDIS_CACHE' in config.keys():

        try:
            from django.core.cache import caches
            cache = caches[config['USE_REDIS_CACHE']]
        except ImportError:
            from django.core.cache import get_cache
            cache = get_cache(config['USE_REDIS_CACHE'])

        if hasattr(cache, 'client'):
            # We're using django-redis. The cache's `client` attribute
            # is a pluggable backend that return its Redis connection as
            # its `client`
            try:
                # To get Redis connection on django-redis >= 3.4.0
                # we need to use cache.client.get_client() instead of
                # cache.client.client used in older versions
                try:
                    return cache.client.get_client()
                except AttributeError:
                    return cache.client.client
            except NotImplementedError:
                pass
        else:
            # We're using django-redis-cache
            try:
                return cache._client
            except AttributeError:
                # For django-redis-cache > 0.13.1
                return cache.get_master_client()

    if 'UNIX_SOCKET_PATH' in config:
        return redis_cls(unix_socket_path=config['UNIX_SOCKET_PATH'], db=config['DB'])

    if 'SENTINELS' in config:
        sentinel = Sentinel(config['SENTINELS'])
        return sentinel.master_for(
            service_name=config['MASTER_NAME'], redis_class=redis_cls, db=config.get('DB'),
            password=config.get('PASSWORD'), socket_timeout=config.get('SOCKET_TIMEOUT')
        )

    return redis_cls(host=config['HOST'], port=config['PORT'], db=config['DB'], password=config.get('PASSWORD'))
示例#55
0
 def __init__(self, sentinels, service, **kwargs):
     sentinel = Sentinel(sentinels, **kwargs)
     try:
         master = sentinel.discover_master(service)
     except MasterNotFoundError:
         raise
     try:
         slaves = sentinel.discover_slaves(service)
     except SlaveNotFoundError:
         self._slaves = None
     if version_info[0] < 3:
         super(AutoRedisSentinel, self).__init__(master, slaves=slaves, **kwargs)
     else:
         super().__init__(master, slaves=slaves, **kwargs)
示例#56
0
 def client(self):
     kwargs = {
         'retry_on_timeout': False,
         'socket_keepalive': None,
         'socket_connect_timeout': 1,
         'socket_timeout':  self.socket_timeout or 1,
         # redis连接池要小于进程允许的最大连接数
         'max_connections': 10240,
     }
     sentinel = Sentinel(self.sentinels, min_other_sentinels=self.min_other_sentinels, **kwargs)
     kwargs.update({
         'db': self.db,
     })
     return sentinel.master_for(self.service_name, redis_class=Redis, **kwargs)
示例#57
0
    def run(self, terms, inject=None, **kwargs):
        values = []
        terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject)

        for term in terms:
            params = self.parse_params(term)

            try:
                conn = Sentinel(params['server'], socket_timeout=1)
                res = conn.discover_master(params['name'])
                if res is None:
                    res = ""
                values.append(":".join((str(v) for v in res)))
            except Exception, e:
                print(e)
示例#58
0
    def connect(self, index=0, write=True):
        """
        Creates a redis connection with connection pool.
        """
        master_name, sentinel_hosts, db = self.parse_connection_string(self._connection_string)

        sentinel_timeout = self._options.get("SENTINEL_TIMEOUT", 1)
        sentinel = Sentinel(sentinel_hosts, socket_timeout=sentinel_timeout)

        if write:
            host, port = sentinel.discover_master(master_name)
        else:
            host, port = random.choice([sentinel.discover_master(master_name)] + sentinel.discover_slaves(master_name))

        return self.connection_factory.connect(host, port, db)