def __init__(self, connection_class=Connection, max_connections=None, failure_callback=None, candidates=None, retry=2, depth=1, **connection_kwargs): """ adds `candidates` and `retry` `candidates` is a list of maps whose members define how failover candidate differ from each other. For example, 3 instance across 2 hosts:: >>> [dict(host='h1', port='6953'), ... dict(host='h1', port='6952'), ... dict(host='h2', port='6952')] `retry` defines how many times the connection pool will attempt to connect to the active candidate before failing over using the default failure callback. """ self.depth = depth self.retry = retry if failure_callback is None: failure_callback = self.callback self.connection_kwargs = dict(self.default_spec, failure_callback=failure_callback, **connection_kwargs) self.candidates = dict(dict(self.connection_kwargs, **c) for c in candidates) BaseCxnPool.__init__(self, connection_class=Connection, max_connections=None, **connection_kwargs)
def __init__(self, hosts, **kwargs): host = hosts[0] if isinstance(host, str): self.pool = ConnectionPool.from_url(host, **kwargs) else: db = host.get('db', 0) self.pool = ConnectionPool(host=host['host'], port=host['port'], db=db, **kwargs)
def init_pools(self, hosts, **kwargs): # Ensure consistency of db value kwargs.update({'db': 0}) for host in hosts: if isinstance(host, str): addr = self.parse_addr(host) pool = ConnectionPool.from_url(host, **kwargs) else: addr = '{}:{}'.format(host['host'], host['port']) pool = ConnectionPool(host=host['host'], port=host['port'], **kwargs) self.cluster_pools[addr] = pool self.reset_slots(**kwargs)
def __init__(self, host='localhost', port=6379, db=0, password=None, socket_timeout=None, connection_pool=None, charset='utf-8', errors='strict', unix_socket_path=None): if not connection_pool: kwargs = { 'db': db, 'password': password, 'socket_timeout': socket_timeout, 'encoding': charset, 'encoding_errors': errors } # based on input, setup appropriate connection args if unix_socket_path: kwargs.update({ 'path': unix_socket_path, 'connection_class': UnixDomainSocketConnection }) else: kwargs.update({'host': host, 'port': port}) connection_pool = ConnectionPool(**kwargs) self.connection_pool = connection_pool self.response_callbacks = self.__class__.RESPONSE_CALLBACKS.copy()
def test_pool(self, max_connections): """ A child will create its own connections when using a pool created by a parent. """ pool = ConnectionPool.from_url('redis://localhost', max_connections=max_connections) conn = pool.get_connection('ping') main_conn_pid = conn.pid with exit_callback(pool.release, conn): conn.send_command('ping') assert conn.read_response() == b'PONG' def target(pool): with exit_callback(pool.disconnect): conn = pool.get_connection('ping') assert conn.pid != main_conn_pid with exit_callback(pool.release, conn): assert conn.send_command('ping') is None assert conn.read_response() == b'PONG' proc = multiprocessing.Process(target=target, args=(pool,)) proc.start() proc.join(3) assert proc.exitcode is 0 # Check that connection is still alive after fork process has exited # and disconnected the connections in its pool conn = pool.get_connection('ping') with exit_callback(pool.release, conn): assert conn.send_command('ping') is None assert conn.read_response() == b'PONG'
def test_close_pool_in_main(self, max_connections): """ A child process that uses the same pool as its parent isn't affected when the parent disconnects all connections within the pool. """ pool = ConnectionPool.from_url('redis://localhost', max_connections=max_connections) conn = pool.get_connection('ping') assert conn.send_command('ping') is None assert conn.read_response() == b'PONG' def target(pool, disconnect_event): conn = pool.get_connection('ping') with exit_callback(pool.release, conn): assert conn.send_command('ping') is None assert conn.read_response() == b'PONG' disconnect_event.wait() assert conn.send_command('ping') is None assert conn.read_response() == b'PONG' ev = multiprocessing.Event() proc = multiprocessing.Process(target=target, args=(pool, ev)) proc.start() pool.disconnect() ev.set() proc.join(3) assert proc.exitcode is 0
def test_close_pool_in_main(self, max_connections, master_host): """ A child process that uses the same pool as its parent isn't affected when the parent disconnects all connections within the pool. """ pool = ConnectionPool.from_url( f"redis://{master_host[0]}:{master_host[1]}", max_connections=max_connections, ) conn = pool.get_connection("ping") assert conn.send_command("ping") is None assert conn.read_response() == b"PONG" def target(pool, disconnect_event): conn = pool.get_connection("ping") with exit_callback(pool.release, conn): assert conn.send_command("ping") is None assert conn.read_response() == b"PONG" disconnect_event.wait() assert conn.send_command("ping") is None assert conn.read_response() == b"PONG" ev = multiprocessing.Event() proc = multiprocessing.Process(target=target, args=(pool, ev)) proc.start() pool.disconnect() ev.set() proc.join(3) assert proc.exitcode == 0
def test_pool(self, max_connections): """ A child will create its own connections when using a pool created by a parent. """ pool = ConnectionPool.from_url('redis://localhost', max_connections=max_connections) conn = pool.get_connection('ping') main_conn_pid = conn.pid with exit_callback(pool.release, conn): conn.send_command('ping') assert conn.read_response() == b'PONG' def target(pool): with exit_callback(pool.disconnect): conn = pool.get_connection('ping') assert conn.pid != main_conn_pid with exit_callback(pool.release, conn): assert conn.send_command('ping') is None assert conn.read_response() == b'PONG' proc = multiprocessing.Process(target=target, args=(pool, )) proc.start() proc.join(3) assert proc.exitcode is 0 # Check that connection is still alive after fork process has exited # and disconnected the connections in its pool conn = pool.get_connection('ping') with exit_callback(pool.release, conn): assert conn.send_command('ping') is None assert conn.read_response() == b'PONG'
def __init__(self, client_ns_versions, host='localhost', port=6379, db=0, password=None, socket_timeout=None, socket_connect_timeout=None, socket_keepalive=None, socket_keepalive_options=None, connection_pool=None, unix_socket_path=None, encoding='utf-8', encoding_errors='strict', charset=None, errors=None, decode_responses=False, retry_on_timeout=False, ssl=False, ssl_keyfile=None, ssl_certfile=None, ssl_cert_reqs=None, ssl_ca_certs=None): kwargs = { 'db': db, 'password': password, 'socket_timeout': socket_timeout, 'encoding': encoding, 'encoding_errors': encoding_errors, 'decode_responses': decode_responses, 'retry_on_timeout': retry_on_timeout, 'fun_ptr': self.verify_ns_info } # TCP specific options kwargs.update({ 'host': host, 'port': port, 'socket_connect_timeout': socket_connect_timeout, 'socket_keepalive': socket_keepalive, 'socket_keepalive_options': socket_keepalive_options, }) self.connection_pool = ConnectionPool(connection_class=NoReConnection, **kwargs) self._use_lua_lock = None self.response_callbacks = self.__class__.RESPONSE_CALLBACKS.copy() self.client_ns_versions = dict() #dict of lists of namespace versions self.ns_versions_dict = dict() # check to see if we are the initial version and must init for (ns, v) in client_ns_versions: self.append_new_version(v, ns, startup=True) #logging.debug(self.client_ns_versions) # check to see if existing updates need to be loaded into this client self.upd_dict = dict() self.load_upd_tuples() self.client_setname(self.ns_dict_as_string())
def setUp(self): pool = ConnectionPool( host='localhost', port=6379, db=9, encoding='utf-8', decode_responses=True, parser_class=HiredisParser) self.client = redis.Redis(connection_pool=pool) self.client.flushdb()
def test_pool(self, max_connections, master_host): """ A child will create its own connections when using a pool created by a parent. """ pool = ConnectionPool.from_url( f"redis://{master_host[0]}:{master_host[1]}", max_connections=max_connections, ) conn = pool.get_connection("ping") main_conn_pid = conn.pid with exit_callback(pool.release, conn): conn.send_command("ping") assert conn.read_response() == b"PONG" def target(pool): with exit_callback(pool.disconnect): conn = pool.get_connection("ping") assert conn.pid != main_conn_pid with exit_callback(pool.release, conn): assert conn.send_command("ping") is None assert conn.read_response() == b"PONG" proc = multiprocessing.Process(target=target, args=(pool, )) proc.start() proc.join(3) assert proc.exitcode == 0 # Check that connection is still alive after fork process has exited # and disconnected the connections in its pool conn = pool.get_connection("ping") with exit_callback(pool.release, conn): assert conn.send_command("ping") is None assert conn.read_response() == b"PONG"
def _construct_connection_pool(pool): """ Construct a blocking socket connection pool based on asynchronous pool """ _pool = ConnectionPool(Connection, pool.max_connections, **pool.connection_kwargs) return _pool
class RedisClient(Redis): """Redis client, 使用一个全局的连接池""" _pool = ConnectionPool(**config['redis']) def __init__(self, **kwargs): kwargs.setdefault('connection_pool', self._pool) super().__init__(**kwargs)
def __init__(self, host='localhost', port=7711, password=None, socket_timeout=None, socket_connect_timeout=None, socket_keepalive=None, socket_keepalive_options=None, connection_pool=None, unix_socket_path=None, encoding='utf-8', encoding_errors='strict', decode_responses=False, retry_on_timeout=False, job_origin_ttl_secs=5, record_job_origin=False): """ job_origin_ttl_secs is the number of seconds to store counts of incoming jobs. The higher the throughput you're expecting, the lower this number should be. """ self.record_job_origin = record_job_origin kwargs = { 'password': password, 'socket_timeout': socket_timeout, 'encoding': encoding, 'encoding_errors': encoding_errors, 'decode_responses': decode_responses, 'retry_on_timeout': retry_on_timeout, 'db': 0, } # based on input, setup appropriate connection args if unix_socket_path is not None: kwargs.update({ 'path': unix_socket_path, 'connection_class': UnixDomainSocketConnection }) else: # TCP specific options kwargs.update({ 'host': host, 'port': port, 'socket_connect_timeout': socket_connect_timeout, 'socket_keepalive': socket_keepalive, 'socket_keepalive_options': socket_keepalive_options, }) if not connection_pool: connection_pool = ConnectionPool(**kwargs) self.response_callbacks = self.__class__.RESPONSE_CALLBACKS.copy() self.connection_pool = {'default': connection_pool} self.default_node = 'default' self._job_score = RollingCounter(ttl_secs=job_origin_ttl_secs) self.__connect_cluster(kwargs)
def __init__(self, hosts, **kwargs): self.pools = [] for host in hosts: pool = ConnectionPool(host=host['host'], port=host['port'], **kwargs) self.pools.append(pool) self._len = len(pool) self.idx = 0
def __connect_cluster(self, connection_kwargs): hi = self.hello() self.default_node = bin_to_str(hi['id'][:8]) self.connection_pool.pop('default') for node, ip, port, version in hi['nodes']: connection_kwargs.update(dict(host=ip, port=port)) self.connection_pool[bin_to_str( node[:8])] = ConnectionPool(**connection_kwargs)
def test_conflicting_init_args(self): options = {'host': 'localhost', 'url': 'redis://localhost', 'connection_pool': ConnectionPool()} combinations = itertools.combinations(options.items(), 2) for kwargs in (dict(item) for item in combinations): self.assertRaises(ConfigurationError, lambda: RedisHuey(**kwargs)) # None values are fine, however. RedisHuey(host=None, port=None, db=None, url='redis://localhost')
def from_url(cls, url, db=None, **kwargs): if 'max_connections' not in kwargs: kwargs['max_connections'] = 100 kwargs.pop('timeout', None) connection_pool = ConnectionPool.from_url(url, db=db, **kwargs) return cls(connection_pool=connection_pool)
def test_conflicting_init_args(self): options = { 'host': 'localhost', 'url': 'redis://localhost', 'connection_pool': ConnectionPool() } combinations = itertools.combinations(options.items(), 2) for kwargs in (dict(item) for item in combinations): self.assertRaises(ValueError, lambda: RedisStorage(**kwargs))
def normal_poll(): pool = ConnectionPool(host='127.0.0.1', port=6379, db=0) cli = Redis(connection_pool=pool) poller = poll() conns = [(key, cli.connection_pool.get_connection('_')) for key in keys] register(conns, poller) try: start_poll(conns, poller, cli) except KeyboardInterrupt: unregister(conns, poller)
def test_pool(self): pool = ConnectionPool(host=self.ip, port=self.port, db=1, max_connections=20) r = Redis(connection_pool=pool) r.set("huawei", "huawei") print r.get("huawei") r.delete("huawei") print r.ping() print r.get("huawei") print 2**2
def __init__(self, ip="", port=6379, defalutDb=1, max_connections=30, password=""): self.pool = ConnectionPool(host=ip, port=port, db=defalutDb, max_connections=max_connections, password=password) self.redis = Redis(connection_pool=self.pool)
def _shared_pool(**opts): if 'host' in opts: key = '%s:%s/%s' % (opts['host'], opts['port'], opts['db'], ) else: key = '%s/%s' % (opts['path'], opts['db']) pool = _pool_cache.get(key) if pool is not None: return pool with _pool_lock: pool = _pool_cache.get(key) if pool is not None: return pool pool = ConnectionPool(**opts) _pool_cache[key] = pool return pool
def get_pool(self, key=None): if not key: pool = self.get_random_pool() return pool addr = self._key_to_addr(key) if addr: pool = self.cluster_pools.get(addr) if not pool: host, port = addr.split(':') pool = ConnectionPool(host=host, port=int(port), **self.pool_kwargs) self.cluster_pools[addr] = pool return pool raise ClusterError("No slot found for key...", key)
def _shared_pool(**opts): if "host" in opts: key = "{}:{}/{}".format(opts["host"], opts["port"], opts["db"]) else: key = "{}/{}".format(opts["path"], opts["db"]) pool = _pool_cache.get(key) if pool is not None: return pool with _pool_lock: pool = _pool_cache.get(key) if pool is not None: return pool pool = ConnectionPool(**opts) _pool_cache[key] = pool return pool
def from_url(cls, url, **kwargs): """ Return a Disque client object configured from the given URL. For example:: disque://[:password]@localhost:6379 unix://[:password]@/path/to/socket.sock Any additional querystring arguments and keyword arguments will be passed along to the ConnectionPool class's initializer. In the case of conflicting arguments, querystring arguments always win. """ connection_pool = ConnectionPool.from_url(url, **kwargs) return cls(connection_pool=connection_pool)
def __init__(self, host='localhost', port=6379, db=0, password=None, connection_pool=None, max_connections=None): if not connection_pool: kwargs = { 'host': host, 'port': port, 'db': db, 'password': password, 'max_connections': max_connections } connection_pool = ConnectionPool(**kwargs) self.connection_pool = connection_pool
def main(argv=sys.argv): if len(argv) < 2: usage(argv) config_uri = argv[1] options = parse_vars(argv[2:]) setup_logging(config_uri) settings = get_appsettings(config_uri, options=options) logging.getLogger('c2corg_ui').setLevel(logging.INFO) log.info('Cache Redis: {0}'.format(settings['redis.url'])) # we don't really need a connection pool here, but the `from_url` # function is convenient redis_pool = ConnectionPool.from_url( settings['redis.url'], max_connections=1) # remove all keys from the database r = Redis(connection_pool=redis_pool) r.flushdb() log.info('Flushed cache')
def main(argv=sys.argv): if len(argv) < 2: usage(argv) config_uri = argv[1] options = parse_vars(argv[2:]) setup_logging(config_uri) settings = get_appsettings(config_uri, options=options) logging.getLogger("c2corg_api").setLevel(logging.INFO) redis_url = "{0}?db={1}".format(settings["redis.url"], settings["redis.db_cache"]) log.info("Cache Redis: {0}".format(redis_url)) # we don't really need a connection pool here, but the `from_url` # function is convenient redis_pool = ConnectionPool.from_url(redis_url, max_connections=1) # remove all keys from the database r = Redis(connection_pool=redis_pool) r.flushdb() log.info("Flushed cache")
def init_app(self, app): config = Config(app) redis.connection_pool = ConnectionPool.from_url( config.value('REDIS_URL')) self.register_blueprint(app) if config.value('TOKEN_URL') is not False: app.add_url_rule(config.url_rule_for('TOKEN_URL'), view_func=views.access_token, methods=['POST']) if config.value('MANAGEMENT_URL') is not False: app.add_url_rule(config.url_rule_for('MANAGEMENT_URL'), view_func=views.management, methods=['POST', 'GET']) mongo.init_app(app) self.mongo = mongo oauth.init_app(app) oauth._validator = MyRequestValidator()
def reset_slots(self, **kwargs): pool = self.get_random_pool() resp = pool.execute_command('CLUSTER', 'SLOTS') if not isinstance(resp, (list, tuple)): raise RedisError('Unable to locate redis slots.') _pools = {} for elem in resp: _start, _end, master = elem[0], elem[1], elem[2] ip, port = self._stringconv(master[0]), self._stringconv(master[1]) addr = '{}:{}'.format(ip, port) for i in range(int(_start), int(_end) + 1): self.mapping[i] = addr if addr not in _pools: p = (self.cluster_pools.pop(addr, None) or ConnectionPool(host=ip, port=port, **kwargs)) p.addr = addr _pools[addr] = p self.cluster_pools.clear() self.cluster_pools = _pools
def init_app(self, app): config = Config(app) redis.connection_pool = ConnectionPool.from_url( config.value('REDIS_URL')) self.register_blueprint(app) if config.value('TOKEN_URL') is not False: app.add_url_rule( config.url_rule_for('TOKEN_URL'), view_func=views.access_token, methods=['POST'] ) if config.value('MANAGEMENT_URL') is not False: app.add_url_rule( config.url_rule_for('MANAGEMENT_URL'), view_func=views.management, methods=['POST', 'GET'] ) mongo.init_app(app, config_prefix='SENTINEL_MONGO') self.mongo = mongo oauth.init_app(app) oauth._validator = MyRequestValidator()
def from_url(cls, url, db=None, **kwargs): """ Return a Redis client object configured from the given URL. For example:: redis://[:password]@localhost:6379/0 unix://[:password]@/path/to/socket.sock?db=0 There are several ways to specify a database number. The parse function will return the first specified option: 1. A ``db`` querystring option, e.g. redis://localhost?db=0 2. If using the redis:// scheme, the path argument of the url, e.g. redis://localhost/0 3. The ``db`` argument to this function. If none of these options are specified, db=0 is used. Any additional querystring arguments and keyword arguments will be passed along to the ConnectionPool class's initializer. In the case of conflicting arguments, querystring arguments always win. """ connection_pool = ConnectionPool.from_url(url, db=db, **kwargs) return cls(connection_pool=connection_pool)
# -*- coding: utf-8 -*- import redis import threading import logging from redis.connection import ConnectionPool redis_settings = {"servers": "localhost", "port": 6379, "db": 11} _pool0 = ConnectionPool(**redis_settings) class RedisClient(object): instance = None locker = threading.Lock() def __init__(self): """ intialize the client of redis include port db and servers """ try: self.servers = redis_settings["servers"] self.port = redis_settings["port"] self.db = redis_settings["db"] self.redis = redis.Redis(connection_pool=_pool0) except Exception, e: logging.error(e) @classmethod def get_instance(cls): """ get the instance of RedisClient
from redis.connection import ConnectionPool from redis.client import StrictRedis from .config import get_config pool = ConnectionPool.from_url(get_config()["REDIS_URL"]) redis_store = StrictRedis(connection_pool=pool)
from flask import Flask from flask.ext.sentinel import ResourceOwnerPasswordCredentials, oauth from flask.ext.sentinel.core import redis from redis.connection import ConnectionPool app = Flask(__name__) app.config.from_object('setup') # workaround on a redis connection bug in flask-sentinel v0.0.2 redis.connection_pool=ConnectionPool.from_url(app.config['SENTINEL_REDIS_URL']) ResourceOwnerPasswordCredentials(app) @app.route('/endpoint') @oauth.require_oauth() def restricted_access(): return "You made it through and accessed the protected resource!" if __name__ == '__main__': app.run(ssl_context='adhoc')