def __init__(self, connection_class=Connection, max_connections=None, failure_callback=None,
candidates=None, retry=2, depth=1, **connection_kwargs):
"""
adds `candidates` and `retry`
`candidates` is a list of maps whose members define how
failover candidate differ from each other. For example, 3
instance across 2 hosts::
>>> [dict(host='h1', port='6953'),
... dict(host='h1', port='6952'),
... dict(host='h2', port='6952')]
`retry` defines how many times the connection pool will
attempt to connect to the active candidate before failing
over using the default failure callback.
"""
self.depth = depth
self.retry = retry
if failure_callback is None:
failure_callback = self.callback
self.connection_kwargs = dict(self.default_spec,
failure_callback=failure_callback,
**connection_kwargs)
self.candidates = dict(dict(self.connection_kwargs, **c) for c in candidates)
BaseCxnPool.__init__(self, connection_class=Connection,
max_connections=None,
**connection_kwargs)
def __init__(self, hosts, **kwargs):
host = hosts[0]
if isinstance(host, str):
self.pool = ConnectionPool.from_url(host, **kwargs)
else:
db = host.get('db', 0)
self.pool = ConnectionPool(host=host['host'],
port=host['port'],
db=db,
**kwargs)
def init_pools(self, hosts, **kwargs):
# Ensure consistency of db value
kwargs.update({'db': 0})
for host in hosts:
if isinstance(host, str):
addr = self.parse_addr(host)
pool = ConnectionPool.from_url(host, **kwargs)
else:
addr = '{}:{}'.format(host['host'], host['port'])
pool = ConnectionPool(host=host['host'],
port=host['port'],
**kwargs)
self.cluster_pools[addr] = pool
self.reset_slots(**kwargs)
def __init__(self,
host='localhost',
port=6379,
db=0,
password=None,
socket_timeout=None,
connection_pool=None,
charset='utf-8',
errors='strict',
unix_socket_path=None):
if not connection_pool:
kwargs = {
'db': db,
'password': password,
'socket_timeout': socket_timeout,
'encoding': charset,
'encoding_errors': errors
}
# based on input, setup appropriate connection args
if unix_socket_path:
kwargs.update({
'path': unix_socket_path,
'connection_class': UnixDomainSocketConnection
})
else:
kwargs.update({'host': host, 'port': port})
connection_pool = ConnectionPool(**kwargs)
self.connection_pool = connection_pool
self.response_callbacks = self.__class__.RESPONSE_CALLBACKS.copy()
def test_pool(self, max_connections):
"""
A child will create its own connections when using a pool created
by a parent.
"""
pool = ConnectionPool.from_url('redis://localhost',
max_connections=max_connections)
conn = pool.get_connection('ping')
main_conn_pid = conn.pid
with exit_callback(pool.release, conn):
conn.send_command('ping')
assert conn.read_response() == b'PONG'
def target(pool):
with exit_callback(pool.disconnect):
conn = pool.get_connection('ping')
assert conn.pid != main_conn_pid
with exit_callback(pool.release, conn):
assert conn.send_command('ping') is None
assert conn.read_response() == b'PONG'
proc = multiprocessing.Process(target=target, args=(pool,))
proc.start()
proc.join(3)
assert proc.exitcode is 0
# Check that connection is still alive after fork process has exited
# and disconnected the connections in its pool
conn = pool.get_connection('ping')
with exit_callback(pool.release, conn):
assert conn.send_command('ping') is None
assert conn.read_response() == b'PONG'
def test_close_pool_in_main(self, max_connections):
"""
A child process that uses the same pool as its parent isn't affected
when the parent disconnects all connections within the pool.
"""
pool = ConnectionPool.from_url('redis://localhost',
max_connections=max_connections)
conn = pool.get_connection('ping')
assert conn.send_command('ping') is None
assert conn.read_response() == b'PONG'
def target(pool, disconnect_event):
conn = pool.get_connection('ping')
with exit_callback(pool.release, conn):
assert conn.send_command('ping') is None
assert conn.read_response() == b'PONG'
disconnect_event.wait()
assert conn.send_command('ping') is None
assert conn.read_response() == b'PONG'
ev = multiprocessing.Event()
proc = multiprocessing.Process(target=target, args=(pool, ev))
proc.start()
pool.disconnect()
ev.set()
proc.join(3)
assert proc.exitcode is 0
def test_close_pool_in_main(self, max_connections):
"""
A child process that uses the same pool as its parent isn't affected
when the parent disconnects all connections within the pool.
"""
pool = ConnectionPool.from_url('redis://localhost',
max_connections=max_connections)
conn = pool.get_connection('ping')
assert conn.send_command('ping') is None
assert conn.read_response() == b'PONG'
def target(pool, disconnect_event):
conn = pool.get_connection('ping')
with exit_callback(pool.release, conn):
assert conn.send_command('ping') is None
assert conn.read_response() == b'PONG'
disconnect_event.wait()
assert conn.send_command('ping') is None
assert conn.read_response() == b'PONG'
ev = multiprocessing.Event()
proc = multiprocessing.Process(target=target, args=(pool, ev))
proc.start()
pool.disconnect()
ev.set()
proc.join(3)
assert proc.exitcode is 0
def test_close_pool_in_main(self, max_connections, master_host):
"""
A child process that uses the same pool as its parent isn't affected
when the parent disconnects all connections within the pool.
"""
pool = ConnectionPool.from_url(
f"redis://{master_host[0]}:{master_host[1]}",
max_connections=max_connections,
)
conn = pool.get_connection("ping")
assert conn.send_command("ping") is None
assert conn.read_response() == b"PONG"
def target(pool, disconnect_event):
conn = pool.get_connection("ping")
with exit_callback(pool.release, conn):
assert conn.send_command("ping") is None
assert conn.read_response() == b"PONG"
disconnect_event.wait()
assert conn.send_command("ping") is None
assert conn.read_response() == b"PONG"
ev = multiprocessing.Event()
proc = multiprocessing.Process(target=target, args=(pool, ev))
proc.start()
pool.disconnect()
ev.set()
proc.join(3)
assert proc.exitcode == 0
def test_pool(self, max_connections):
"""
A child will create its own connections when using a pool created
by a parent.
"""
pool = ConnectionPool.from_url('redis://localhost',
max_connections=max_connections)
conn = pool.get_connection('ping')
main_conn_pid = conn.pid
with exit_callback(pool.release, conn):
conn.send_command('ping')
assert conn.read_response() == b'PONG'
def target(pool):
with exit_callback(pool.disconnect):
conn = pool.get_connection('ping')
assert conn.pid != main_conn_pid
with exit_callback(pool.release, conn):
assert conn.send_command('ping') is None
assert conn.read_response() == b'PONG'
proc = multiprocessing.Process(target=target, args=(pool, ))
proc.start()
proc.join(3)
assert proc.exitcode is 0
# Check that connection is still alive after fork process has exited
# and disconnected the connections in its pool
conn = pool.get_connection('ping')
with exit_callback(pool.release, conn):
assert conn.send_command('ping') is None
assert conn.read_response() == b'PONG'
def __init__(self,
client_ns_versions,
host='localhost',
port=6379,
db=0,
password=None,
socket_timeout=None,
socket_connect_timeout=None,
socket_keepalive=None,
socket_keepalive_options=None,
connection_pool=None,
unix_socket_path=None,
encoding='utf-8',
encoding_errors='strict',
charset=None,
errors=None,
decode_responses=False,
retry_on_timeout=False,
ssl=False,
ssl_keyfile=None,
ssl_certfile=None,
ssl_cert_reqs=None,
ssl_ca_certs=None):
kwargs = {
'db': db,
'password': password,
'socket_timeout': socket_timeout,
'encoding': encoding,
'encoding_errors': encoding_errors,
'decode_responses': decode_responses,
'retry_on_timeout': retry_on_timeout,
'fun_ptr': self.verify_ns_info
}
# TCP specific options
kwargs.update({
'host': host,
'port': port,
'socket_connect_timeout': socket_connect_timeout,
'socket_keepalive': socket_keepalive,
'socket_keepalive_options': socket_keepalive_options,
})
self.connection_pool = ConnectionPool(connection_class=NoReConnection,
**kwargs)
self._use_lua_lock = None
self.response_callbacks = self.__class__.RESPONSE_CALLBACKS.copy()
self.client_ns_versions = dict()
#dict of lists of namespace versions
self.ns_versions_dict = dict()
# check to see if we are the initial version and must init
for (ns, v) in client_ns_versions:
self.append_new_version(v, ns, startup=True)
#logging.debug(self.client_ns_versions)
# check to see if existing updates need to be loaded into this client
self.upd_dict = dict()
self.load_upd_tuples()
self.client_setname(self.ns_dict_as_string())
def setUp(self):
pool = ConnectionPool(
host='localhost', port=6379, db=9,
encoding='utf-8', decode_responses=True,
parser_class=HiredisParser)
self.client = redis.Redis(connection_pool=pool)
self.client.flushdb()
def test_pool(self, max_connections, master_host):
"""
A child will create its own connections when using a pool created
by a parent.
"""
pool = ConnectionPool.from_url(
f"redis://{master_host[0]}:{master_host[1]}",
max_connections=max_connections,
)
conn = pool.get_connection("ping")
main_conn_pid = conn.pid
with exit_callback(pool.release, conn):
conn.send_command("ping")
assert conn.read_response() == b"PONG"
def target(pool):
with exit_callback(pool.disconnect):
conn = pool.get_connection("ping")
assert conn.pid != main_conn_pid
with exit_callback(pool.release, conn):
assert conn.send_command("ping") is None
assert conn.read_response() == b"PONG"
proc = multiprocessing.Process(target=target, args=(pool, ))
proc.start()
proc.join(3)
assert proc.exitcode == 0
# Check that connection is still alive after fork process has exited
# and disconnected the connections in its pool
conn = pool.get_connection("ping")
with exit_callback(pool.release, conn):
assert conn.send_command("ping") is None
assert conn.read_response() == b"PONG"
def _construct_connection_pool(pool):
""" Construct a blocking socket connection pool based on asynchronous pool
"""
_pool = ConnectionPool(Connection, pool.max_connections,
**pool.connection_kwargs)
return _pool
class RedisClient(Redis):
"""Redis client, 使用一个全局的连接池"""
_pool = ConnectionPool(**config['redis'])
def __init__(self, **kwargs):
kwargs.setdefault('connection_pool', self._pool)
super().__init__(**kwargs)
def __init__(self,
host='localhost',
port=7711,
password=None,
socket_timeout=None,
socket_connect_timeout=None,
socket_keepalive=None,
socket_keepalive_options=None,
connection_pool=None,
unix_socket_path=None,
encoding='utf-8',
encoding_errors='strict',
decode_responses=False,
retry_on_timeout=False,
job_origin_ttl_secs=5,
record_job_origin=False):
"""
job_origin_ttl_secs is the number of seconds to store counts of
incoming jobs. The higher the throughput you're expecting, the lower
this number should be.
"""
self.record_job_origin = record_job_origin
kwargs = {
'password': password,
'socket_timeout': socket_timeout,
'encoding': encoding,
'encoding_errors': encoding_errors,
'decode_responses': decode_responses,
'retry_on_timeout': retry_on_timeout,
'db': 0,
}
# based on input, setup appropriate connection args
if unix_socket_path is not None:
kwargs.update({
'path': unix_socket_path,
'connection_class': UnixDomainSocketConnection
})
else:
# TCP specific options
kwargs.update({
'host': host,
'port': port,
'socket_connect_timeout': socket_connect_timeout,
'socket_keepalive': socket_keepalive,
'socket_keepalive_options': socket_keepalive_options,
})
if not connection_pool:
connection_pool = ConnectionPool(**kwargs)
self.response_callbacks = self.__class__.RESPONSE_CALLBACKS.copy()
self.connection_pool = {'default': connection_pool}
self.default_node = 'default'
self._job_score = RollingCounter(ttl_secs=job_origin_ttl_secs)
self.__connect_cluster(kwargs)
def __init__(self, hosts, **kwargs):
self.pools = []
for host in hosts:
pool = ConnectionPool(host=host['host'],
port=host['port'],
**kwargs)
self.pools.append(pool)
self._len = len(pool)
self.idx = 0
def __connect_cluster(self, connection_kwargs):
hi = self.hello()
self.default_node = bin_to_str(hi['id'][:8])
self.connection_pool.pop('default')
for node, ip, port, version in hi['nodes']:
connection_kwargs.update(dict(host=ip, port=port))
self.connection_pool[bin_to_str(
node[:8])] = ConnectionPool(**connection_kwargs)
def test_conflicting_init_args(self):
options = {'host': 'localhost', 'url': 'redis://localhost',
'connection_pool': ConnectionPool()}
combinations = itertools.combinations(options.items(), 2)
for kwargs in (dict(item) for item in combinations):
self.assertRaises(ConfigurationError, lambda: RedisHuey(**kwargs))
# None values are fine, however.
RedisHuey(host=None, port=None, db=None, url='redis://localhost')
def from_url(cls, url, db=None, **kwargs):
if 'max_connections' not in kwargs:
kwargs['max_connections'] = 100
kwargs.pop('timeout', None)
connection_pool = ConnectionPool.from_url(url, db=db, **kwargs)
return cls(connection_pool=connection_pool)
def test_conflicting_init_args(self):
options = {
'host': 'localhost',
'url': 'redis://localhost',
'connection_pool': ConnectionPool()
}
combinations = itertools.combinations(options.items(), 2)
for kwargs in (dict(item) for item in combinations):
self.assertRaises(ValueError, lambda: RedisStorage(**kwargs))
def normal_poll():
pool = ConnectionPool(host='127.0.0.1', port=6379, db=0)
cli = Redis(connection_pool=pool)
poller = poll()
conns = [(key, cli.connection_pool.get_connection('_')) for key in keys]
register(conns, poller)
try:
start_poll(conns, poller, cli)
except KeyboardInterrupt:
unregister(conns, poller)
def test_pool(self):
pool = ConnectionPool(host=self.ip,
port=self.port,
db=1,
max_connections=20)
r = Redis(connection_pool=pool)
r.set("huawei", "huawei")
print r.get("huawei")
r.delete("huawei")
print r.ping()
print r.get("huawei")
print 2**2
def __init__(self,
ip="",
port=6379,
defalutDb=1,
max_connections=30,
password=""):
self.pool = ConnectionPool(host=ip,
port=port,
db=defalutDb,
max_connections=max_connections,
password=password)
self.redis = Redis(connection_pool=self.pool)
def _shared_pool(**opts):
if 'host' in opts:
key = '%s:%s/%s' % (opts['host'], opts['port'], opts['db'], )
else:
key = '%s/%s' % (opts['path'], opts['db'])
pool = _pool_cache.get(key)
if pool is not None:
return pool
with _pool_lock:
pool = _pool_cache.get(key)
if pool is not None:
return pool
pool = ConnectionPool(**opts)
_pool_cache[key] = pool
return pool
def get_pool(self, key=None):
if not key:
pool = self.get_random_pool()
return pool
addr = self._key_to_addr(key)
if addr:
pool = self.cluster_pools.get(addr)
if not pool:
host, port = addr.split(':')
pool = ConnectionPool(host=host,
port=int(port),
**self.pool_kwargs)
self.cluster_pools[addr] = pool
return pool
raise ClusterError("No slot found for key...", key)
def _shared_pool(**opts):
if "host" in opts:
key = "{}:{}/{}".format(opts["host"], opts["port"], opts["db"])
else:
key = "{}/{}".format(opts["path"], opts["db"])
pool = _pool_cache.get(key)
if pool is not None:
return pool
with _pool_lock:
pool = _pool_cache.get(key)
if pool is not None:
return pool
pool = ConnectionPool(**opts)
_pool_cache[key] = pool
return pool
def from_url(cls, url, **kwargs):
"""
Return a Disque client object configured from the given URL.
For example::
disque://[:password]@localhost:6379
unix://[:password]@/path/to/socket.sock
Any additional querystring arguments and keyword arguments will be
passed along to the ConnectionPool class's initializer. In the case
of conflicting arguments, querystring arguments always win.
"""
connection_pool = ConnectionPool.from_url(url, **kwargs)
return cls(connection_pool=connection_pool)
def from_url(cls, url, **kwargs):
"""
Return a Disque client object configured from the given URL.
For example::
disque://[:password]@localhost:6379
unix://[:password]@/path/to/socket.sock
Any additional querystring arguments and keyword arguments will be
passed along to the ConnectionPool class's initializer. In the case
of conflicting arguments, querystring arguments always win.
"""
connection_pool = ConnectionPool.from_url(url, **kwargs)
return cls(connection_pool=connection_pool)
def __init__(self,
host='localhost',
port=6379,
db=0,
password=None,
connection_pool=None,
max_connections=None):
if not connection_pool:
kwargs = {
'host': host,
'port': port,
'db': db,
'password': password,
'max_connections': max_connections
}
connection_pool = ConnectionPool(**kwargs)
self.connection_pool = connection_pool
def main(argv=sys.argv):
if len(argv) < 2:
usage(argv)
config_uri = argv[1]
options = parse_vars(argv[2:])
setup_logging(config_uri)
settings = get_appsettings(config_uri, options=options)
logging.getLogger('c2corg_ui').setLevel(logging.INFO)
log.info('Cache Redis: {0}'.format(settings['redis.url']))
# we don't really need a connection pool here, but the `from_url`
# function is convenient
redis_pool = ConnectionPool.from_url(
settings['redis.url'], max_connections=1)
# remove all keys from the database
r = Redis(connection_pool=redis_pool)
r.flushdb()
log.info('Flushed cache')
def main(argv=sys.argv):
if len(argv) < 2:
usage(argv)
config_uri = argv[1]
options = parse_vars(argv[2:])
setup_logging(config_uri)
settings = get_appsettings(config_uri, options=options)
logging.getLogger('c2corg_ui').setLevel(logging.INFO)
log.info('Cache Redis: {0}'.format(settings['redis.url']))
# we don't really need a connection pool here, but the `from_url`
# function is convenient
redis_pool = ConnectionPool.from_url(
settings['redis.url'], max_connections=1)
# remove all keys from the database
r = Redis(connection_pool=redis_pool)
r.flushdb()
log.info('Flushed cache')
def main(argv=sys.argv):
if len(argv) < 2:
usage(argv)
config_uri = argv[1]
options = parse_vars(argv[2:])
setup_logging(config_uri)
settings = get_appsettings(config_uri, options=options)
logging.getLogger("c2corg_api").setLevel(logging.INFO)
redis_url = "{0}?db={1}".format(settings["redis.url"], settings["redis.db_cache"])
log.info("Cache Redis: {0}".format(redis_url))
# we don't really need a connection pool here, but the `from_url`
# function is convenient
redis_pool = ConnectionPool.from_url(redis_url, max_connections=1)
# remove all keys from the database
r = Redis(connection_pool=redis_pool)
r.flushdb()
log.info("Flushed cache")
def init_app(self, app):
config = Config(app)
redis.connection_pool = ConnectionPool.from_url(
config.value('REDIS_URL'))
self.register_blueprint(app)
if config.value('TOKEN_URL') is not False:
app.add_url_rule(config.url_rule_for('TOKEN_URL'),
view_func=views.access_token,
methods=['POST'])
if config.value('MANAGEMENT_URL') is not False:
app.add_url_rule(config.url_rule_for('MANAGEMENT_URL'),
view_func=views.management,
methods=['POST', 'GET'])
mongo.init_app(app)
self.mongo = mongo
oauth.init_app(app)
oauth._validator = MyRequestValidator()
def reset_slots(self, **kwargs):
pool = self.get_random_pool()
resp = pool.execute_command('CLUSTER', 'SLOTS')
if not isinstance(resp, (list, tuple)):
raise RedisError('Unable to locate redis slots.')
_pools = {}
for elem in resp:
_start, _end, master = elem[0], elem[1], elem[2]
ip, port = self._stringconv(master[0]), self._stringconv(master[1])
addr = '{}:{}'.format(ip, port)
for i in range(int(_start), int(_end) + 1):
self.mapping[i] = addr
if addr not in _pools:
p = (self.cluster_pools.pop(addr, None)
or ConnectionPool(host=ip, port=port, **kwargs))
p.addr = addr
_pools[addr] = p
self.cluster_pools.clear()
self.cluster_pools = _pools
def init_app(self, app):
config = Config(app)
redis.connection_pool = ConnectionPool.from_url(
config.value('REDIS_URL'))
self.register_blueprint(app)
if config.value('TOKEN_URL') is not False:
app.add_url_rule(
config.url_rule_for('TOKEN_URL'),
view_func=views.access_token,
methods=['POST']
)
if config.value('MANAGEMENT_URL') is not False:
app.add_url_rule(
config.url_rule_for('MANAGEMENT_URL'),
view_func=views.management,
methods=['POST', 'GET']
)
mongo.init_app(app, config_prefix='SENTINEL_MONGO')
self.mongo = mongo
oauth.init_app(app)
oauth._validator = MyRequestValidator()
def from_url(cls, url, db=None, **kwargs):
"""
Return a Redis client object configured from the given URL.
For example::
redis://[:password]@localhost:6379/0
unix://[:password]@/path/to/socket.sock?db=0
There are several ways to specify a database number. The parse function
will return the first specified option:
1. A ``db`` querystring option, e.g. redis://localhost?db=0
2. If using the redis:// scheme, the path argument of the url, e.g.
redis://localhost/0
3. The ``db`` argument to this function.
If none of these options are specified, db=0 is used.
Any additional querystring arguments and keyword arguments will be
passed along to the ConnectionPool class's initializer. In the case
of conflicting arguments, querystring arguments always win.
"""
connection_pool = ConnectionPool.from_url(url, db=db, **kwargs)
return cls(connection_pool=connection_pool)
# -*- coding: utf-8 -*-
import redis
import threading
import logging
from redis.connection import ConnectionPool
redis_settings = {"servers": "localhost", "port": 6379, "db": 11}
_pool0 = ConnectionPool(**redis_settings)
class RedisClient(object):
instance = None
locker = threading.Lock()
def __init__(self):
""" intialize the client of redis include port db and servers """
try:
self.servers = redis_settings["servers"]
self.port = redis_settings["port"]
self.db = redis_settings["db"]
self.redis = redis.Redis(connection_pool=_pool0)
except Exception, e:
logging.error(e)
@classmethod
def get_instance(cls):
"""
get the instance of RedisClient
from redis.connection import ConnectionPool from redis.client import StrictRedis from .config import get_config pool = ConnectionPool.from_url(get_config()["REDIS_URL"]) redis_store = StrictRedis(connection_pool=pool)
from flask import Flask
from flask.ext.sentinel import ResourceOwnerPasswordCredentials, oauth
from flask.ext.sentinel.core import redis
from redis.connection import ConnectionPool
app = Flask(__name__)
app.config.from_object('setup')
# workaround on a redis connection bug in flask-sentinel v0.0.2
redis.connection_pool=ConnectionPool.from_url(app.config['SENTINEL_REDIS_URL'])
ResourceOwnerPasswordCredentials(app)
@app.route('/endpoint')
@oauth.require_oauth()
def restricted_access():
return "You made it through and accessed the protected resource!"
if __name__ == '__main__':
app.run(ssl_context='adhoc')
