def __init__(self, ci):
log.debug('create connection = %s', ci)
t = ci.type
self.t = t
if t == 1:
log.debug('create redis connection.')
self.conn = StrictRedis(host=ci.host, port=ci.port, db=ci.db)
elif t == 2:
log.debug('create redis cluster connection.')
nodes = json.loads(ci.host)
pool = ClusterConnectionPool(startup_nodes=nodes)
self.conn = StrictRedisCluster(connection_pool=pool, decode_responses=True)
elif t == 3:
log.debug('create redis connection from zookeeper.')
client = zk.Client(hosts=ci.host, read_only=True)
node = client.get(ci.path)
arr = str(node[0], encoding='utf-8').split('\n')
address = []
for h in arr:
if h is '':
continue
a = h.split(':')
address.append({'host': a[0], 'port': int(a[1])})
pool = ClusterConnectionPool(startup_nodes=address)
self.conn = StrictRedisCluster(connection_pool=pool, decode_responses=True)
else:
raise AttributeError('illegal ConnInfo type.')
if self.test():
self.ci = ci
log.info('connect redis(%s) success', ci.host)
def __init__(self, redis_server, pw=None):
#self.redis_client_ = redis.Redis(host='127.0.0.1', port=9966)
if pw:
pool = ClusterConnectionPool(startup_nodes=redis_server, password=pw, skip_full_coverage_check=True, decode_responses=True)
self.redis_client_ = StrictRedisCluster(connection_pool=pool)
else:
pool = ClusterConnectionPool(startup_nodes=redis_server, skip_full_coverage_check=True, decode_responses=True)
self.redis_client_ = StrictRedisCluster(connection_pool=pool)
def getCon(cls, confSection, confFile='/config/test.yml'):
"""从redis的连接池中获取一个redis连接，如果没有则创建一个连接池
:param confSection: 配置的section名
:type confSection: string
"""
try:
key = confSection
type = ConfigUtil.get(confSection, 'Type', confFile)
if key not in cls.pools:
server = ConfigUtil.get(confSection, 'Server', confFile)
if type == 'Redis':
host, port, db = server.split(':')
passWd = ConfigUtil.get(confSection, 'Passwd', confFile)
pool = redis.ConnectionPool(host=host, port=port, db=db, password=passWd)
elif type == 'Redis_Cluster':
startup_nodes = [{"host": s.split(':')[0], "port": s.split(':')[1]} for s in server.split(',')]
pool = ClusterConnectionPool(startup_nodes=startup_nodes)
cls.pools[key] = pool
else:
pass
pool = cls.pools[key]
if type == 'Redis':
r = redis.Redis(connection_pool=pool)
elif type == 'Redis_Cluster':
r = StrictRedisCluster(connection_pool=pool, decode_responses=True)
return r
except Exception as e:
print(e)
def test_pool(self, max_connections):
"""
A child will create its own connections when using a pool created
by a parent.
"""
pool = ClusterConnectionPool.from_url('redis://localhost:7000',
max_connections=max_connections)
conn = pool.get_random_connection()
main_conn_pid = conn.pid
with exit_callback(pool.release, conn):
conn.send_command('ping')
assert conn.read_response() == b'PONG'
def target(pool):
with exit_callback(pool.disconnect):
conn = pool.get_random_connection()
assert conn.pid != main_conn_pid
with exit_callback(pool.release, conn):
assert conn.send_command('ping') is None
assert conn.read_response() == b'PONG'
proc = multiprocessing.Process(target=target, args=(pool, ))
proc.start()
proc.join(3)
assert proc.exitcode is 0
# Check that connection is still alive after fork process has exited
# and disconnected the connections in its pool
conn = pool.get_random_connection()
with exit_callback(pool.release, conn):
assert conn.send_command('ping') is None
assert conn.read_response() == b'PONG'
def test_close_pool_in_main(self, max_connections):
"""
A child process that uses the same pool as its parent isn't affected
when the parent disconnects all connections within the pool.
"""
pool = ClusterConnectionPool.from_url('redis://localhost:7000',
max_connections=max_connections)
conn = pool.get_random_connection()
assert conn.send_command('ping') is None
assert conn.read_response() == b'PONG'
def target(pool, disconnect_event):
conn = pool.get_random_connection()
with exit_callback(pool.release, conn):
assert conn.send_command('ping') is None
assert conn.read_response() == b'PONG'
disconnect_event.wait()
assert conn.send_command('ping') is None
assert conn.read_response() == b'PONG'
ev = multiprocessing.Event()
proc = multiprocessing.Process(target=target, args=(pool, ev))
proc.start()
pool.disconnect()
ev.set()
proc.join(3)
assert proc.exitcode is 0
def __init__(self,
cluster_host_port_mapping=[{
"host": "127.0.0.1",
"port": "6379"
}],
ttl=60,
cache_size=3):
self.startup_nodes = cluster_host_port_mapping
self.ttl = ttl
self.conn = None
self.max_conn = 50
self.max_conn_per_node = 10
self.cache_size = cache_size
self.CACHE_KEYS = "cache_lru_keys"
self.CACHE_STORE = "cache_lru_store"
self.pool = ClusterConnectionPool(
max_connections=self.max_conn,
max_connections_per_node=self.max_conn_per_node,
startup_nodes=self.startup_nodes,
init_slot_cache=True,
skip_full_coverage_check=True)
# Flushes keys so that results are consistent for the example client code. Not to be used in real-world deployment.
self.flush_all()
def get_pool(self, connection_kwargs=None, max_connections=None, connection_class=DummyConnection):
connection_kwargs = connection_kwargs or {}
pool = ClusterConnectionPool(
connection_class=connection_class,
max_connections=max_connections,
startup_nodes=[{"host": "127.0.0.1", "port": 7000}],
**connection_kwargs)
return pool
def _get_connection_pool(self, params):
'''创建连接池'''
cp_params = dict(params)
cp_params.update({'decode_responses': True})
startup_nodes = self._parse_startup_nodes()
pool = ClusterConnectionPool(startup_nodes, **cp_params)
return pool
def get_pool(connection_kwargs=None, max_connections=None, max_connections_per_node=None, connection_class=DummyConnection, init_slot_cache=True):
connection_kwargs = connection_kwargs or {}
pool = ClusterConnectionPool(
connection_class=connection_class,
max_connections=max_connections,
max_connections_per_node=max_connections_per_node,
startup_nodes=[{"host": "127.0.0.1", "port": 7000}],
init_slot_cache=init_slot_cache,
**connection_kwargs)
return pool
def get_redis():
"""
Obtains a cluster aware redis connection.
:return ClusterConnectionPool: The redis connection pool
"""
global _redis_instance
# Due to a pending release from the `redis-py-cluster` project, pulling in
# connection code here. Once https://github.com/Grokzen/redis-py-cluster/pull/195
# is released, this can be reset to the original line (left for ease of transition).
# return rediscluster.RedisCluster.from_url(redis_config.URL, skip_full_coverage_check=True)
if _redis_instance is None:
connection_pool = ClusterConnectionPool.from_url(redis_config.URL, skip_full_coverage_check=True)
_redis_instance = rediscluster.RedisCluster(connection_pool=connection_pool, skip_full_coverage_check=True)
return _redis_instance
def __new__(cls, *args, **kwargs):
pid = os.getpid()
if not hasattr(cls, '_instance') or pid != cls._pid:
print("PID is {} and father PID is {}".format(
os.getpid(), os.getppid()))
if hasattr(cls, "_pid"):
print("Instance PID is {} and PID is {}".format(cls._pid, pid))
if REDIS_CLUSTER:
pool = ClusterConnectionPool(*args, **kwargs)
cls._instance = RedisCluster(connection_pool=pool,
decode_responses=True,
password=REDIS_PASSWORD)
cls._pid = pid
else:
pool = ConnectionPool(*args, **kwargs)
cls._instance = Redis(connection_pool=pool,
password=REDIS_PASSWORD)
cls._pid = pid
return cls._instance
def getClusterConnection(self):
if self.useTLS:
# workaround for error on
# got an unexpected keyword argument 'ssl'
# we enforce the connection_class instead of setting ssl=True
pool = ClusterConnectionPool(
startup_nodes=self.getMasterNodesList(),
connection_class=SSLClusterConnection,
ssl_cert_reqs=None,
ssl_keyfile=self.shards[0].getTLSKeyFile(),
ssl_certfile=self.shards[0].getTLSCertFile(),
ssl_ca_certs=self.shards[0].getTLSCACertFile(),
)
if pool.connection_kwargs:
pool.connection_kwargs.pop('ssl', None)
return rediscluster.RedisCluster(
startup_nodes=self.getMasterNodesList(), connection_pool=pool)
else:
return rediscluster.RedisCluster(
startup_nodes=self.getMasterNodesList(), decode_responses=True)
def cluster_poll():
startup = [
{
'host': '127.0.0.1',
'port': 30001
},
{
'host': '127.0.0.1',
'port': 30002
},
{
'host': '127.0.0.1',
'port': 30003
},
{
'host': '127.0.0.1',
'port': 30004
},
{
'host': '127.0.0.1',
'port': 30005
},
{
'host': '127.0.0.1',
'port': 30006
},
]
pool = ClusterConnectionPool(startup_nodes=startup)
cli = RedisCluster(connection_pool=pool)
poller = poll()
conns = [(key, cli.connection_pool.get_connection_by_key(key))
for key in keys]
register(conns, poller)
try:
start_poll(conns, poller, cli)
except KeyboardInterrupt:
unregister(conns, poller)
def _get_pool(self, asynchronous=False):
params = self._connparams(asynchronous=asynchronous)
params['skip_full_coverage_check'] = True
return ClusterConnectionPool(**params)
def _get_pool(self):
params = self._connparams()
self.keyprefix_fanout = self.keyprefix_fanout.format(db=params['db'])
return ClusterConnectionPool(**params)