async def test_access_correct_slave_with_readonly_mode_client(sr):
"""
Test that the client can get value normally with readonly mode
when we connect to correct slave.
"""
# we assume this key is set on 127.0.0.1:7000(7003)
await sr.set('foo16706', 'foo')
await asyncio.sleep(1)
with patch.object(ClusterConnectionPool,
'get_node_by_slot') as return_slave_mock:
return_slave_mock.return_value = {
'name': '127.0.0.1:7004',
'host': '127.0.0.1',
'port': 7004,
'server_type': 'slave',
}
master_value = {
'host': '127.0.0.1',
'name': '127.0.0.1:7000',
'port': 7000,
'server_type': 'master'
}
with patch.object(ClusterConnectionPool,
'get_master_node_by_slot',
return_value=master_value) as return_master_mock:
readonly_client = StrictRedisCluster(host="127.0.0.1",
port=7000,
readonly=True)
assert b('foo') == await readonly_client.get('foo16706')
readonly_client = StrictRedisCluster.from_url(
url="redis://127.0.0.1:7000/0", readonly=True)
assert b('foo') == await readonly_client.get('foo16706')
def monkey_link(host=None, port=None, *args, **kwargs):
"""
Helper function to return custom slots cache data from different redis nodes
"""
if port == 7000:
result = [[0, 5460, [b'127.0.0.1', 7000], [b'127.0.0.1', 7003]],
[
5461, 10922, [b'127.0.0.1', 7001],
[b'127.0.0.1', 7004]
]]
elif port == 7001:
result = [[0, 5460, [b'127.0.0.1', 7001], [b'127.0.0.1', 7003]],
[
5461, 10922, [b'127.0.0.1', 7000],
[b'127.0.0.1', 7004]
]]
else:
result = []
r = StrictRedisCluster(host=host, port=port, decode_responses=True)
orig_execute_command = r.execute_command
def execute_command(*args, **kwargs):
if args == ("cluster", "slots"):
return result
elif args == ('CONFIG GET', 'cluster-require-full-coverage'):
return {'cluster-require-full-coverage': 'yes'}
else:
return orig_execute_command(*args, **kwargs)
r.execute_command = execute_command
return r
def monkey_link(host=None, port=None, *args, **kwargs):
"""
Helper function to return custom slots cache data from different redis nodes
"""
if port == 7000:
result = {
(0, 5460): [{'host': '127.0.0.1', 'port': 7000, 'node_id': str(uuid.uuid4()), 'server_type': 'master'},
{'host': '127.0.0.1', 'port': 7003, 'node_id': str(uuid.uuid4()), 'server_type': 'slave'}],
(5461, 10922): [{'host': '127.0.0.1', 'port': 7001, 'node_id': str(uuid.uuid4()), 'server_type': 'master'},
{'host': '127.0.0.1', 'port': 7004, 'node_id': str(uuid.uuid4()), 'server_type': 'slave'}],
}
elif port == 7001:
result = {
(0, 5460): [{'host': '127.0.0.1', 'port': 7001, 'node_id': str(uuid.uuid4()), 'server_type': 'master'},
{'host': '127.0.0.1', 'port': 7003, 'node_id': str(uuid.uuid4()), 'server_type': 'slave'}],
(5461, 10922): [{'host': '127.0.0.1', 'port': 7000, 'node_id': str(uuid.uuid4()), 'server_type': 'master'},
{'host': '127.0.0.1', 'port': 7004, 'node_id': str(uuid.uuid4()), 'server_type': 'slave'}],
}
else:
result = dict()
r = StrictRedisCluster(host=host, port=port, decode_responses=True)
orig_execute_command = r.execute_command
async def execute_command(*args, **kwargs):
if args == ('CLUSTER SLOTS',):
return result
elif args == ('CONFIG GET', 'cluster-require-full-coverage'):
return {'cluster-require-full-coverage': 'yes'}
else:
return orig_execute_command(*args, **kwargs)
r.execute_command = execute_command
return r
async def test_moved_redirection():
"""
Test that the client handles MOVED response.
At first call it should return a MOVED ResponseError that will point
the client to the next server it should talk to.
Important thing to verify is that it tries to talk to the second node.
"""
r0 = StrictRedisCluster(host="127.0.0.1", port=7000)
r2 = StrictRedisCluster(host='127.0.0.1', port=7002)
await r0.flushdb()
await r2.flushdb()
assert await r0.set("foo", "bar")
assert await r2.get('foo') == b'bar'
def test_pubsub_thread_publish(event_loop):
"""
This test will never fail but it will still show and be viable to use
and to test the threading capability of the connectionpool and the publish
mechanism.
"""
startup_nodes = [{"host": "127.0.0.1", "port": "7000"}]
r = StrictRedisCluster(
startup_nodes=startup_nodes,
max_connections=16,
max_connections_per_node=16,
)
async def t_run(rc):
for i in range(0, 50):
await rc.publish('foo', 'bar')
await rc.publish('bar', 'foo')
await rc.publish('asd', 'dsa')
await rc.publish('dsa', 'asd')
await rc.publish('qwe', 'bar')
await rc.publish('ewq', 'foo')
await rc.publish('wer', 'dsa')
await rc.publish('rew', 'asd')
# Use this for debugging
# print(rc.connection_pool._available_connections)
# print(rc.connection_pool._in_use_connections)
# print(rc.connection_pool._created_connections)
try:
for i in range(10):
asyncio.run_coroutine_threadsafe(t_run(r), event_loop)
except Exception:
print("Error: unable to start thread")
def __init__(self,
host="localhost",
port=6379,
db=None,
password=None,
encoding='utf-8',
socket_keepalive=False,
connection_pool=None,
startup_nodes=None,
create_connection_timeout=None,
project='',
**kwargs):
if project:
project = f'{project}:'
self.cluster_flag = False
self.project = project
if startup_nodes:
from aredis import StrictRedisCluster
if isinstance(startup_nodes, (str, bytes)):
startup_nodes = _normalize_startup_nodes(startup_nodes)
self._redis = StrictRedisCluster(startup_nodes=startup_nodes,
decode_responses=True,
encoding=encoding,
skip_full_coverage_check=True,
**kwargs)
self.cluster_flag = True
else:
self._redis = StrictRedis(host=host,
port=port,
db=db,
password=password,
encoding=encoding,
socket_keepalive=socket_keepalive,
connection_pool=connection_pool,
**kwargs)
def test_skip_full_coverage_check():
"""
Test if the cluster_require_full_coverage NodeManager method was not called with the flag activated
"""
c = StrictRedisCluster("192.168.0.1", 7001, skip_full_coverage_check=True)
c.connection_pool.nodes.cluster_require_full_coverage = MagicMock()
c.connection_pool.nodes.initialize()
assert not c.connection_pool.nodes.cluster_require_full_coverage.called
async def test_moved_redirection_on_slave_with_default_client(sr):
"""
Test that the client is redirected normally with default
(readonly_mode=False) client even when we connect always to slave.
"""
await assert_moved_redirection_on_slave(
sr, ClusterConnectionPool,
StrictRedisCluster(host="127.0.0.1", port=7000, reinitialize_steps=1))
def test_host_port_startup_node():
"""
Test that it is possible to use host & port arguments as startup node args
"""
h = "192.168.0.1"
p = 7000
c = StrictRedisCluster(host=h, port=p)
assert {"host": h, "port": p} in c.connection_pool.nodes.startup_nodes
async def test_moved_redirection_on_slave_with_readonly_mode_client(sr):
"""
Ditto with READONLY mode.
"""
await assert_moved_redirection_on_slave(
sr,
ClusterConnectionPool,
StrictRedisCluster(host="127.0.0.1", port=7000, readonly=True, reinitialize_steps=1)
)
async def example():
client = StrictRedisCluster(host='172.17.0.2', port=7001)
await client.flushdb()
await client.set('foo', 1)
await client.lpush('a', 1)
print(await client.cluster_slots())
await client.rpoplpush('a', 'b')
assert await client.rpop('b') == b'1'
async def example_cluster():
client = StrictRedisCluster(host='127.0.0.1', port=7001)
await client.flushdb()
await client.set('foo', 1)
await client.lpush('a', 1)
print(await client.cluster_slots())
# 'a' and 'b' are in different slots
await client.rpoplpush('a', 'b')
assert await client.rpop('b') == b'1'
async def test_refresh_table_asap(monkeypatch):
"""
If this variable is set externally, initialize() should be called.
"""
async def return_none(*args, **kwargs):
return None
monkeypatch.setattr(ClusterConnectionPool, 'initialize', return_none)
monkeypatch.setattr(StrictRedisCluster, 'parse_response', return_none)
r = StrictRedisCluster(host="127.0.0.1", port=7000)
r.connection_pool.nodes.slots[12182] = [{
"host": "127.0.0.1",
"port": 7002,
"name": "127.0.0.1:7002",
"server_type": "master",
}]
r.refresh_table_asap = True
await r.set("foo", "bar")
assert r.refresh_table_asap is False
await r.flushdb()
def test_blocked_strict_redis_args():
"""
Some arguments should explicitly be blocked because they will not work in a cluster setup
"""
params = {'startup_nodes': [{'host': '127.0.0.1', 'port': 7000}]}
c = StrictRedisCluster(**params)
assert c.connection_pool.connection_kwargs[
"stream_timeout"] == ClusterConnectionPool.RedisClusterDefaultTimeout
with pytest.raises(RedisClusterException) as ex:
_get_client(db=1)
assert str(ex.value).startswith(
"Argument 'db' is not possible to use in cluster mode")
async def func2():
cluster = StrictRedisCluster(startup_nodes=[{
'host': '127.0.0.1',
'port': 7001
}],
decode_responses=True)
while True:
foobar = int(await cluster.get('foobar'))
print('thread: get `foobar` = {}'.format(foobar))
if foobar >= 0:
print(
'thread: cluster get foobar == {}, decrease it'.format(foobar))
await cluster.decr('foobar', 1)
if foobar < 0:
print('thread: break loop now')
break
async def run_func1():
cluster = StrictRedisCluster(startup_nodes=[{
'host': '127.0.0.1',
'port': 7001
}],
decode_responses=True)
print('before transaction: set key `foobar` = 0')
await cluster.set('foobar', 0)
try:
await cluster.transaction(func1, 'foobar', watch_delay=2)
except Exception as exc:
print(exc)
print('after transaction: `foobar` = {}'.format(await
cluster.get('foobar')))
print('wait for thread to end...')
await asyncio.sleep(1)
def test_custom_connectionpool():
"""
Test that a custom connection pool will be used by StrictRedisCluster
"""
h = "192.168.0.1"
p = 7001
pool = DummyConnectionPool(host=h,
port=p,
connection_class=DummyConnection,
startup_nodes=[{
'host': h,
'port': p
}])
c = StrictRedisCluster(connection_pool=pool)
assert c.connection_pool is pool
assert c.connection_pool.connection_class == DummyConnection
assert {"host": h, "port": p} in c.connection_pool.nodes.startup_nodes
async def example():
cluster = StrictRedisCluster(startup_nodes=[{
'host': '127.0.0.1',
'port': 7001
}])
slots = await cluster.cluster_slots()
master_node = slots[(5461, 10922)][0]['node_id']
slave_node = slots[(5461, 10922)][1]['node_id']
print('master: {}'.format(master_node))
print('slave: {}'.format(slave_node))
print('nodes: {}'.format(await cluster.cluster_info()))
for time in range(2):
# forget a node twice to see if error will be raised
try:
await cluster.cluster_forget(master_node)
except Exception as exc:
logging.error(exc)
slots = await cluster.cluster_slots()
print(slots[(5461, 10922)])
async def test_pipeline_ask_redirection():
"""
Test that the server handles ASK response when used in pipeline.
At first call it should return a ASK ResponseError that will point
the client to the next server it should talk to.
Important thing to verify is that it tries to talk to the second node.
"""
r = StrictRedisCluster(host="127.0.0.1", port=7000)
with patch.object(StrictRedisCluster,
'parse_response') as parse_response:
async def response(connection, *args, **options):
async def response(connection, *args, **options):
async def response(connection, *args, **options):
assert connection.host == "127.0.0.1"
assert connection.port == 7001
return "MOCK_OK"
parse_response.side_effect = response
raise AskError("12182 127.0.0.1:7001")
parse_response.side_effect = response
raise AskError("12182 127.0.0.1:7001")
parse_response.side_effect = response
p = await r.pipeline()
await p.connection_pool.initialize()
p.connection_pool.nodes.nodes['127.0.0.1:7001'] = {
'host': u'127.0.0.1',
'server_type': 'master',
'port': 7001,
'name': '127.0.0.1:7001'
}
await p.set("foo", "bar")
assert await p.execute() == ["MOCK_OK"]
