async def test_execute_with_protocol_error(loop, test_cluster, free_ports):
expected_connection = FakeConnection(free_ports[0],
loop,
return_value=ProtocolError('ERROR'))
with CreateConnectionMock({free_ports[0]: expected_connection}):
with pytest.raises(ProtocolError):
await test_cluster.execute('SET', SLOT_ZERO_KEY, 'value')
expected_connection.execute.assert_called_once_with(
b'SET', SLOT_ZERO_KEY, 'value')
async def test_create_fails(loop, nodes, free_ports):
expected_connections = {
port: FakeConnection(port,
loop,
return_value=ProtocolError('Intentional error'))
for port in free_ports
}
with CreateConnectionMock(expected_connections):
with pytest.raises(RedisClusterError):
await create_cluster(nodes, encoding='utf-8', loop=loop)
async def test_keys_master__with_retries_but_success(mocker):
commands_factory = mock.Mock()
cl = Cluster(["addr1"], commands_factory=commands_factory)
mocked_pooler = mocker.patch.object(cl, "_pooler", new=get_pooler_mock())
pool = mocked_pooler.ensure_pool.return_value
errors = [
ConnectionError(),
LoadingError("LOADING loading"),
ProtocolError(),
ClusterDownError("CLUSTERDOWN cluster is down"),
OSError("socket problem"),
]
errors_iter = iter(errors)
def ensure_pool_se(addr):
try:
raise next(errors_iter)
except StopIteration:
return pool
mocked_pooler.ensure_pool.side_effect = ensure_pool_se
mocked_manager = mocker.patch.object(cl,
"_manager",
new=get_manager_mock())
state = mocked_manager.get_state.return_value
mocker.patch.object(cl, "_execute_retry_slowdown", new=create_async_mock())
result = await cl.keys_master("key")
assert result is commands_factory.return_value
state.slot_master.assert_called_with(12539)
assert state.slot_master.call_count == 6
mocked_pooler.ensure_pool.assert_has_calls(
[mock.call(state.slot_master.return_value.addr)] * 6)
commands_factory.assert_called_once_with(pool)
async def test_execute__success_several_problem_retry(mocker):
cl = Cluster(["addr1"])
mocked_pooler = mocker.patch.object(cl, "_pooler", new=get_pooler_mock())
conn = mocked_pooler._conn
pool = mocked_pooler._pool
pool_execute_count = 0
errors = [
ConnectionRefusedError(),
ConnectionError(),
ConnectTimeoutError(("addr1", 6379)),
AskError("ASK 12539 1.2.3.4:9999"),
ClusterDownError("CLUSTERDOWN cluster is down"),
TryAgainError("TRYAGAIN try again later"),
MovedError("MOVED 12539 4.3.2.1:6666"),
ProtocolError(),
MovedError("MOVED 12539 4.3.2.1:6666"),
]
def conn_execute_se(*args, **kwargs):
nonlocal pool_execute_count
if args[0] == b"ASKING":
return conn.execute.return_value
pool_execute_count += 1
raise errors[pool_execute_count - 1]
def pool_execute_se(*args, **kwargs):
nonlocal pool_execute_count
pool_execute_count += 1
if pool_execute_count <= len(errors):
err = errors[pool_execute_count - 1]
raise err
return ("result", pool_execute_count)
pool.execute.side_effect = pool_execute_se
conn.execute.side_effect = conn_execute_se
mocked_manager = mocker.patch.object(cl,
"_manager",
new=get_manager_mock())
state = mocked_manager.get_state.return_value
mocked_execute_retry_slowdown = mocker.patch.object(
cl, "_execute_retry_slowdown", new=create_async_mock())
result = await cl.execute("get", "key")
assert mocked_manager.get_state.call_count == 10
assert mocked_pooler.ensure_pool.call_count == 10
assert mocked_manager.require_reload_state.call_count == 8
assert result == ("result", 10)
ensure_pool_calls = mocked_pooler.ensure_pool.call_args_list
# #1 attempt
assert ensure_pool_calls[0] == mock.call(
state.slot_master.return_value.addr)
# #2 attempt to random replica node
assert ensure_pool_calls[1] == mock.call(
state.random_slot_replica.return_value.addr)
# #3 attempt try to execute on random node in cluster
assert ensure_pool_calls[2] == mock.call(
state.random_node.return_value.addr)
# #4 attempt try random node in cluster
assert ensure_pool_calls[3] == mock.call(
state.random_node.return_value.addr)
# #5 attempt as ASKING to node from MovedError
assert ensure_pool_calls[4] == mock.call(Address("1.2.3.4", 9999))
# #6 attempt again random node because CLUSTERDOWN
assert ensure_pool_calls[5] == mock.call(
state.random_node.return_value.addr)
# #7 attempt again random node because TRYAGAIN
assert ensure_pool_calls[6] == mock.call(
state.random_node.return_value.addr)
# #8 attempt to node from MovedError
assert ensure_pool_calls[7] == mock.call(Address("4.3.2.1", 6666))
# #9 attempt again random node because ProtocolError
assert ensure_pool_calls[8] == mock.call(
state.random_node.return_value.addr)
# #10 attempt to node from MovedError, finally
assert ensure_pool_calls[9] == mock.call(Address("4.3.2.1", 6666))
# cl.determine_slot(b'key') == 12539
state.slot_master.assert_called_once_with(12539)
assert state.random_slot_replica.call_count == 1
assert state.random_slot_replica.call_args == mock.call(12539)
assert state.random_node.call_count == 5
assert mocked_execute_retry_slowdown.call_count == 9
asking_calls = [
c for c in conn.execute.call_args_list if c == mock.call(b"ASKING")
]
assert len(asking_calls) == 1
@pytest.mark.parametrize(
"error, retry_node_addr, require_reload_state",
[
(ConnectionError(), Address("random_replica", 6379), 1),
(ConnectionRefusedError(), Address("random_replica", 6379), 1),
(OSError(), Address("random_replica", 6379), 1),
(ConnectTimeoutError(object()), Address("random_replica", 6379), 1),
(ClusterDownError("CLUSTERDOWN clusterdown"), Address("random",
6379), 1),
(TryAgainError("TRYAGAIN tryagain"), Address("random", 6379), 1),
(MovedError("MOVED 12539 1.2.3.4:1000"), Address("1.2.3.4", 1000), 1),
(AskError("ASK 12539 1.2.3.4:1000"), Address("1.2.3.4", 1000), 0),
(LoadingError("LOADING loading"), Address("random", 6379), 1),
(ProtocolError(), Address("random", 6379), 1),
(ConnectionClosedError(), Address("random_replica", 6379), 1),
(ConnectionForcedCloseError(), Address("random_replica", 6379), 1),
(PoolClosedError(), Address("random_replica", 6379), 1),
],
)
async def test_execute__retryable_errors(mocker, error, retry_node_addr,
require_reload_state):
cl = Cluster(["addr1"])
mocked_pooler = mocker.patch.object(cl, "_pooler", new=get_pooler_mock())
pool = mocked_pooler._pool
conn = mocked_pooler._conn
pool_execute_count = 0