async def test_execute_with_protocol_error(loop, test_cluster, free_ports): expected_connection = FakeConnection(free_ports[0], loop, return_value=ProtocolError('ERROR')) with CreateConnectionMock({free_ports[0]: expected_connection}): with pytest.raises(ProtocolError): await test_cluster.execute('SET', SLOT_ZERO_KEY, 'value') expected_connection.execute.assert_called_once_with( b'SET', SLOT_ZERO_KEY, 'value')
async def test_create_fails(loop, nodes, free_ports): expected_connections = { port: FakeConnection(port, loop, return_value=ProtocolError('Intentional error')) for port in free_ports } with CreateConnectionMock(expected_connections): with pytest.raises(RedisClusterError): await create_cluster(nodes, encoding='utf-8', loop=loop)
async def test_keys_master__with_retries_but_success(mocker): commands_factory = mock.Mock() cl = Cluster(["addr1"], commands_factory=commands_factory) mocked_pooler = mocker.patch.object(cl, "_pooler", new=get_pooler_mock()) pool = mocked_pooler.ensure_pool.return_value errors = [ ConnectionError(), LoadingError("LOADING loading"), ProtocolError(), ClusterDownError("CLUSTERDOWN cluster is down"), OSError("socket problem"), ] errors_iter = iter(errors) def ensure_pool_se(addr): try: raise next(errors_iter) except StopIteration: return pool mocked_pooler.ensure_pool.side_effect = ensure_pool_se mocked_manager = mocker.patch.object(cl, "_manager", new=get_manager_mock()) state = mocked_manager.get_state.return_value mocker.patch.object(cl, "_execute_retry_slowdown", new=create_async_mock()) result = await cl.keys_master("key") assert result is commands_factory.return_value state.slot_master.assert_called_with(12539) assert state.slot_master.call_count == 6 mocked_pooler.ensure_pool.assert_has_calls( [mock.call(state.slot_master.return_value.addr)] * 6) commands_factory.assert_called_once_with(pool)
async def test_execute__success_several_problem_retry(mocker): cl = Cluster(["addr1"]) mocked_pooler = mocker.patch.object(cl, "_pooler", new=get_pooler_mock()) conn = mocked_pooler._conn pool = mocked_pooler._pool pool_execute_count = 0 errors = [ ConnectionRefusedError(), ConnectionError(), ConnectTimeoutError(("addr1", 6379)), AskError("ASK 12539 1.2.3.4:9999"), ClusterDownError("CLUSTERDOWN cluster is down"), TryAgainError("TRYAGAIN try again later"), MovedError("MOVED 12539 4.3.2.1:6666"), ProtocolError(), MovedError("MOVED 12539 4.3.2.1:6666"), ] def conn_execute_se(*args, **kwargs): nonlocal pool_execute_count if args[0] == b"ASKING": return conn.execute.return_value pool_execute_count += 1 raise errors[pool_execute_count - 1] def pool_execute_se(*args, **kwargs): nonlocal pool_execute_count pool_execute_count += 1 if pool_execute_count <= len(errors): err = errors[pool_execute_count - 1] raise err return ("result", pool_execute_count) pool.execute.side_effect = pool_execute_se conn.execute.side_effect = conn_execute_se mocked_manager = mocker.patch.object(cl, "_manager", new=get_manager_mock()) state = mocked_manager.get_state.return_value mocked_execute_retry_slowdown = mocker.patch.object( cl, "_execute_retry_slowdown", new=create_async_mock()) result = await cl.execute("get", "key") assert mocked_manager.get_state.call_count == 10 assert mocked_pooler.ensure_pool.call_count == 10 assert mocked_manager.require_reload_state.call_count == 8 assert result == ("result", 10) ensure_pool_calls = mocked_pooler.ensure_pool.call_args_list # #1 attempt assert ensure_pool_calls[0] == mock.call( state.slot_master.return_value.addr) # #2 attempt to random replica node assert ensure_pool_calls[1] == mock.call( state.random_slot_replica.return_value.addr) # #3 attempt try to execute on random node in cluster assert ensure_pool_calls[2] == mock.call( state.random_node.return_value.addr) # #4 attempt try random node in cluster assert ensure_pool_calls[3] == mock.call( state.random_node.return_value.addr) # #5 attempt as ASKING to node from MovedError assert ensure_pool_calls[4] == mock.call(Address("1.2.3.4", 9999)) # #6 attempt again random node because CLUSTERDOWN assert ensure_pool_calls[5] == mock.call( state.random_node.return_value.addr) # #7 attempt again random node because TRYAGAIN assert ensure_pool_calls[6] == mock.call( state.random_node.return_value.addr) # #8 attempt to node from MovedError assert ensure_pool_calls[7] == mock.call(Address("4.3.2.1", 6666)) # #9 attempt again random node because ProtocolError assert ensure_pool_calls[8] == mock.call( state.random_node.return_value.addr) # #10 attempt to node from MovedError, finally assert ensure_pool_calls[9] == mock.call(Address("4.3.2.1", 6666)) # cl.determine_slot(b'key') == 12539 state.slot_master.assert_called_once_with(12539) assert state.random_slot_replica.call_count == 1 assert state.random_slot_replica.call_args == mock.call(12539) assert state.random_node.call_count == 5 assert mocked_execute_retry_slowdown.call_count == 9 asking_calls = [ c for c in conn.execute.call_args_list if c == mock.call(b"ASKING") ] assert len(asking_calls) == 1
@pytest.mark.parametrize( "error, retry_node_addr, require_reload_state", [ (ConnectionError(), Address("random_replica", 6379), 1), (ConnectionRefusedError(), Address("random_replica", 6379), 1), (OSError(), Address("random_replica", 6379), 1), (ConnectTimeoutError(object()), Address("random_replica", 6379), 1), (ClusterDownError("CLUSTERDOWN clusterdown"), Address("random", 6379), 1), (TryAgainError("TRYAGAIN tryagain"), Address("random", 6379), 1), (MovedError("MOVED 12539 1.2.3.4:1000"), Address("1.2.3.4", 1000), 1), (AskError("ASK 12539 1.2.3.4:1000"), Address("1.2.3.4", 1000), 0), (LoadingError("LOADING loading"), Address("random", 6379), 1), (ProtocolError(), Address("random", 6379), 1), (ConnectionClosedError(), Address("random_replica", 6379), 1), (ConnectionForcedCloseError(), Address("random_replica", 6379), 1), (PoolClosedError(), Address("random_replica", 6379), 1), ], ) async def test_execute__retryable_errors(mocker, error, retry_node_addr, require_reload_state): cl = Cluster(["addr1"]) mocked_pooler = mocker.patch.object(cl, "_pooler", new=get_pooler_mock()) pool = mocked_pooler._pool conn = mocked_pooler._conn pool_execute_count = 0