async def test_waiters_LIFO(self, event_loop, mocker,
not_closed_connection):
# Check that waiters queue are seen as LIFO queue, where we try to rescue the latency
# of the last ones.
mocker.patch("emcache.connection_pool.create_protocol",
CoroutineMock(return_value=not_closed_connection))
connection_pool = ConnectionPool("localhost", 11211, 1, 1, None, None,
lambda _: _, False, False, None)
waiters_woken_up = []
async def coro(connection_context):
async with connection_context as _:
waiters_woken_up.append(connection_context)
# Try to get a connection many times.
connection_context1 = connection_pool.create_connection_context()
connection_context2 = connection_pool.create_connection_context()
connection_context3 = connection_pool.create_connection_context()
task1 = event_loop.create_task(coro(connection_context1))
task2 = event_loop.create_task(coro(connection_context2))
task3 = event_loop.create_task(coro(connection_context3))
await asyncio.gather(task1, task2, task3)
# check that where called in the right order
assert waiters_woken_up == [
connection_context3, connection_context2, connection_context1
]
# test specific atributes of the metrics
assert connection_pool.metrics().operations_waited == 3
async def test_connection_context_one_create_connection(
self, event_loop, mocker, not_closed_connection):
# Checks that while there is an ongoing connection creation, mo more connections
# will be created.
create_protocol = mocker.patch(
"emcache.connection_pool.create_protocol",
CoroutineMock(return_value=not_closed_connection))
connection_pool = ConnectionPool("localhost", 11211, 3, 1, None, None,
lambda _: _, False, False, None)
async def coro(connection_context):
async with connection_context as _:
pass
# Try to get a connection many times.
connection_context1 = connection_pool.create_connection_context()
connection_context2 = connection_pool.create_connection_context()
connection_context3 = connection_pool.create_connection_context()
task1 = event_loop.create_task(coro(connection_context1))
task2 = event_loop.create_task(coro(connection_context2))
task3 = event_loop.create_task(coro(connection_context3))
await asyncio.gather(task1, task2, task3)
# check that we have called the create_protocol just once
create_protocol.assert_called_once()
async def test_waiters_cancellation_is_supported(self, event_loop, mocker,
not_closed_connection):
# Check that waiters that are cancelled are suported and do not break
# the flow for wake up pending ones.
mocker.patch("emcache.connection_pool.create_protocol",
CoroutineMock(return_value=not_closed_connection))
connection_pool = ConnectionPool("localhost", 11211, 1, 1, None, None,
lambda _: _, False, False, None)
waiters_woken_up = []
begin = asyncio.Event()
end = asyncio.Event()
async def lazy_coro(connection_context):
async with connection_context as _:
begin.set()
waiters_woken_up.append(connection_context)
await end.wait()
async def coro(connection_context):
async with connection_context as _:
waiters_woken_up.append(connection_context)
connection_context1 = connection_pool.create_connection_context()
connection_context2 = connection_pool.create_connection_context()
connection_context3 = connection_pool.create_connection_context()
# Because of the LIFO nature, the context 3 should be the first one
# on having assigned the unique connection that will be available once
# is created.
task1 = event_loop.create_task(lazy_coro(connection_context3))
# Others will remain block until the first one finishes.
task2 = event_loop.create_task(coro(connection_context2))
task3 = event_loop.create_task(coro(connection_context1))
# waits till connection_context3 reaches the context
await begin.wait()
# cancel the connection_context2 that should be part
# of the waiters.
task2.cancel()
# notify the connection_context3 can move forward
end.set()
# wait till all tasks are finished
await asyncio.gather(task1, task2, task3, return_exceptions=True)
# check that where called in the right order, and the one cancelled
# does not appear here.
assert waiters_woken_up == [connection_context3, connection_context1]
# check that there are no waiters pending, also the cancelled ones
# was removed.
assert len(connection_pool._waiters) == 0
async def test_connection_context_create_connection_if_context_returns_with_exception(
self, event_loop, mocker, not_closed_connection):
# Checks that if a connection returns with an exception, if there are waiters a new connection is created.
create_protocol = mocker.patch(
"emcache.connection_pool.create_protocol",
CoroutineMock(
side_effect=[not_closed_connection, not_closed_connection]),
)
connection_pool = ConnectionPool("localhost", 11211, 1, 1, None, None,
lambda _: _, False, False, None)
async def coro(connection_context, raise_exception=False):
async with connection_context as _:
if raise_exception:
raise Exception()
# Try to get a connection many times. The first one will
# get and use the connection_closed, when this connection is
# returned back to the pool, a new one will be created
connection_context1 = connection_pool.create_connection_context()
connection_context2 = connection_pool.create_connection_context()
connection_context3 = connection_pool.create_connection_context()
task1 = event_loop.create_task(coro(connection_context1))
task2 = event_loop.create_task(coro(connection_context2))
# Remember that due to the LIFO nature, we have to raise the exception using the last
# pushed connection context
task3 = event_loop.create_task(
coro(connection_context3, raise_exception=True))
await asyncio.gather(task1, task2, task3, return_exceptions=True)
# check that we have called the create_protocol twice
create_protocol.assert_has_calls([
call("localhost",
11211,
ssl=False,
ssl_verify=False,
ssl_extra_ca=None,
timeout=None),
call("localhost",
11211,
ssl=False,
ssl_verify=False,
ssl_extra_ca=None,
timeout=None),
])
assert connection_pool.total_connections == 1
# test specific atributes of the metrics
assert connection_pool.metrics().connections_closed == 1
assert connection_pool.metrics().operations_executed_with_error == 1
assert connection_pool.metrics().operations_executed == 2
async def test_connection_context_create_connection_if_closed(
self, event_loop, mocker, not_closed_connection):
# Checks that if a connection is returned in closed state, if there are waiters a new connection is created.
connection_closed = Mock()
connection_closed.closed.return_value = True
create_protocol = mocker.patch(
"emcache.connection_pool.create_protocol",
CoroutineMock(
side_effect=[connection_closed, not_closed_connection]),
)
connection_pool = ConnectionPool("localhost", 11211, 1, 1, None, None,
lambda _: _, False, False, None)
async def coro(connection_context):
async with connection_context as _:
pass
# Try to get a connection many times. The first one will
# get and use the connection_closed, when this connection is
# returned back to the pool, a new one will be created
connection_context1 = connection_pool.create_connection_context()
connection_context2 = connection_pool.create_connection_context()
connection_context3 = connection_pool.create_connection_context()
task1 = event_loop.create_task(coro(connection_context1))
task2 = event_loop.create_task(coro(connection_context2))
task3 = event_loop.create_task(coro(connection_context3))
await asyncio.gather(task1, task2, task3)
# check that we have called the create_protocol twice
create_protocol.assert_has_calls([
call("localhost",
11211,
ssl=False,
ssl_verify=False,
ssl_extra_ca=None,
timeout=None),
call("localhost",
11211,
ssl=False,
ssl_verify=False,
ssl_extra_ca=None,
timeout=None),
])
assert connection_pool.total_connections == 1
# test specific atributes of the metrics
assert connection_pool.metrics().connections_closed == 1
async def test_unhealthy_one_event(self, event_loop, mocker,
not_closed_connection):
# Check that unhealthy status is reported once and only once
mocker.patch("emcache.connection_pool.asyncio.sleep", CoroutineMock())
# Simulate twice of the errors that would need to happen for
# reaching unhealthy status.
errors = [asyncio.TimeoutError
] * (DECLARE_UNHEALTHY_CONNECTION_POOL_AFTER_RETRIES * 2)
mocker.patch(
"emcache.connection_pool.create_protocol",
CoroutineMock(side_effect=errors),
)
cb_mock = Mock()
connection_pool = ConnectionPool("localhost", 11211, 1, 1, None, 1.0,
cb_mock, False, False, None)
async def coro(connection_context):
async with connection_context as _:
pass
connection_context = connection_pool.create_connection_context()
# wait for a new connection available in another task
task = event_loop.create_task(coro(connection_context))
with pytest.raises(asyncio.TimeoutError):
await asyncio.wait_for(task, timeout=0.01)
# check that was called once the unhealth report
cb_mock.assert_called_once_with(False)
async def test_unhealthy_healthy(self, event_loop, mocker,
not_closed_connection):
mocker.patch("emcache.connection_pool.asyncio.sleep", CoroutineMock())
# Simulate as many timeouts as many errors would need to happen for
# reaching the max retries till the connection pool is declared unhealthy
errors = [asyncio.TimeoutError
] * DECLARE_UNHEALTHY_CONNECTION_POOL_AFTER_RETRIES
mocker.patch(
"emcache.connection_pool.create_protocol",
CoroutineMock(side_effect=errors + [not_closed_connection]),
)
cb_mock = Mock()
connection_pool = ConnectionPool("localhost", 11211, 1, 1, None, 1.0,
cb_mock, False, False, None)
async def coro(connection_context):
async with connection_context as _:
pass
connection_context = connection_pool.create_connection_context()
# wait for a new connection available in another task
task = event_loop.create_task(coro(connection_context))
await task
# check that cb for telling that the connection pool became unhealthy and later on healty
cb_mock.assert_has_calls([call(False), call(True)])
async def test_connections_are_released_if_waiter_is_cancelled_just_after_wakeup(
self, event_loop, mocker, not_closed_connection):
async def never_return_connection(*args, **kwargs):
await asyncio.sleep(1e10)
mocker.patch("emcache.connection_pool.create_protocol",
CoroutineMock(side_effect=never_return_connection))
connection_pool = ConnectionPool("localhost", 11211, 1, 1, None, None,
lambda _: _, False, False, None)
async def coro(connection_context):
async with connection_context as _:
pass
connection_context1 = connection_pool.create_connection_context()
task1 = event_loop.create_task(coro(connection_context1))
# wait till all tasks are waiting
await asyncio.sleep(0.1)
with pytest.raises(asyncio.CancelledError):
# Force waiter to be waken up
connection_pool._wakeup_next_waiter_or_append_to_unused(
not_closed_connection)
# Cancel task immediately after
task1.cancel()
await task1
# Waiter should have been removed too
assert len(connection_pool._waiters) == 0
async def test_connection_context_not_use_a_closed_connections(
self, event_loop, mocker, not_closed_connection):
# Checks that if a connection is in closed state its not used and purged.
connection_closed = Mock()
connection_closed.closed.return_value = True
connection_pool = ConnectionPool("localhost", 11211, 2, 1, None, None,
lambda _: _, False, False, None)
# we add by hand a closed connection and a none closed one to the pool
connection_pool._unused_connections.append(not_closed_connection)
connection_pool._unused_connections.append(connection_closed)
connection_pool._total_connections = 2
connection_context = connection_pool.create_connection_context()
async with connection_context as connection:
assert connection is not_closed_connection
# double check that the closed one has been removed and the none closed
# one is still there.
assert not_closed_connection in connection_pool._unused_connections
assert connection_closed not in connection_pool._unused_connections
assert connection_pool.total_connections == 1
# test specific atributes of the metrics
assert connection_pool.metrics().connections_closed == 1
async def test_connection_context_connection(self, mocker,
not_closed_connection):
mocker.patch("emcache.connection_pool.create_protocol",
CoroutineMock(return_value=not_closed_connection))
connection_pool = ConnectionPool("localhost", 11211, 1, 1, None, None,
lambda _: _, False, False, None)
connection_context = connection_pool.create_connection_context()
async with connection_context as connection_from_pool:
assert connection_from_pool is not_closed_connection
async def test_create_connection_tries_again_if_error(
self, event_loop, mocker, not_closed_connection, exc):
# If there is an error trying to acquire a connection and there are waiters waiting we will keep
# trying to create a connection
# Avoid the annoying sleep added by the backoff
mocker.patch("emcache.connection_pool.asyncio.sleep", CoroutineMock())
# First we return a Timeout and second try returns an usable connection
create_protocol = mocker.patch(
"emcache.connection_pool.create_protocol",
CoroutineMock(side_effect=[exc, not_closed_connection]),
)
connection_pool = ConnectionPool("localhost", 11211, 1, 1, None, 1.0,
lambda _: _, False, False, None)
async def coro(connection_context):
async with connection_context as _:
pass
connection_context = connection_pool.create_connection_context()
# wait for a new connection available in another task
task = event_loop.create_task(coro(connection_context))
await task
# check that we have called twice the create_protocol
create_protocol.assert_has_calls([
call("localhost",
11211,
ssl=False,
ssl_verify=False,
ssl_extra_ca=None,
timeout=1.0),
call("localhost",
11211,
ssl=False,
ssl_verify=False,
ssl_extra_ca=None,
timeout=1.0),
])
# test specific atributes of the metrics
assert connection_pool.metrics().cur_connections == 1
assert connection_pool.metrics().connections_created == 1
assert connection_pool.metrics().connections_created_with_error == 1
async def test_remove_from_purge_tracking_connection_with_error(
self, event_loop, mocker):
connection = Mock()
mocker.patch("emcache.connection_pool.create_protocol",
CoroutineMock(return_value=connection))
connection_pool = ConnectionPool("localhost", 11211, 1, 1, 60, None,
lambda _: _, False, False, None)
# Create a new connection and use it
connection_context = connection_pool.create_connection_context()
with pytest.raises(Exception):
async with connection_context as _:
raise Exception()
connection.close.assert_called()
assert connection not in connection_pool._connections_last_time_used
assert connection not in connection_pool._unused_connections
async def test_create_connection_max_observed_latencies(
self, event_loop, mocker, not_closed_connection):
mocker.patch("emcache.connection_pool.create_protocol",
CoroutineMock(return_value=not_closed_connection))
connection_pool = ConnectionPool("localhost", 11211, 1, 1, None, None,
lambda _: _, False, False, None)
# Perform twice operations than the maximum observed latencies by removing
# the connection at each operation
for i in range(_MAX_CREATE_CONNECTION_LATENCIES_OBSERVED * 2):
async with connection_pool.create_connection_context():
pass
# remove the connection explicitly, behind the scenes connection pool
# will be forced to create a new one
connection_pool._close_connection(not_closed_connection)
# We should never have more than the maximum number of latencies observed
assert len(connection_pool._create_connection_latencies
) == _MAX_CREATE_CONNECTION_LATENCIES_OBSERVED
async def test_create_connection_backoff(self, event_loop, mocker,
not_closed_connection):
sleep = mocker.patch("emcache.connection_pool.asyncio.sleep",
CoroutineMock())
# Simulate as many timeouts as many errors would need to happen for
# reaching the max backoff plus 1
errors = [asyncio.TimeoutError] * 8
mocker.patch(
"emcache.connection_pool.create_protocol",
CoroutineMock(side_effect=errors + [not_closed_connection]),
)
connection_pool = ConnectionPool("localhost", 11211, 1, 1, None, 1.0,
lambda _: _, False, False, None)
async def coro(connection_context):
async with connection_context as _:
pass
connection_context = connection_pool.create_connection_context()
# wait for a new connection available in another task
task = event_loop.create_task(coro(connection_context))
await task
# check that the sleep call was called with the right values
sleep.assert_has_calls([
call(1),
call(2),
call(4),
call(8),
call(16),
call(32),
call(CREATE_CONNECTION_MAX_BACKOFF),
call(CREATE_CONNECTION_MAX_BACKOFF),
])
async def test_create_connection_do_not_try_again_unknown_error(
self, event_loop, mocker):
# If there is an error trying to acquire a connection and there are waiters waiting we will keep
# trying to create a connection
# Avoid the annoying sleep added by the backoff
mocker.patch("emcache.connection_pool.asyncio.sleep", CoroutineMock())
create_protocol = mocker.patch(
"emcache.connection_pool.create_protocol",
CoroutineMock(side_effect=Exception),
)
connection_pool = ConnectionPool("localhost", 11211, 1, 1, None, 1.0,
lambda _: _, False, False, None)
async def coro(connection_context):
async with connection_context as _:
pass
connection_context = connection_pool.create_connection_context()
# wait for a new connection available in another task
task = event_loop.create_task(coro(connection_context))
with pytest.raises(asyncio.TimeoutError):
await asyncio.wait_for(task, timeout=0.01)
# check that we have called once the create_protocol
create_protocol.assert_called_once_with("localhost",
11211,
ssl=False,
ssl_verify=False,
ssl_extra_ca=None,
timeout=1.0)
assert connection_pool.total_connections == 0
async def test_create_and_update_connection_timestamp(
self, event_loop, mocker, not_closed_connection):
mocker.patch("emcache.connection_pool.create_protocol",
CoroutineMock(return_value=not_closed_connection))
now = time.monotonic()
connection_pool = ConnectionPool("localhost", 11211, 1, 1, 60, None,
lambda _: _, False, False, None)
# Create a new connection and use it
connection_context = connection_pool.create_connection_context()
async with connection_context as _:
# Check that the first timestamp saved is the one that was used
# during the creation time.
assert connection_pool._connections_last_time_used[
not_closed_connection] > now
creation_timestamp = connection_pool._connections_last_time_used[
not_closed_connection]
# Finally when the connection is relased back to the pool the timestamp
# is also upated.
assert connection_pool._connections_last_time_used[
not_closed_connection] > creation_timestamp