async def test_waiters_LIFO(self, event_loop, mocker, not_closed_connection): # Check that waiters queue are seen as LIFO queue, where we try to rescue the latency # of the last ones. mocker.patch("emcache.connection_pool.create_protocol", CoroutineMock(return_value=not_closed_connection)) connection_pool = ConnectionPool("localhost", 11211, 1, 1, None, None, lambda _: _, False, False, None) waiters_woken_up = [] async def coro(connection_context): async with connection_context as _: waiters_woken_up.append(connection_context) # Try to get a connection many times. connection_context1 = connection_pool.create_connection_context() connection_context2 = connection_pool.create_connection_context() connection_context3 = connection_pool.create_connection_context() task1 = event_loop.create_task(coro(connection_context1)) task2 = event_loop.create_task(coro(connection_context2)) task3 = event_loop.create_task(coro(connection_context3)) await asyncio.gather(task1, task2, task3) # check that where called in the right order assert waiters_woken_up == [ connection_context3, connection_context2, connection_context1 ] # test specific atributes of the metrics assert connection_pool.metrics().operations_waited == 3
async def test_connection_context_one_create_connection( self, event_loop, mocker, not_closed_connection): # Checks that while there is an ongoing connection creation, mo more connections # will be created. create_protocol = mocker.patch( "emcache.connection_pool.create_protocol", CoroutineMock(return_value=not_closed_connection)) connection_pool = ConnectionPool("localhost", 11211, 3, 1, None, None, lambda _: _, False, False, None) async def coro(connection_context): async with connection_context as _: pass # Try to get a connection many times. connection_context1 = connection_pool.create_connection_context() connection_context2 = connection_pool.create_connection_context() connection_context3 = connection_pool.create_connection_context() task1 = event_loop.create_task(coro(connection_context1)) task2 = event_loop.create_task(coro(connection_context2)) task3 = event_loop.create_task(coro(connection_context3)) await asyncio.gather(task1, task2, task3) # check that we have called the create_protocol just once create_protocol.assert_called_once()
async def test_waiters_cancellation_is_supported(self, event_loop, mocker, not_closed_connection): # Check that waiters that are cancelled are suported and do not break # the flow for wake up pending ones. mocker.patch("emcache.connection_pool.create_protocol", CoroutineMock(return_value=not_closed_connection)) connection_pool = ConnectionPool("localhost", 11211, 1, 1, None, None, lambda _: _, False, False, None) waiters_woken_up = [] begin = asyncio.Event() end = asyncio.Event() async def lazy_coro(connection_context): async with connection_context as _: begin.set() waiters_woken_up.append(connection_context) await end.wait() async def coro(connection_context): async with connection_context as _: waiters_woken_up.append(connection_context) connection_context1 = connection_pool.create_connection_context() connection_context2 = connection_pool.create_connection_context() connection_context3 = connection_pool.create_connection_context() # Because of the LIFO nature, the context 3 should be the first one # on having assigned the unique connection that will be available once # is created. task1 = event_loop.create_task(lazy_coro(connection_context3)) # Others will remain block until the first one finishes. task2 = event_loop.create_task(coro(connection_context2)) task3 = event_loop.create_task(coro(connection_context1)) # waits till connection_context3 reaches the context await begin.wait() # cancel the connection_context2 that should be part # of the waiters. task2.cancel() # notify the connection_context3 can move forward end.set() # wait till all tasks are finished await asyncio.gather(task1, task2, task3, return_exceptions=True) # check that where called in the right order, and the one cancelled # does not appear here. assert waiters_woken_up == [connection_context3, connection_context1] # check that there are no waiters pending, also the cancelled ones # was removed. assert len(connection_pool._waiters) == 0
async def test_connection_context_create_connection_if_context_returns_with_exception( self, event_loop, mocker, not_closed_connection): # Checks that if a connection returns with an exception, if there are waiters a new connection is created. create_protocol = mocker.patch( "emcache.connection_pool.create_protocol", CoroutineMock( side_effect=[not_closed_connection, not_closed_connection]), ) connection_pool = ConnectionPool("localhost", 11211, 1, 1, None, None, lambda _: _, False, False, None) async def coro(connection_context, raise_exception=False): async with connection_context as _: if raise_exception: raise Exception() # Try to get a connection many times. The first one will # get and use the connection_closed, when this connection is # returned back to the pool, a new one will be created connection_context1 = connection_pool.create_connection_context() connection_context2 = connection_pool.create_connection_context() connection_context3 = connection_pool.create_connection_context() task1 = event_loop.create_task(coro(connection_context1)) task2 = event_loop.create_task(coro(connection_context2)) # Remember that due to the LIFO nature, we have to raise the exception using the last # pushed connection context task3 = event_loop.create_task( coro(connection_context3, raise_exception=True)) await asyncio.gather(task1, task2, task3, return_exceptions=True) # check that we have called the create_protocol twice create_protocol.assert_has_calls([ call("localhost", 11211, ssl=False, ssl_verify=False, ssl_extra_ca=None, timeout=None), call("localhost", 11211, ssl=False, ssl_verify=False, ssl_extra_ca=None, timeout=None), ]) assert connection_pool.total_connections == 1 # test specific atributes of the metrics assert connection_pool.metrics().connections_closed == 1 assert connection_pool.metrics().operations_executed_with_error == 1 assert connection_pool.metrics().operations_executed == 2
async def test_connection_context_create_connection_if_closed( self, event_loop, mocker, not_closed_connection): # Checks that if a connection is returned in closed state, if there are waiters a new connection is created. connection_closed = Mock() connection_closed.closed.return_value = True create_protocol = mocker.patch( "emcache.connection_pool.create_protocol", CoroutineMock( side_effect=[connection_closed, not_closed_connection]), ) connection_pool = ConnectionPool("localhost", 11211, 1, 1, None, None, lambda _: _, False, False, None) async def coro(connection_context): async with connection_context as _: pass # Try to get a connection many times. The first one will # get and use the connection_closed, when this connection is # returned back to the pool, a new one will be created connection_context1 = connection_pool.create_connection_context() connection_context2 = connection_pool.create_connection_context() connection_context3 = connection_pool.create_connection_context() task1 = event_loop.create_task(coro(connection_context1)) task2 = event_loop.create_task(coro(connection_context2)) task3 = event_loop.create_task(coro(connection_context3)) await asyncio.gather(task1, task2, task3) # check that we have called the create_protocol twice create_protocol.assert_has_calls([ call("localhost", 11211, ssl=False, ssl_verify=False, ssl_extra_ca=None, timeout=None), call("localhost", 11211, ssl=False, ssl_verify=False, ssl_extra_ca=None, timeout=None), ]) assert connection_pool.total_connections == 1 # test specific atributes of the metrics assert connection_pool.metrics().connections_closed == 1
async def test_unhealthy_one_event(self, event_loop, mocker, not_closed_connection): # Check that unhealthy status is reported once and only once mocker.patch("emcache.connection_pool.asyncio.sleep", CoroutineMock()) # Simulate twice of the errors that would need to happen for # reaching unhealthy status. errors = [asyncio.TimeoutError ] * (DECLARE_UNHEALTHY_CONNECTION_POOL_AFTER_RETRIES * 2) mocker.patch( "emcache.connection_pool.create_protocol", CoroutineMock(side_effect=errors), ) cb_mock = Mock() connection_pool = ConnectionPool("localhost", 11211, 1, 1, None, 1.0, cb_mock, False, False, None) async def coro(connection_context): async with connection_context as _: pass connection_context = connection_pool.create_connection_context() # wait for a new connection available in another task task = event_loop.create_task(coro(connection_context)) with pytest.raises(asyncio.TimeoutError): await asyncio.wait_for(task, timeout=0.01) # check that was called once the unhealth report cb_mock.assert_called_once_with(False)
async def test_unhealthy_healthy(self, event_loop, mocker, not_closed_connection): mocker.patch("emcache.connection_pool.asyncio.sleep", CoroutineMock()) # Simulate as many timeouts as many errors would need to happen for # reaching the max retries till the connection pool is declared unhealthy errors = [asyncio.TimeoutError ] * DECLARE_UNHEALTHY_CONNECTION_POOL_AFTER_RETRIES mocker.patch( "emcache.connection_pool.create_protocol", CoroutineMock(side_effect=errors + [not_closed_connection]), ) cb_mock = Mock() connection_pool = ConnectionPool("localhost", 11211, 1, 1, None, 1.0, cb_mock, False, False, None) async def coro(connection_context): async with connection_context as _: pass connection_context = connection_pool.create_connection_context() # wait for a new connection available in another task task = event_loop.create_task(coro(connection_context)) await task # check that cb for telling that the connection pool became unhealthy and later on healty cb_mock.assert_has_calls([call(False), call(True)])
async def test_connections_are_released_if_waiter_is_cancelled_just_after_wakeup( self, event_loop, mocker, not_closed_connection): async def never_return_connection(*args, **kwargs): await asyncio.sleep(1e10) mocker.patch("emcache.connection_pool.create_protocol", CoroutineMock(side_effect=never_return_connection)) connection_pool = ConnectionPool("localhost", 11211, 1, 1, None, None, lambda _: _, False, False, None) async def coro(connection_context): async with connection_context as _: pass connection_context1 = connection_pool.create_connection_context() task1 = event_loop.create_task(coro(connection_context1)) # wait till all tasks are waiting await asyncio.sleep(0.1) with pytest.raises(asyncio.CancelledError): # Force waiter to be waken up connection_pool._wakeup_next_waiter_or_append_to_unused( not_closed_connection) # Cancel task immediately after task1.cancel() await task1 # Waiter should have been removed too assert len(connection_pool._waiters) == 0
async def test_connection_context_not_use_a_closed_connections( self, event_loop, mocker, not_closed_connection): # Checks that if a connection is in closed state its not used and purged. connection_closed = Mock() connection_closed.closed.return_value = True connection_pool = ConnectionPool("localhost", 11211, 2, 1, None, None, lambda _: _, False, False, None) # we add by hand a closed connection and a none closed one to the pool connection_pool._unused_connections.append(not_closed_connection) connection_pool._unused_connections.append(connection_closed) connection_pool._total_connections = 2 connection_context = connection_pool.create_connection_context() async with connection_context as connection: assert connection is not_closed_connection # double check that the closed one has been removed and the none closed # one is still there. assert not_closed_connection in connection_pool._unused_connections assert connection_closed not in connection_pool._unused_connections assert connection_pool.total_connections == 1 # test specific atributes of the metrics assert connection_pool.metrics().connections_closed == 1
async def test_connection_context_connection(self, mocker, not_closed_connection): mocker.patch("emcache.connection_pool.create_protocol", CoroutineMock(return_value=not_closed_connection)) connection_pool = ConnectionPool("localhost", 11211, 1, 1, None, None, lambda _: _, False, False, None) connection_context = connection_pool.create_connection_context() async with connection_context as connection_from_pool: assert connection_from_pool is not_closed_connection
async def test_create_connection_tries_again_if_error( self, event_loop, mocker, not_closed_connection, exc): # If there is an error trying to acquire a connection and there are waiters waiting we will keep # trying to create a connection # Avoid the annoying sleep added by the backoff mocker.patch("emcache.connection_pool.asyncio.sleep", CoroutineMock()) # First we return a Timeout and second try returns an usable connection create_protocol = mocker.patch( "emcache.connection_pool.create_protocol", CoroutineMock(side_effect=[exc, not_closed_connection]), ) connection_pool = ConnectionPool("localhost", 11211, 1, 1, None, 1.0, lambda _: _, False, False, None) async def coro(connection_context): async with connection_context as _: pass connection_context = connection_pool.create_connection_context() # wait for a new connection available in another task task = event_loop.create_task(coro(connection_context)) await task # check that we have called twice the create_protocol create_protocol.assert_has_calls([ call("localhost", 11211, ssl=False, ssl_verify=False, ssl_extra_ca=None, timeout=1.0), call("localhost", 11211, ssl=False, ssl_verify=False, ssl_extra_ca=None, timeout=1.0), ]) # test specific atributes of the metrics assert connection_pool.metrics().cur_connections == 1 assert connection_pool.metrics().connections_created == 1 assert connection_pool.metrics().connections_created_with_error == 1
async def test_remove_from_purge_tracking_connection_with_error( self, event_loop, mocker): connection = Mock() mocker.patch("emcache.connection_pool.create_protocol", CoroutineMock(return_value=connection)) connection_pool = ConnectionPool("localhost", 11211, 1, 1, 60, None, lambda _: _, False, False, None) # Create a new connection and use it connection_context = connection_pool.create_connection_context() with pytest.raises(Exception): async with connection_context as _: raise Exception() connection.close.assert_called() assert connection not in connection_pool._connections_last_time_used assert connection not in connection_pool._unused_connections
async def test_create_connection_max_observed_latencies( self, event_loop, mocker, not_closed_connection): mocker.patch("emcache.connection_pool.create_protocol", CoroutineMock(return_value=not_closed_connection)) connection_pool = ConnectionPool("localhost", 11211, 1, 1, None, None, lambda _: _, False, False, None) # Perform twice operations than the maximum observed latencies by removing # the connection at each operation for i in range(_MAX_CREATE_CONNECTION_LATENCIES_OBSERVED * 2): async with connection_pool.create_connection_context(): pass # remove the connection explicitly, behind the scenes connection pool # will be forced to create a new one connection_pool._close_connection(not_closed_connection) # We should never have more than the maximum number of latencies observed assert len(connection_pool._create_connection_latencies ) == _MAX_CREATE_CONNECTION_LATENCIES_OBSERVED
async def test_create_connection_backoff(self, event_loop, mocker, not_closed_connection): sleep = mocker.patch("emcache.connection_pool.asyncio.sleep", CoroutineMock()) # Simulate as many timeouts as many errors would need to happen for # reaching the max backoff plus 1 errors = [asyncio.TimeoutError] * 8 mocker.patch( "emcache.connection_pool.create_protocol", CoroutineMock(side_effect=errors + [not_closed_connection]), ) connection_pool = ConnectionPool("localhost", 11211, 1, 1, None, 1.0, lambda _: _, False, False, None) async def coro(connection_context): async with connection_context as _: pass connection_context = connection_pool.create_connection_context() # wait for a new connection available in another task task = event_loop.create_task(coro(connection_context)) await task # check that the sleep call was called with the right values sleep.assert_has_calls([ call(1), call(2), call(4), call(8), call(16), call(32), call(CREATE_CONNECTION_MAX_BACKOFF), call(CREATE_CONNECTION_MAX_BACKOFF), ])
async def test_create_connection_do_not_try_again_unknown_error( self, event_loop, mocker): # If there is an error trying to acquire a connection and there are waiters waiting we will keep # trying to create a connection # Avoid the annoying sleep added by the backoff mocker.patch("emcache.connection_pool.asyncio.sleep", CoroutineMock()) create_protocol = mocker.patch( "emcache.connection_pool.create_protocol", CoroutineMock(side_effect=Exception), ) connection_pool = ConnectionPool("localhost", 11211, 1, 1, None, 1.0, lambda _: _, False, False, None) async def coro(connection_context): async with connection_context as _: pass connection_context = connection_pool.create_connection_context() # wait for a new connection available in another task task = event_loop.create_task(coro(connection_context)) with pytest.raises(asyncio.TimeoutError): await asyncio.wait_for(task, timeout=0.01) # check that we have called once the create_protocol create_protocol.assert_called_once_with("localhost", 11211, ssl=False, ssl_verify=False, ssl_extra_ca=None, timeout=1.0) assert connection_pool.total_connections == 0
async def test_create_and_update_connection_timestamp( self, event_loop, mocker, not_closed_connection): mocker.patch("emcache.connection_pool.create_protocol", CoroutineMock(return_value=not_closed_connection)) now = time.monotonic() connection_pool = ConnectionPool("localhost", 11211, 1, 1, 60, None, lambda _: _, False, False, None) # Create a new connection and use it connection_context = connection_pool.create_connection_context() async with connection_context as _: # Check that the first timestamp saved is the one that was used # during the creation time. assert connection_pool._connections_last_time_used[ not_closed_connection] > now creation_timestamp = connection_pool._connections_last_time_used[ not_closed_connection] # Finally when the connection is relased back to the pool the timestamp # is also upated. assert connection_pool._connections_last_time_used[ not_closed_connection] > creation_timestamp