def create_bus_client_with_unhappy_schema(validate=True, strict_validation=True):
# Use the base transport as a dummy, it only needs to have a
# close() method on it in order to keep the client.close() method happy
schema = Schema(
schema_transport=TransportPool(
transport_class=lightbus.Transport,
config=None,
transport_config=NamedTuple("DummyTransportConfig")(),
)
)
# Fake loading of remote schemas from schema transport
schema._remote_schemas = {}
config = Config.load_dict(
{"apis": {"default": {"validate": validate, "strict_validation": strict_validation}}}
)
fake_schema = {"parameters": {"p": {}}, "response": {}}
mocker.patch.object(schema, "get_rpc_schema", autospec=True, return_value=fake_schema),
mocker.patch.object(schema, "get_event_schema", autospec=True, return_value=fake_schema),
# Make sure the test api named "api" has a schema, otherwise strict_validation
# will fail it
schema.local_schemas["api"] = fake_schema
mocker.patch(
"jsonschema.validate", autospec=True, side_effect=ValidationError("test error")
),
dummy_bus.client.schema = schema
dummy_bus.client.config = config
return dummy_bus.client
def _instantiate_transport_pool(self, transport_class: Type["Transport"],
transport_config: NamedTuple,
config: "Config"):
transport_pool = TransportPool(transport_class=transport_class,
transport_config=transport_config,
config=config)
return transport_pool
async def _result_listener(
self,
result_transport: TransportPool,
timeout: float,
rpc_message: RpcMessage,
return_path: str,
options: dict,
result_queue: InternalQueue,
):
try:
logger.debug("Result listener is waiting")
result = await asyncio.wait_for(
result_transport.receive_result(
rpc_message=rpc_message, return_path=return_path, options=options
),
timeout=timeout,
)
except asyncio.TimeoutError as e:
logger.debug("Result listener timed out")
await result_queue.put(e)
else:
logger.debug("Result listener received result, putting onto result queue")
await result_queue.put(result)
finally:
self.listener_tasks.discard(asyncio.current_task())
def test_equal(dummy_pool):
"""Test the __eq__ method"""
assert dummy_pool == TransportPool(
transport_class=DebugEventTransport,
transport_config=DebugEventTransport.Config(),
config=Config.default(),
)
async def redis_pool(redis_server_url):
pool = TransportPool(
transport_class=RedisEventTransport,
transport_config=RedisEventTransport.Config(url=redis_server_url),
config=Config.default(),
)
yield pool
await pool.close()
async def test_checkout_checkin_asyncio(mocker, redis_pool: TransportPool,
get_total_redis_connections):
"""Check in/out many connections concurrently (using asyncio tasks)
Unlike using threads, this should grow the pool
"""
# mocker.spy in pytest-mock runs afoul of this change in 3.8.1
# https://bugs.python.org/issue38857
# We therefore use mocker.spy for python 3.7, or the new AsyncMock in 3.8
# See: https://github.com/pytest-dev/pytest-mock/issues/178
if sys.version_info >= (3, 8):
from unittest.mock import AsyncMock
redis_pool.grow = AsyncMock(wraps=redis_pool.grow)
redis_pool._create_transport = AsyncMock(
wraps=redis_pool._create_transport)
redis_pool._close_transport = AsyncMock(
wraps=redis_pool._close_transport)
else:
mocker.spy(redis_pool, "grow")
mocker.spy(redis_pool, "_create_transport")
mocker.spy(redis_pool, "_close_transport")
async def _check_in_out():
transport = await redis_pool.checkout()
# Ensure we do something here in order to slow down the execution
# time, thereby ensuring our pool starts to fill up. We also need to
# use the connection to ensure the connection is lazy loaded
await transport.send_event(EventMessage(api_name="api",
event_name="event"),
options={})
await asyncio.sleep(0.02)
await redis_pool.checkin(transport)
async def _check_in_out_loop():
for _ in range(0, 500 // 20):
await _check_in_out()
tasks = [asyncio.create_task(_check_in_out_loop()) for _ in range(0, 20)]
await asyncio.wait(tasks)
await redis_pool.close()
assert redis_pool.grow.call_count == 20
assert redis_pool._create_transport.call_count == 20
assert redis_pool._close_transport.call_count == 20
assert await get_total_redis_connections() == 1
async def dummy_pool():
pool = TransportPool(
transport_class=DebugEventTransport,
transport_config=DebugEventTransport.Config(),
config=Config.default(),
)
yield pool
await pool.close()
def test_hash_equal(dummy_pool):
"""Test the __hash__ method"""
assert hash(dummy_pool) == hash(
TransportPool(
transport_class=DebugEventTransport,
transport_config=DebugEventTransport.Config(),
config=Config.default(),
))
def test_not_equal(redis_pool, redis_server_url):
"""Test the __eq__ method"""
assert redis_pool != TransportPool(
transport_class=RedisEventTransport,
transport_config=RedisEventTransport.Config(url=redis_server_url,
service_name="123"),
config=Config.default(),
)
async def test_checkout_checkin_threaded(mocker, redis_pool: TransportPool,
run_in_many_threads,
get_total_redis_connections):
"""Check in/out many connections concurrently (using threads)
Note that this will not grow the pool. See the doc string for TransportPool
"""
# mocker.spy in pytest-mock runs afoul of this change in 3.8.1
# https://bugs.python.org/issue38857
# We therefore use mocker.spy for python 3.7, or the new AsyncMock in 3.8.
# See: https://github.com/pytest-dev/pytest-mock/issues/178
if sys.version_info >= (3, 8):
from unittest.mock import AsyncMock
redis_pool.grow = AsyncMock(wraps=redis_pool.grow)
redis_pool._create_transport = AsyncMock(
wraps=redis_pool._create_transport)
redis_pool._close_transport = AsyncMock(
wraps=redis_pool._close_transport)
else:
mocker.spy(redis_pool, "grow")
mocker.spy(redis_pool, "_create_transport")
mocker.spy(redis_pool, "_close_transport")
async def _check_in_out():
transport = await redis_pool.checkout()
# Ensure we do something here in order to slow down the execution
# time, thereby ensuring our pool starts to fill up. We also need to
# use the connection to ensure the connection is lazy loaded
await transport.send_event(EventMessage(api_name="api",
event_name="event"),
options={})
await asyncio.sleep(0.02)
await redis_pool.checkin(transport)
run_in_many_threads(_check_in_out, executions=500, max_workers=20)
# We're running in a thread, so we never grow the pool. Instead
# we just return one-off connections which will be closed
# when they get checked back in
assert not redis_pool.grow.called
assert redis_pool._create_transport.call_count == 500
assert redis_pool._close_transport.call_count == 500
assert await get_total_redis_connections() == 1
async def attr_test_pool(redis_server_url):
"""Used for testing attribute access only"""
class AttrTestEventTransport(EventTransport):
class_prop = True
async def async_method(self):
return True
def sync_method(self):
return True
async def _async_private_method(self):
return True
pool = TransportPool(
transport_class=AttrTestEventTransport,
transport_config=AttrTestEventTransport.Config(),
config=Config.default(),
)
yield pool
await pool.close()
