async def test_before_server_start(dummy_bus: BusPath, loop, get_dummy_events):
registry.add(TestApi())
listener = await dummy_bus.example.test.my_event.listen_async(
lambda *a, **kw: None)
await asyncio.sleep(0.1) # Give the bus a moment to kick up the listener
state_plugin = StatePlugin(service_name="foo", process_name="bar")
state_plugin.ping_enabled = False
await state_plugin.before_server_start(client=dummy_bus.client)
await cancel(listener)
dummy_events = get_dummy_events()
assert len(dummy_events) == 1
event_message = dummy_events[0]
assert event_message.api_name == "internal.state"
assert event_message.event_name == "server_started"
assert event_message.kwargs["api_names"] == ["example.test"]
assert event_message.kwargs["listening_for"] == ["example.test.my_event"]
assert event_message.kwargs["metrics_enabled"] == False
assert event_message.kwargs["ping_enabled"] == False
assert event_message.kwargs["ping_interval"] == 60
assert event_message.kwargs["service_name"] == "foo"
assert event_message.kwargs["process_name"] == "bar"
async def test_ping(dummy_bus: BusPath, loop, get_dummy_events):
# We check the pings message contains a list of registries, so register one
registry.add(TestApi())
# Likewise for event listeners
await dummy_bus.example.test.my_event.listen_async(lambda *a, **kw: None)
# Let the state plugin send a ping then cancel it
state_plugin = StatePlugin(service_name="foo", process_name="bar")
state_plugin.ping_interval = 0.1
task = asyncio.ensure_future(
state_plugin._send_ping(client=dummy_bus.client), loop=loop)
await asyncio.sleep(0.15)
await cancel(task)
dummy_events = get_dummy_events()
assert len(dummy_events) == 1
event_message = dummy_events[0]
assert event_message.api_name == "internal.state"
assert event_message.event_name == "server_ping"
assert event_message.kwargs["api_names"] == ["example.test"]
assert event_message.kwargs["listening_for"] == ["example.test.my_event"]
assert event_message.kwargs["metrics_enabled"] == False
assert event_message.kwargs["ping_enabled"] == True
assert event_message.kwargs["ping_interval"] == 0.1
assert event_message.kwargs["service_name"] == "foo"
assert event_message.kwargs["process_name"] == "bar"
async def test_ping(dummy_bus: BusNode, loop, get_dummy_events):
# We check the pings message contains a list of registries, so register one
registry.add(TestApi())
# Likewise for event listeners
await dummy_bus.example.test.my_event.listen_async(lambda: None)
# Let the state plugin send a ping then cancel it
state_plugin = StatePlugin()
state_plugin.ping_interval = 0.1
task = asyncio.ensure_future(
state_plugin._send_ping(bus_client=dummy_bus.bus_client), loop=loop)
await asyncio.sleep(0.15)
task.cancel()
dummy_events = get_dummy_events()
assert len(dummy_events) == 1
event_message = dummy_events[0]
assert event_message.api_name == 'internal.state'
assert event_message.event_name == 'server_ping'
assert event_message.kwargs['api_names'] == ['example.test']
assert event_message.kwargs['listening_for'] == ['example.test.my_event']
assert event_message.kwargs['metrics_enabled'] == False
assert event_message.kwargs['ping_enabled'] == True
assert event_message.kwargs['ping_interval'] == 0.1
assert event_message.kwargs['process_name']
async def test_send_event(dummy_bus: BusPath, get_dummy_events):
manually_set_plugins(
plugins={
"metrics": MetricsPlugin(service_name="foo", process_name="bar")
})
registry.add(TestApi())
await dummy_bus.example.test.my_event.fire_async(f=123)
# What events were fired?
event_messages = get_dummy_events()
assert len(
event_messages
) == 2 # First is the actual event, followed by the metrics event
# rpc_response_received
assert event_messages[1].api_name == "internal.metrics"
assert event_messages[1].event_name == "event_fired"
assert event_messages[1].kwargs.pop("timestamp")
assert event_messages[1].kwargs == {
"api_name": "example.test",
"event_name": "my_event",
"event_id": "event_id",
"kwargs": {
"f": 123
},
"service_name": "foo",
"process_name": "bar",
}
async def test_multiple_event_transports(loop, server, redis_server_b):
"""Configure a bus with two redis transports and ensure they write to the correct redis servers"""
registry.add(ApiA())
registry.add(ApiB())
manually_set_plugins(plugins={})
redis_server_a = server
port_a = redis_server_a.tcp_address.port
port_b = redis_server_b.tcp_address.port
logging.warning(f"Server A port: {port_a}")
logging.warning(f"Server B port: {port_b}")
config = Config.load_dict(
{
"bus": {"schema": {"transport": {"redis": {"url": f"redis://localhost:{port_a}"}}}},
"apis": {
"default": {
"event_transport": {
"redis": {
"url": f"redis://localhost:{port_a}",
"stream_use": StreamUse.PER_EVENT.value,
}
}
},
"api_b": {
"event_transport": {
"redis": {
"url": f"redis://localhost:{port_b}",
"stream_use": StreamUse.PER_EVENT.value,
}
}
},
},
}
)
bus = BusPath(name="", parent=None, client=lightbus.BusClient(config=config, loop=loop))
await asyncio.sleep(0.1)
await bus.api_a.event_a.fire_async()
await bus.api_b.event_b.fire_async()
connection_manager_a = bus.client.transport_registry.get_event_transport(
"api_a"
).connection_manager
connection_manager_b = bus.client.transport_registry.get_event_transport(
"api_b"
).connection_manager
with await connection_manager_a() as redis:
assert await redis.xrange("api_a.event_a:stream")
assert await redis.xrange("api_b.event_b:stream") == []
with await connection_manager_b() as redis:
assert await redis.xrange("api_a.event_a:stream") == []
assert await redis.xrange("api_b.event_b:stream")
async def test_execute_events(dummy_bus: BusPath, dummy_listener,
get_dummy_events, mocker):
event_transport = dummy_bus.client.transport_registry.get_event_transport(
"default")
mocker.patch.object(
event_transport,
"_get_fake_message",
return_value=EventMessage(api_name="example.test",
event_name="my_event",
kwargs={"f": 123}),
)
await dummy_listener("example.test", "my_event")
# Setup the bus and do the call
manually_set_plugins(
plugins={
"metrics": MetricsPlugin(service_name="foo", process_name="bar")
})
registry.add(TestApi())
# The dummy transport will fire an every every 0.1 seconds
await asyncio.sleep(0.15)
event_messages = get_dummy_events()
assert len(event_messages) == 2
# before_rpc_execution
assert event_messages[0].api_name == "internal.metrics"
assert event_messages[0].event_name == "event_received"
assert event_messages[0].kwargs.pop("timestamp")
assert event_messages[0].kwargs == {
"api_name": "example.test",
"event_name": "my_event",
"event_id": "event_id",
"kwargs": {
"f": 123
},
"service_name": "foo",
"process_name": "bar",
}
# after_rpc_execution
assert event_messages[1].api_name == "internal.metrics"
assert event_messages[1].event_name == "event_processed"
assert event_messages[1].kwargs.pop("timestamp")
assert event_messages[1].kwargs == {
"api_name": "example.test",
"event_name": "my_event",
"event_id": "event_id",
"kwargs": {
"f": 123
},
"service_name": "foo",
"process_name": "bar",
}
async def test_local_rpc_call(loop, dummy_bus: BusPath, consume_rpcs,
get_dummy_events, mocker):
rpc_transport = dummy_bus.client.transport_registry.get_rpc_transport(
"default")
mocker.patch.object(
rpc_transport,
"_get_fake_messages",
return_value=[
RpcMessage(id="123abc",
api_name="example.test",
procedure_name="my_method",
kwargs={"f": 123})
],
)
# Setup the bus and do the call
manually_set_plugins(
plugins={
"metrics": MetricsPlugin(service_name="foo", process_name="bar")
})
registry.add(TestApi())
task = asyncio.ensure_future(consume_rpcs(dummy_bus), loop=loop)
# The dummy transport will fire an every every 0.1 seconds
await asyncio.sleep(0.15)
await cancel(task)
event_messages = get_dummy_events()
assert len(event_messages) == 2, event_messages
# before_rpc_execution
assert event_messages[0].api_name == "internal.metrics"
assert event_messages[0].event_name == "rpc_call_received"
assert event_messages[0].kwargs.pop("timestamp")
assert event_messages[0].kwargs == {
"api_name": "example.test",
"procedure_name": "my_method",
"id": "123abc",
"service_name": "foo",
"process_name": "bar",
}
# after_rpc_execution
assert event_messages[1].api_name == "internal.metrics"
assert event_messages[1].event_name == "rpc_response_sent"
assert event_messages[1].kwargs.pop("timestamp")
assert event_messages[1].kwargs == {
"api_name": "example.test",
"procedure_name": "my_method",
"id": "123abc",
"result": "value",
"service_name": "foo",
"process_name": "bar",
}
def run_forever(self, *, loop=None, consume_rpcs=True, consume_events=True, plugins=None):
logger.info(LBullets(
"Lightbus getting ready to run. Brokers in use",
items={
"RPC transport": L(
'{}.{}',
self.rpc_transport.__module__, Bold(self.rpc_transport.__class__.__name__)
),
"Result transport": L(
'{}.{}', self.result_transport.__module__,
Bold(self.result_transport.__class__.__name__)
),
"Event transport": L(
'{}.{}', self.event_transport.__module__,
Bold(self.event_transport.__class__.__name__)
),
}
))
self.setup(plugins=plugins)
registry.add(LightbusStateApi())
registry.add(LightbusMetricsApi())
if consume_rpcs:
if registry.public():
logger.info(LBullets(
"APIs in registry ({})".format(len(registry.all())),
items=registry.all()
))
else:
if consume_events:
logger.warning(
"No APIs have been registered, lightbus may still receive events "
"but Lightbus will not handle any incoming RPCs"
)
else:
logger.error(
"No APIs have been registered, yet Lightbus has been configured to only "
"handle RPCs. There is therefore nothing for lightbus to do. Exiting."
)
return
block(handle_aio_exceptions(
plugin_hook('before_server_start', bus_client=self, loop=loop)
), timeout=5)
loop = loop or asyncio.get_event_loop()
self._run_forever(loop, consume_rpcs, consume_events)
loop.run_until_complete(handle_aio_exceptions(
plugin_hook('after_server_stopped', bus_client=self, loop=loop)
))
async def test_after_server_stopped(dummy_bus: BusNode, loop,
get_dummy_events):
registry.add(TestApi())
await dummy_bus.example.test.my_event.listen_async(lambda: None)
await StatePlugin().after_server_stopped(bus_client=dummy_bus.bus_client,
loop=loop)
dummy_events = get_dummy_events()
assert len(dummy_events) == 1
event_message = dummy_events[0]
assert event_message.api_name == 'internal.state'
assert event_message.event_name == 'server_stopped'
assert event_message.kwargs['process_name']
async def test_execute_events(dummy_bus: BusNode, event_consumer,
get_dummy_events, mocker):
mocker.patch.object(dummy_bus.bus_client.event_transport,
'_get_fake_messages',
return_value=[
EventMessage(api_name='example.test',
event_name='my_event',
kwargs={'f': 123})
])
# Setup the bus and do the call
manually_set_plugins(plugins={'metrics': MetricsPlugin()})
registry.add(TestApi())
# The dummy transport will fire an every every 0.1 seconds
await asyncio.sleep(0.15)
event_messages = get_dummy_events()
assert len(event_messages) == 2
# before_rpc_execution
assert event_messages[0].api_name == 'internal.metrics'
assert event_messages[0].event_name == 'event_received'
assert event_messages[0].kwargs.pop('timestamp')
assert event_messages[0].kwargs.pop('process_name')
assert event_messages[0].kwargs == {
'api_name': 'example.test',
'event_name': 'my_event',
'event_id': 'event_id',
'kwargs': {
'f': 123
}
}
# after_rpc_execution
assert event_messages[1].api_name == 'internal.metrics'
assert event_messages[1].event_name == 'event_processed'
assert event_messages[1].kwargs.pop('timestamp')
assert event_messages[1].kwargs.pop('process_name')
assert event_messages[1].kwargs == {
'api_name': 'example.test',
'event_name': 'my_event',
'event_id': 'event_id',
'kwargs': {
'f': 123
}
}
async def test_listen_to_multiple_events_across_multiple_transports(loop, server, redis_server_b):
registry.add(ApiA())
registry.add(ApiB())
manually_set_plugins(plugins={})
redis_server_a = server
port_a = redis_server_a.tcp_address.port
port_b = redis_server_b.tcp_address.port
logging.warning(f"Server A port: {port_a}")
logging.warning(f"Server B port: {port_b}")
config = Config.load_dict(
{
"bus": {"schema": {"transport": {"redis": {"url": f"redis://localhost:{port_a}"}}}},
"apis": {
"default": {"event_transport": {"redis": {"url": f"redis://localhost:{port_a}"}}},
"api_b": {"event_transport": {"redis": {"url": f"redis://localhost:{port_b}"}}},
},
}
)
bus = BusPath(name="", parent=None, client=lightbus.BusClient(config=config, loop=loop))
await asyncio.sleep(0.1)
calls = 0
def listener(*args, **kwargs):
nonlocal calls
calls += 1
await bus.listen_multiple_async([bus.api_a.event_a, bus.api_b.event_b], listener=listener)
await asyncio.sleep(0.1)
await bus.api_a.event_a.fire_async()
await bus.api_b.event_b.fire_async()
await asyncio.sleep(0.1)
assert calls == 2
async def test_remote_rpc_call(dummy_bus: BusPath, get_dummy_events):
# Setup the bus and do the call
manually_set_plugins(
plugins={
"metrics": MetricsPlugin(service_name="foo", process_name="bar")
})
registry.add(TestApi())
await dummy_bus.example.test.my_method.call_async(f=123)
# What events were fired?
event_messages = get_dummy_events()
assert len(event_messages) == 2
# rpc_call_sent
assert event_messages[0].api_name == "internal.metrics"
assert event_messages[0].event_name == "rpc_call_sent"
# Pop these next two as the values are variable
assert event_messages[0].kwargs.pop("timestamp")
assert event_messages[0].kwargs.pop("id")
assert event_messages[0].kwargs == {
"api_name": "example.test",
"procedure_name": "my_method",
"kwargs": {
"f": 123
},
"service_name": "foo",
"process_name": "bar",
}
# rpc_response_received
assert event_messages[1].api_name == "internal.metrics"
assert event_messages[1].event_name == "rpc_response_received"
# Pop these next two as the values are variable
assert event_messages[1].kwargs.pop("timestamp")
assert event_messages[1].kwargs.pop("id")
assert event_messages[1].kwargs == {
"api_name": "example.test",
"procedure_name": "my_method",
"service_name": "foo",
"process_name": "bar",
}
def run_forever(self, *, consume_rpcs=True):
registry.add(LightbusStateApi())
registry.add(LightbusMetricsApi())
if consume_rpcs:
logger.info(
LBullets("APIs in registry ({})".format(len(registry.all())),
items=registry.names()))
block(self._plugin_hook("before_server_start"), timeout=5)
self._run_forever(consume_rpcs)
self.loop.run_until_complete(self._plugin_hook("after_server_stopped"))
# Close the bus (which will in turn close the transports)
self.close()
# See if we've set the exit code on the event loop
if hasattr(self.loop, "lightbus_exit_code"):
raise SystemExit(self.loop.lightbus_exit_code)
async def test_before_server_start(dummy_bus: BusNode, loop, get_dummy_events):
registry.add(TestApi())
await dummy_bus.example.test.my_event.listen_async(lambda: None)
state_plugin = StatePlugin()
state_plugin.do_ping = False
await state_plugin.before_server_start(bus_client=dummy_bus.bus_client,
loop=loop)
dummy_events = get_dummy_events()
assert len(dummy_events) == 1
event_message = dummy_events[0]
assert event_message.api_name == 'internal.state'
assert event_message.event_name == 'server_started'
assert event_message.kwargs['api_names'] == ['example.test']
assert event_message.kwargs['listening_for'] == ['example.test.my_event']
assert event_message.kwargs['metrics_enabled'] == False
assert event_message.kwargs['ping_enabled'] == False
assert event_message.kwargs['ping_interval'] == 60
assert event_message.kwargs['process_name']
async def test_after_server_stopped(dummy_bus: BusPath, loop,
get_dummy_events):
registry.add(TestApi())
listener = await dummy_bus.example.test.my_event.listen_async(
lambda *a, **kw: None)
plugin = StatePlugin(service_name="foo", process_name="bar")
plugin._ping_task = asyncio.Future()
await plugin.after_server_stopped(client=dummy_bus.client)
# Give any pending coroutines coroutines a moment to be awaited
await asyncio.sleep(0.001)
dummy_events = get_dummy_events()
assert len(dummy_events) == 1
event_message = dummy_events[0]
assert event_message.api_name == "internal.state"
assert event_message.event_name == "server_stopped"
assert event_message.kwargs["service_name"] == "foo"
assert event_message.kwargs["process_name"] == "bar"
await cancel(listener)
async def test_multiple_rpc_transports(loop, server, redis_server_b, consume_rpcs):
"""Configure a bus with two redis transports and ensure they write to the correct redis servers"""
registry.add(ApiA())
registry.add(ApiB())
manually_set_plugins(plugins={})
redis_server_a = server
port_a = redis_server_a.tcp_address.port
port_b = redis_server_b.tcp_address.port
logging.warning(f"Server A port: {port_a}")
logging.warning(f"Server B port: {port_b}")
config = Config.load_dict(
{
"bus": {"schema": {"transport": {"redis": {"url": f"redis://localhost:{port_a}"}}}},
"apis": {
"default": {
"rpc_transport": {"redis": {"url": f"redis://localhost:{port_a}"}},
"result_transport": {"redis": {"url": f"redis://localhost:{port_a}"}},
},
"api_b": {
"rpc_transport": {"redis": {"url": f"redis://localhost:{port_b}"}},
"result_transport": {"redis": {"url": f"redis://localhost:{port_b}"}},
},
},
}
)
bus = BusPath(name="", parent=None, client=lightbus.BusClient(config=config, loop=loop))
asyncio.ensure_future(consume_rpcs(bus))
await asyncio.sleep(0.1)
await bus.api_a.rpc_a.call_async()
await bus.api_b.rpc_b.call_async()
async def test_remote_rpc_call(dummy_bus: BusNode, get_dummy_events):
# Setup the bus and do the call
manually_set_plugins(plugins={'metrics': MetricsPlugin()})
registry.add(TestApi())
await dummy_bus.example.test.my_method.call_async(f=123)
# What events were fired?
event_messages = get_dummy_events()
assert len(event_messages) == 2
# rpc_call_sent
assert event_messages[0].api_name == 'internal.metrics'
assert event_messages[0].event_name == 'rpc_call_sent'
# Pop these next two as the values are variable
assert event_messages[0].kwargs.pop('timestamp')
assert event_messages[0].kwargs.pop('rpc_id')
assert event_messages[0].kwargs.pop('process_name')
assert event_messages[0].kwargs == {
'api_name': 'example.test',
'procedure_name': 'my_method',
'kwargs': {
'f': 123
},
}
# rpc_response_received
assert event_messages[1].api_name == 'internal.metrics'
assert event_messages[1].event_name == 'rpc_response_received'
# Pop these next two as the values are variable
assert event_messages[1].kwargs.pop('timestamp')
assert event_messages[1].kwargs.pop('rpc_id')
assert event_messages[1].kwargs.pop('process_name')
assert event_messages[1].kwargs == {
'api_name': 'example.test',
'procedure_name': 'my_method',
}
async def test_send_event(dummy_bus: BusNode, get_dummy_events):
manually_set_plugins(plugins={'metrics': MetricsPlugin()})
registry.add(TestApi())
await dummy_bus.example.test.my_event.fire_async(f=123)
# What events were fired?
event_messages = get_dummy_events()
assert len(
event_messages
) == 2 # First is the actual event, followed by the metrics event
# rpc_response_received
assert event_messages[1].api_name == 'internal.metrics'
assert event_messages[1].event_name == 'event_fired'
assert event_messages[1].kwargs.pop('timestamp')
assert event_messages[1].kwargs.pop('process_name')
assert event_messages[1].kwargs == {
'api_name': 'example.test',
'event_name': 'my_event',
'event_id': 'event_id',
'kwargs': {
'f': 123
}
}
def dummy_api():
from tests.dummy_api import DummyApi
dummy_api = DummyApi()
registry.add(dummy_api)
return dummy_api
