def test_remove_all_plugins(): assert get_plugins() == OrderedDict() manually_set_plugins(OrderedDict([ ('p1', LightbusPlugin()), ])) remove_all_plugins() assert get_plugins() == OrderedDict()
async def test_send_event(dummy_bus: BusPath, get_dummy_events): manually_set_plugins( plugins={ "metrics": MetricsPlugin(service_name="foo", process_name="bar") }) registry.add(TestApi()) await dummy_bus.example.test.my_event.fire_async(f=123) # What events were fired? event_messages = get_dummy_events() assert len( event_messages ) == 2 # First is the actual event, followed by the metrics event # rpc_response_received assert event_messages[1].api_name == "internal.metrics" assert event_messages[1].event_name == "event_fired" assert event_messages[1].kwargs.pop("timestamp") assert event_messages[1].kwargs == { "api_name": "example.test", "event_name": "my_event", "event_id": "event_id", "kwargs": { "f": 123 }, "service_name": "foo", "process_name": "bar", }
def test_is_plugin_loaded(): assert get_plugins() == OrderedDict() assert is_plugin_loaded(LightbusPlugin) == False manually_set_plugins(OrderedDict([ ('p1', LightbusPlugin()), ])) assert is_plugin_loaded(LightbusPlugin) == True
async def test_multiple_event_transports(loop, server, redis_server_b): """Configure a bus with two redis transports and ensure they write to the correct redis servers""" registry.add(ApiA()) registry.add(ApiB()) manually_set_plugins(plugins={}) redis_server_a = server port_a = redis_server_a.tcp_address.port port_b = redis_server_b.tcp_address.port logging.warning(f"Server A port: {port_a}") logging.warning(f"Server B port: {port_b}") config = Config.load_dict( { "bus": {"schema": {"transport": {"redis": {"url": f"redis://localhost:{port_a}"}}}}, "apis": { "default": { "event_transport": { "redis": { "url": f"redis://localhost:{port_a}", "stream_use": StreamUse.PER_EVENT.value, } } }, "api_b": { "event_transport": { "redis": { "url": f"redis://localhost:{port_b}", "stream_use": StreamUse.PER_EVENT.value, } } }, }, } ) bus = BusPath(name="", parent=None, client=lightbus.BusClient(config=config, loop=loop)) await asyncio.sleep(0.1) await bus.api_a.event_a.fire_async() await bus.api_b.event_b.fire_async() connection_manager_a = bus.client.transport_registry.get_event_transport( "api_a" ).connection_manager connection_manager_b = bus.client.transport_registry.get_event_transport( "api_b" ).connection_manager with await connection_manager_a() as redis: assert await redis.xrange("api_a.event_a:stream") assert await redis.xrange("api_b.event_b:stream") == [] with await connection_manager_b() as redis: assert await redis.xrange("api_a.event_a:stream") == [] assert await redis.xrange("api_b.event_b:stream")
async def test_local_rpc_call(loop, dummy_bus: BusPath, consume_rpcs, get_dummy_events, mocker): rpc_transport = dummy_bus.client.transport_registry.get_rpc_transport( "default") mocker.patch.object( rpc_transport, "_get_fake_messages", return_value=[ RpcMessage(id="123abc", api_name="example.test", procedure_name="my_method", kwargs={"f": 123}) ], ) # Setup the bus and do the call manually_set_plugins( plugins={ "metrics": MetricsPlugin(service_name="foo", process_name="bar") }) registry.add(TestApi()) task = asyncio.ensure_future(consume_rpcs(dummy_bus), loop=loop) # The dummy transport will fire an every every 0.1 seconds await asyncio.sleep(0.15) await cancel(task) event_messages = get_dummy_events() assert len(event_messages) == 2, event_messages # before_rpc_execution assert event_messages[0].api_name == "internal.metrics" assert event_messages[0].event_name == "rpc_call_received" assert event_messages[0].kwargs.pop("timestamp") assert event_messages[0].kwargs == { "api_name": "example.test", "procedure_name": "my_method", "id": "123abc", "service_name": "foo", "process_name": "bar", } # after_rpc_execution assert event_messages[1].api_name == "internal.metrics" assert event_messages[1].event_name == "rpc_response_sent" assert event_messages[1].kwargs.pop("timestamp") assert event_messages[1].kwargs == { "api_name": "example.test", "procedure_name": "my_method", "id": "123abc", "result": "value", "service_name": "foo", "process_name": "bar", }
async def test_execute_events(dummy_bus: BusPath, dummy_listener, get_dummy_events, mocker): event_transport = dummy_bus.client.transport_registry.get_event_transport( "default") mocker.patch.object( event_transport, "_get_fake_message", return_value=EventMessage(api_name="example.test", event_name="my_event", kwargs={"f": 123}), ) await dummy_listener("example.test", "my_event") # Setup the bus and do the call manually_set_plugins( plugins={ "metrics": MetricsPlugin(service_name="foo", process_name="bar") }) registry.add(TestApi()) # The dummy transport will fire an every every 0.1 seconds await asyncio.sleep(0.15) event_messages = get_dummy_events() assert len(event_messages) == 2 # before_rpc_execution assert event_messages[0].api_name == "internal.metrics" assert event_messages[0].event_name == "event_received" assert event_messages[0].kwargs.pop("timestamp") assert event_messages[0].kwargs == { "api_name": "example.test", "event_name": "my_event", "event_id": "event_id", "kwargs": { "f": 123 }, "service_name": "foo", "process_name": "bar", } # after_rpc_execution assert event_messages[1].api_name == "internal.metrics" assert event_messages[1].event_name == "event_processed" assert event_messages[1].kwargs.pop("timestamp") assert event_messages[1].kwargs == { "api_name": "example.test", "event_name": "my_event", "event_id": "event_id", "kwargs": { "f": 123 }, "service_name": "foo", "process_name": "bar", }
async def test_random_failures(bus: lightbus.path.BusPath, caplog, fire_dummy_events, dummy_api, mocker): # Use test_history() (below) to repeat any cases which fail caplog.set_level(logging.WARNING) # The metrics plugins will add too much overhead to this test manually_set_plugins({}) event_ok_ids = dict() history = [] async def listener(event_message, field, **kwargs): call_id = field event_ok_ids.setdefault(call_id, 0) event_ok_ids[call_id] += 1 await asyncio.sleep(0.01) fire_task = asyncio.ensure_future( fire_dummy_events(total=100, initial_delay=0.1)) for _ in range(0, 20): logging.warning( "TEST: Still waiting for events to finish. {} so far".format( len(event_ok_ids))) for _ in range(0, 5): listen_task = asyncio.ensure_future( bus.my.dummy.my_event.listen_async(listener)) await asyncio.sleep(0.2) listen_task.cancel() await listen_task if len(event_ok_ids) == 100: logging.warning("TEST: Events finished") break # Cleanup the tasks fire_task.cancel() try: await fire_task fire_task.result() except CancelledError: pass duplicate_calls = sum([n - 1 for n in event_ok_ids.values()]) logger.warning("History: {}".format(",".join("{}{}".format(*x) for x in history))) logger.warning( "Finished with {}/100 events processed, {} duplicated calls".format( len(event_ok_ids), duplicate_calls)) assert set(event_ok_ids.keys()) == set(range(0, 100)) assert duplicate_calls > 0
def test_manually_set_plugins(): assert get_plugins() == OrderedDict() p1 = LightbusPlugin() p2 = LightbusPlugin() manually_set_plugins(OrderedDict([ ('p1', p1), ('p2', p2), ])) assert get_plugins() == OrderedDict([ ('p1', p1), ('p2', p2), ])
async def test_plugin_hook(mocker): """Ensure calling plugin_hook() calls the method on the plugin""" assert get_plugins() == OrderedDict() plugin = LightbusPlugin() manually_set_plugins(OrderedDict([ ('p1', plugin), ])) async def dummy_coroutine(*args, **kwargs): pass m = mocker.patch.object(plugin, 'before_server_start', return_value=dummy_coroutine()) await plugin_hook('before_server_start', bus_client=None, loop=None) assert m.called
async def test_execute_events(dummy_bus: BusNode, event_consumer, get_dummy_events, mocker): mocker.patch.object(dummy_bus.bus_client.event_transport, '_get_fake_messages', return_value=[ EventMessage(api_name='example.test', event_name='my_event', kwargs={'f': 123}) ]) # Setup the bus and do the call manually_set_plugins(plugins={'metrics': MetricsPlugin()}) registry.add(TestApi()) # The dummy transport will fire an every every 0.1 seconds await asyncio.sleep(0.15) event_messages = get_dummy_events() assert len(event_messages) == 2 # before_rpc_execution assert event_messages[0].api_name == 'internal.metrics' assert event_messages[0].event_name == 'event_received' assert event_messages[0].kwargs.pop('timestamp') assert event_messages[0].kwargs.pop('process_name') assert event_messages[0].kwargs == { 'api_name': 'example.test', 'event_name': 'my_event', 'event_id': 'event_id', 'kwargs': { 'f': 123 } } # after_rpc_execution assert event_messages[1].api_name == 'internal.metrics' assert event_messages[1].event_name == 'event_processed' assert event_messages[1].kwargs.pop('timestamp') assert event_messages[1].kwargs.pop('process_name') assert event_messages[1].kwargs == { 'api_name': 'example.test', 'event_name': 'my_event', 'event_id': 'event_id', 'kwargs': { 'f': 123 } }
async def test_event_exception_in_listener_realtime(bus: lightbus.path.BusPath, dummy_api, redis_client): """Start a listener (which errors) and then add events to the stream. The listener will load them one-by-one.""" manually_set_plugins({}) received_messages = [] # Don't shutdown on error bus.client.config.api("default").on_error = OnError.STOP_LISTENER async def listener(event_message, **kwargs): nonlocal received_messages received_messages.append(event_message) raise Exception() await bus.my.dummy.my_event.listen_async(listener, bus_options={"since": "0"}) await asyncio.sleep(0.1) await bus.my.dummy.my_event.fire_async(field="Hello! 😎") await bus.my.dummy.my_event.fire_async(field="Hello! 😎") await bus.my.dummy.my_event.fire_async(field="Hello! 😎") await bus.my.dummy.my_event.fire_async(field="Hello! 😎") await bus.my.dummy.my_event.fire_async(field="Hello! 😎") await asyncio.sleep(0.01) # Died when processing first message, so we only saw one message assert len(received_messages) == 1 # Now check we have not acked any messages messages = await redis_client.xrange("my.dummy.my_event:stream") # Messages 0 is the noop message used to create the stream message_ids = [id_ for id_, *_ in messages] pending_messages = await redis_client.xpending("my.dummy.my_event:stream", "test_cg-default", "-", "+", 10, "test_consumer") pending_message_ids = [id_ for id_, *_ in pending_messages] # The first 4 messages are still pending. Why 4 messages? Because: # - 1. The noop message used to create the stream (because we listened before we fired) # - 2. The first message which caused the error assert len(pending_message_ids) == 2 assert pending_message_ids == message_ids[:2]
def setup(self, plugins: dict=None): """Setup lightbus and get it ready to consume events and/or RPCs You should call this manually if you are calling `consume_rpcs()` or `consume_events()` directly. This you be handled for you if you are calling `run_forever()`. """ if plugins is None: logger.debug("Auto-loading any installed Lightbus plugins...") plugins = autoload_plugins() else: logger.debug("Loading explicitly specified Lightbus plugins....") manually_set_plugins(plugins) if plugins: logger.info(LBullets("Loaded the following plugins ({})".format(len(plugins)), items=plugins)) else: logger.info("No plugins loaded")
async def test_remote_rpc_call(dummy_bus: BusPath, get_dummy_events): # Setup the bus and do the call manually_set_plugins( plugins={ "metrics": MetricsPlugin(service_name="foo", process_name="bar") }) registry.add(TestApi()) await dummy_bus.example.test.my_method.call_async(f=123) # What events were fired? event_messages = get_dummy_events() assert len(event_messages) == 2 # rpc_call_sent assert event_messages[0].api_name == "internal.metrics" assert event_messages[0].event_name == "rpc_call_sent" # Pop these next two as the values are variable assert event_messages[0].kwargs.pop("timestamp") assert event_messages[0].kwargs.pop("id") assert event_messages[0].kwargs == { "api_name": "example.test", "procedure_name": "my_method", "kwargs": { "f": 123 }, "service_name": "foo", "process_name": "bar", } # rpc_response_received assert event_messages[1].api_name == "internal.metrics" assert event_messages[1].event_name == "rpc_response_received" # Pop these next two as the values are variable assert event_messages[1].kwargs.pop("timestamp") assert event_messages[1].kwargs.pop("id") assert event_messages[1].kwargs == { "api_name": "example.test", "procedure_name": "my_method", "service_name": "foo", "process_name": "bar", }
async def test_listen_to_multiple_events_across_multiple_transports(loop, server, redis_server_b): registry.add(ApiA()) registry.add(ApiB()) manually_set_plugins(plugins={}) redis_server_a = server port_a = redis_server_a.tcp_address.port port_b = redis_server_b.tcp_address.port logging.warning(f"Server A port: {port_a}") logging.warning(f"Server B port: {port_b}") config = Config.load_dict( { "bus": {"schema": {"transport": {"redis": {"url": f"redis://localhost:{port_a}"}}}}, "apis": { "default": {"event_transport": {"redis": {"url": f"redis://localhost:{port_a}"}}}, "api_b": {"event_transport": {"redis": {"url": f"redis://localhost:{port_b}"}}}, }, } ) bus = BusPath(name="", parent=None, client=lightbus.BusClient(config=config, loop=loop)) await asyncio.sleep(0.1) calls = 0 def listener(*args, **kwargs): nonlocal calls calls += 1 await bus.listen_multiple_async([bus.api_a.event_a, bus.api_b.event_b], listener=listener) await asyncio.sleep(0.1) await bus.api_a.event_a.fire_async() await bus.api_b.event_b.fire_async() await asyncio.sleep(0.1) assert calls == 2
async def test_event(bus: lightbus.path.BusPath, dummy_api, stream_use): """Full event integration test""" bus.client.transport_registry.get_event_transport("default").stream_use = stream_use manually_set_plugins({}) received_messages = [] async def listener(event_message, **kwargs): nonlocal received_messages received_messages.append(event_message) await bus.my.dummy.my_event.listen_async(listener) await asyncio.sleep(0.01) await bus.my.dummy.my_event.fire_async(field="Hello! 😎") await asyncio.sleep(0.01) # await asyncio.gather(co_fire_event(), co_listen_for_events()) assert len(received_messages) == 1 assert received_messages[0].kwargs == {"field": "Hello! 😎"} assert received_messages[0].api_name == "my.dummy" assert received_messages[0].event_name == "my_event" assert received_messages[0].native_id
async def test_multiple_rpc_transports(loop, server, redis_server_b, consume_rpcs): """Configure a bus with two redis transports and ensure they write to the correct redis servers""" registry.add(ApiA()) registry.add(ApiB()) manually_set_plugins(plugins={}) redis_server_a = server port_a = redis_server_a.tcp_address.port port_b = redis_server_b.tcp_address.port logging.warning(f"Server A port: {port_a}") logging.warning(f"Server B port: {port_b}") config = Config.load_dict( { "bus": {"schema": {"transport": {"redis": {"url": f"redis://localhost:{port_a}"}}}}, "apis": { "default": { "rpc_transport": {"redis": {"url": f"redis://localhost:{port_a}"}}, "result_transport": {"redis": {"url": f"redis://localhost:{port_a}"}}, }, "api_b": { "rpc_transport": {"redis": {"url": f"redis://localhost:{port_b}"}}, "result_transport": {"redis": {"url": f"redis://localhost:{port_b}"}}, }, }, } ) bus = BusPath(name="", parent=None, client=lightbus.BusClient(config=config, loop=loop)) asyncio.ensure_future(consume_rpcs(bus)) await asyncio.sleep(0.1) await bus.api_a.rpc_a.call_async() await bus.api_b.rpc_b.call_async()
async def test_event_exception_in_listener_batch_fetch( bus: lightbus.path.BusPath, dummy_api, redis_client): """Add a number of events to a stream the startup a listener which errors. The listener will fetch them all at once.""" manually_set_plugins({}) received_messages = [] # Don't shutdown on error bus.client.config.api("default").on_error = OnError.STOP_LISTENER async def listener(event_message, **kwargs): nonlocal received_messages received_messages.append(event_message) raise Exception() await bus.my.dummy.my_event.fire_async(field="Hello! 😎") await bus.my.dummy.my_event.fire_async(field="Hello! 😎") await bus.my.dummy.my_event.fire_async(field="Hello! 😎") await bus.my.dummy.my_event.listen_async(listener, bus_options={"since": "0"}) await asyncio.sleep(0.1) # Died when processing first message, so we only saw one message assert len(received_messages) == 1 # Now check we have not acked any of them messages = await redis_client.xrange("my.dummy.my_event:stream") # No message0 here because the stream already exists (because we've just added events to it) message1_id, message2_id, message3_id = [id_ for id_, *_ in messages] pending_messages = await redis_client.xpending("my.dummy.my_event:stream", "test_cg-default", "-", "+", 10, "test_consumer") assert len(pending_messages) == 3
async def test_send_event(dummy_bus: BusNode, get_dummy_events): manually_set_plugins(plugins={'metrics': MetricsPlugin()}) registry.add(TestApi()) await dummy_bus.example.test.my_event.fire_async(f=123) # What events were fired? event_messages = get_dummy_events() assert len( event_messages ) == 2 # First is the actual event, followed by the metrics event # rpc_response_received assert event_messages[1].api_name == 'internal.metrics' assert event_messages[1].event_name == 'event_fired' assert event_messages[1].kwargs.pop('timestamp') assert event_messages[1].kwargs.pop('process_name') assert event_messages[1].kwargs == { 'api_name': 'example.test', 'event_name': 'my_event', 'event_id': 'event_id', 'kwargs': { 'f': 123 } }
async def test_remote_rpc_call(dummy_bus: BusNode, get_dummy_events): # Setup the bus and do the call manually_set_plugins(plugins={'metrics': MetricsPlugin()}) registry.add(TestApi()) await dummy_bus.example.test.my_method.call_async(f=123) # What events were fired? event_messages = get_dummy_events() assert len(event_messages) == 2 # rpc_call_sent assert event_messages[0].api_name == 'internal.metrics' assert event_messages[0].event_name == 'rpc_call_sent' # Pop these next two as the values are variable assert event_messages[0].kwargs.pop('timestamp') assert event_messages[0].kwargs.pop('rpc_id') assert event_messages[0].kwargs.pop('process_name') assert event_messages[0].kwargs == { 'api_name': 'example.test', 'procedure_name': 'my_method', 'kwargs': { 'f': 123 }, } # rpc_response_received assert event_messages[1].api_name == 'internal.metrics' assert event_messages[1].event_name == 'rpc_response_received' # Pop these next two as the values are variable assert event_messages[1].kwargs.pop('timestamp') assert event_messages[1].kwargs.pop('rpc_id') assert event_messages[1].kwargs.pop('process_name') assert event_messages[1].kwargs == { 'api_name': 'example.test', 'procedure_name': 'my_method', }
def do_add_base_plugin(): manually_set_plugins(plugins={"base": LightbusPlugin()})
def test_is_plugin_loaded(): assert get_plugins() is None assert is_plugin_loaded(LightbusPlugin) == False manually_set_plugins(OrderedDict([("p1", LightbusPlugin())])) assert is_plugin_loaded(LightbusPlugin) == True
def test_remove_all_plugins(): assert get_plugins() is None manually_set_plugins(OrderedDict([("p1", LightbusPlugin())])) remove_all_plugins() assert get_plugins() is None
def test_manually_set_plugins(): assert get_plugins() is None p1 = LightbusPlugin() p2 = LightbusPlugin() manually_set_plugins(OrderedDict([("p1", p1), ("p2", p2)])) assert get_plugins() == OrderedDict([("p1", p1), ("p2", p2)])
async def setup_async(self, plugins: dict = None): """Setup lightbus and get it ready to consume events and/or RPCs You should call this manually if you are calling `consume_rpcs()` directly. This you be handled for you if you are calling `run_forever()`. """ logger.info( LBullets( "Lightbus is setting up", items={ "service_name (set with -s or LIGHTBUS_SERVICE_NAME)": Bold(self.config.service_name), "process_name (with with -p or LIGHTBUS_PROCESS_NAME)": Bold(self.config.process_name), }, )) # Log the transport information rpc_transport = self.transport_registry.get_rpc_transport("default", default=None) result_transport = self.transport_registry.get_result_transport( "default", default=None) event_transport = self.transport_registry.get_event_transport( "default", default=None) log_transport_information(rpc_transport, result_transport, event_transport, self.schema.schema_transport, logger) # Log the plugins we have if plugins is None: logger.debug("Auto-loading any installed Lightbus plugins...") # Force auto-loading as many commands need to do config-less best-effort # plugin loading. But now we have the config loaded so we can # make sure we load the plugins properly. plugins = autoload_plugins(self.config, force=True) else: logger.debug("Loading explicitly specified Lightbus plugins....") manually_set_plugins(plugins) if plugins: logger.info( LBullets("Loaded the following plugins ({})".format( len(plugins)), items=plugins)) else: logger.info("No plugins loaded") # Load schema logger.debug("Loading schema...") await self.schema.load_from_bus() # Share the schema of the registered APIs for api in registry.all(): await self.schema.add_api(api) logger.info( LBullets( "Loaded the following remote schemas ({})".format( len(self.schema.remote_schemas)), items=self.schema.remote_schemas.keys(), )) for transport in self.transport_registry.get_all_transports(): await transport.open()