async def send_event(self, event_message: EventMessage, options: dict, bus_client: "BusClient"): """Publish an event""" stream = self._get_stream_names( listen_for=[(event_message.api_name, event_message.event_name)])[0] logger.debug( LBullets( L( "Enqueuing event message {} in Redis stream {}", Bold(event_message), Bold(stream), ), items=dict(**event_message.get_metadata(), kwargs=event_message.get_kwargs()), )) # Performance: I suspect getting a connection from the connection manager each time is causing # performance issues. Need to confirm. with await self.connection_manager() as redis: start_time = time.time() await redis.xadd( stream=stream, fields=self.serializer(event_message), max_len=self.max_stream_length or None, exact_len=False, ) logger.debug( L( "Enqueued event message {} in Redis in {} stream {}", Bold(event_message), human_time(time.time() - start_time), Bold(stream), ))
async def test_event_execution(track_called_hooks, new_bus, worker: Worker, loop, add_base_plugin, dummy_api): add_base_plugin() bus = new_bus() bus.client.register_api(dummy_api) hook_tracker = track_called_hooks(bus.client) bus.client.listen_for_event("my.dummy", "my_event", lambda *a, **kw: None, listener_name="test") async with worker(bus): await asyncio.sleep(0.1) # Send the event message using a lower-level API to avoid triggering the # before_event_sent & after_event_sent plugin hooks. We don't care about those here event_message = EventMessage(api_name="my.dummy", event_name="my_event", kwargs={"field": "a"}) event_transport = bus.client.transport_registry.get_event_transport( "default") await event_transport.send_event(event_message, options={}) await asyncio.sleep(0.1) # FYI: There is a chance of events firing twice (because the dummy_bus keeps firing events), assert "before_event_execution" in hook_tracker() assert "after_event_execution" in hook_tracker()
async def test_event_execution(called_hooks, dummy_bus: BusPath, loop, add_base_plugin, dummy_api): add_base_plugin() await dummy_bus.client.register_api_async(dummy_api) await dummy_bus.client.listen_for_event("my.dummy", "my_event", lambda *a, **kw: None, listener_name="test") await asyncio.sleep(0.1) # Send the event message using a lower-level API to avoid triggering the # before_event_sent & after_event_sent plugin hooks. We don't care about those here event_message = EventMessage(api_name="my.dummy", event_name="my_event", kwargs={"field": 1}) event_transport = dummy_bus.client.transport_registry.get_event_transport( "default") await event_transport.send_event(event_message, options={}, bus_client=None) await asyncio.sleep(0.1) # There is a chance of events firing twice (because the dummy_bus keeps firing events), # so cast to the lists to sets first before comparing assert set( called_hooks()) == {"before_event_execution", "after_event_execution"}
async def test_event_execution(called_hooks, dummy_bus: BusPath, loop, add_base_plugin, dummy_api): add_base_plugin() task = await dummy_bus.client.listen_for_event("my.dummy", "my_event", lambda *a, **kw: None) await asyncio.sleep(0.1) # Send the event message using a lower-level API to avoid triggering the # before_event_sent & after_event_sent plugin hooks. We don't care about those here event_message = EventMessage(api_name="my.dummy", event_name="my_event", kwargs={"field": 1}) event_transport = dummy_bus.client.transport_registry.get_event_transport( "default") await event_transport.send_event(event_message, options={}) await asyncio.sleep(0.1) assert called_hooks() == [ "before_event_execution", "after_event_execution" ] task.cancel() try: await task except asyncio.CancelledError: pass
async def after_server_stopped(self, *, bus_client: BusClient, loop): await bus_client.event_transport.send_event( EventMessage(api_name='internal.state', event_name='server_stopped', kwargs=dict(process_name=bus_client.process_name, )), options={}, )
async def _send_ping(self, bus_client: BusClient): while True: await asyncio.sleep(self.ping_interval) await bus_client.event_transport.send_event( EventMessage(api_name='internal.state', event_name='server_ping', kwargs=self.get_state_kwargs(bus_client)), options={}, )
async def test_send_event_return_value(redis_event_transport: RedisEventTransport, redis_client): event_message = await redis_event_transport.send_event( EventMessage(api_name="my.api", event_name="my_event", id="123", kwargs={"field": "value"}), options={}, ) assert isinstance(event_message, EventMessage) assert event_message.id assert event_message.native_id assert re.match(r"\d+-0", event_message.native_id)
def test_blob_serializer(): serializer = BlobMessageSerializer() serialized = serializer( EventMessage(api_name="my.api", event_name="my_event", id="123", kwargs={"field": "value"}) ) assert json.loads(serialized) == { "metadata": {"api_name": "my.api", "event_name": "my_event", "id": "123", "version": 1}, "kwargs": {"field": "value"}, }
async def test_execute_events(dummy_bus: BusPath, dummy_listener, get_dummy_events, mocker): event_transport = dummy_bus.client.transport_registry.get_event_transport( "default") mocker.patch.object( event_transport, "_get_fake_message", return_value=EventMessage(api_name="example.test", event_name="my_event", kwargs={"f": 123}), ) await dummy_listener("example.test", "my_event") # Setup the bus and do the call manually_set_plugins( plugins={ "metrics": MetricsPlugin(service_name="foo", process_name="bar") }) registry.add(TestApi()) # The dummy transport will fire an every every 0.1 seconds await asyncio.sleep(0.15) event_messages = get_dummy_events() assert len(event_messages) == 2 # before_rpc_execution assert event_messages[0].api_name == "internal.metrics" assert event_messages[0].event_name == "event_received" assert event_messages[0].kwargs.pop("timestamp") assert event_messages[0].kwargs == { "api_name": "example.test", "event_name": "my_event", "event_id": "event_id", "kwargs": { "f": 123 }, "service_name": "foo", "process_name": "bar", } # after_rpc_execution assert event_messages[1].api_name == "internal.metrics" assert event_messages[1].event_name == "event_processed" assert event_messages[1].kwargs.pop("timestamp") assert event_messages[1].kwargs == { "api_name": "example.test", "event_name": "my_event", "event_id": "event_id", "kwargs": { "f": 123 }, "service_name": "foo", "process_name": "bar", }
async def send_event(self, event_message: EventMessage, options: dict): """Publish an event""" stream = '{}.{}:stream'.format(event_message.api_name, event_message.event_name) logger.debug( LBullets( L("Enqueuing event message {} in Redis stream {}", Bold(event_message), Bold(stream)), items=event_message.to_dict() ) ) pool = await self.get_redis_pool() with await pool as redis: start_time = time.time() # TODO: MAXLEN await redis.xadd(stream=stream, fields=encode_message_fields(event_message.to_dict())) logger.info(L( "Enqueued event message {} in Redis in {} stream {}", Bold(event_message), human_time(time.time() - start_time), Bold(stream) ))
async def after_worker_stopped(self, *, client: "BusClient"): event_transport = client.transport_registry.get_event_transport("internal.metrics") await event_transport.send_event( EventMessage( api_name="internal.state", event_name="worker_stopped", kwargs=dict(process_name=self.process_name, service_name=self.service_name), ), options={}, ) await cancel(self._ping_task)
async def _send_ping(self, client: "BusClient"): event_transport = client.transport_registry.get_event_transport("internal.metrics") while True: await asyncio.sleep(self.ping_interval) await event_transport.send_event( EventMessage( api_name="internal.state", event_name="worker_ping", kwargs=self.get_state_kwargs(client), ), options={}, )
async def fire_event(self, api_name, name, kwargs: dict = None, options: dict = None): kwargs = kwargs or {} try: api = registry.get(api_name) except UnknownApi: raise UnknownApi( "Lightbus tried to fire the event {api_name}.{name}, but could not find API {api_name} in the " "registry. An API being in the registry implies you are an authority on that API. Therefore, " "Lightbus requires the API to be in the registry as it is a bad idea to fire " "events on behalf of remote APIs. However, this could also be caused by a typo in the " "API name or event name, or be because the API class has not been " "imported. ".format(**locals())) self._validate_name(api_name, "event", name) try: event = api.get_event(name) except EventNotFound: raise EventNotFound( "Lightbus tried to fire the event {api_name}.{name}, but the API {api_name} does not " "seem to contain an event named {name}. You may need to define the event, you " "may also be using the incorrect API. Also check for typos.". format(**locals())) if set(kwargs.keys()) != _parameter_names(event.parameters): raise InvalidEventArguments( "Invalid event arguments supplied when firing event. Attempted to fire event with " "{} arguments: {}. Event expected {}: {}".format( len(kwargs), sorted(kwargs.keys()), len(event.parameters), sorted(_parameter_names(event.parameters)), )) kwargs = deform_to_bus(kwargs) event_message = EventMessage(api_name=api.meta.name, event_name=name, kwargs=kwargs) self._validate(event_message, "outgoing") event_transport = self.transport_registry.get_event_transport(api_name) await self._plugin_hook("before_event_sent", event_message=event_message) logger.info( L("📤 Sending event {}.{}".format(Bold(api_name), Bold(name)))) await event_transport.send_event(event_message, options=options) await self._plugin_hook("after_event_sent", event_message=event_message)
async def test_reconnect_upon_send_event( redis_event_transport: RedisEventTransport, redis_client, get_total_redis_connections ): await redis_client.execute(b"CLIENT", b"KILL", b"TYPE", b"NORMAL") assert await get_total_redis_connections() == 1 await redis_event_transport.send_event( EventMessage(api_name="my.api", event_name="my_event", id="123", kwargs={"field": "value"}), options={}, ) messages = await redis_client.xrange("my.api.my_event:stream") assert len(messages) == 1 assert await get_total_redis_connections() == 2
async def fire_event(self, api_name, name, kwargs: dict = None, options: dict = None): kwargs = kwargs or {} try: api = self.api_registry.get(api_name) except UnknownApi: raise UnknownApi( "Lightbus tried to fire the event {api_name}.{name}, but no API named {api_name} was found in the " "registry. An API being in the registry implies you are an authority on that API. Therefore, " "Lightbus requires the API to be in the registry as it is a bad idea to fire " "events on behalf of remote APIs. However, this could also be caused by a typo in the " "API name or event name, or be because the API class has not been " "registered using bus.client.register_api(). ".format(**locals()) ) validate_event_or_rpc_name(api_name, "event", name) try: event = api.get_event(name) except EventNotFound: raise EventNotFound( "Lightbus tried to fire the event {api_name}.{name}, but the API {api_name} does not " "seem to contain an event named {name}. You may need to define the event, you " "may also be using the incorrect API. Also check for typos.".format(**locals()) ) parameter_names = {p.name if isinstance(p, Parameter) else p for p in event.parameters} if set(kwargs.keys()) != parameter_names: raise InvalidEventArguments( "Invalid event arguments supplied when firing event. Attempted to fire event with " "{} arguments: {}. Event expected {}: {}".format( len(kwargs), sorted(kwargs.keys()), len(event.parameters), sorted(parameter_names), ) ) kwargs = deform_to_bus(kwargs) event_message = EventMessage( api_name=api.meta.name, event_name=name, kwargs=kwargs, version=api.meta.version ) validate_outgoing(self.config, self.schema, event_message) await self.hook_registry.execute("before_event_sent", event_message=event_message) logger.info(L("📤 Sending event {}.{}".format(Bold(api_name), Bold(name)))) await self.producer.send(SendEventCommand(message=event_message, options=options)).wait() await self.hook_registry.execute("after_event_sent", event_message=event_message)
async def test_history_get_all_multiple_batches( redis_event_transport: RedisEventTransport, redis_client ): message = EventMessage(native_id="", api_name="my_api", event_name="my_event") data = ByFieldMessageSerializer()(message) await redis_client.xadd("my_api.my_event:stream", data, message_id=b"1-0") await redis_client.xadd("my_api.my_event:stream", data, message_id=b"2-0") await redis_client.xadd("my_api.my_event:stream", data, message_id=b"3-0") await redis_client.xadd("my_api.my_event:stream", data, message_id=b"4-0") await redis_client.xadd("my_api.my_event:stream", data, message_id=b"5-0") messages = redis_event_transport.history("my_api", "my_event", batch_size=2) message_ids = {m.native_id async for m in messages} assert len(message_ids) == 5
async def test_send_event(redis_event_transport: RedisEventTransport, redis_client): await redis_event_transport.send_event( EventMessage(api_name="my.api", event_name="my_event", id="123", kwargs={"field": "value"}), options={}, ) messages = await redis_client.xrange("my.api.my_event:stream") assert len(messages) == 1 assert messages[0][1] == { b"api_name": b"my.api", b"event_name": b"my_event", b"id": b"123", b"version": b"1", b":field": b'"value"', }
def send_event(self, bus_client, event_name_, **kwargs) -> Coroutine: """Send an event to the bus Note that we bypass using BusClient directly, otherwise we would trigger this plugin again thereby causing an infinite loop. """ kwargs.setdefault('timestamp', datetime.utcnow().timestamp()) kwargs.setdefault('process_name', bus_client.process_name) return bus_client.event_transport.send_event( EventMessage(api_name='internal.metrics', event_name=event_name_, kwargs=kwargs), options={}, )
def test_by_field_serializer(): serializer = ByFieldMessageSerializer() serialized = serializer( EventMessage( api_name="my.api", event_name="my_event", kwargs={"field": "value"}, id="123", version=2 ) ) assert serialized == { "api_name": "my.api", "event_name": "my_event", ":field": '"value"', "id": "123", "version": 2, }
async def test_send_event(redis_event_transport: RedisEventTransport, redis_client): await redis_event_transport.send_event(EventMessage( api_name='my.api', event_name='my_event', kwargs={'field': 'value'}, ), options={}) messages = await redis_client.xrange('my.api.my_event:stream') assert len(messages) == 1 assert messages[0][1] == { b'api_name': b'"my.api"', b'event_name': b'"my_event"', b'kw:field': b'"value"', }
async def test_max_len_set_to_none( redis_event_transport: RedisEventTransport, redis_client, caplog ): """Make sure the event stream does not get truncated when max_stream_length = None """ caplog.set_level(logging.WARNING) redis_event_transport.max_stream_length = None for x in range(0, 200): await redis_event_transport.send_event( EventMessage(api_name="my.api", event_name="my_event", kwargs={"field": "value"}), options={}, ) messages = await redis_client.xrange("my.api.my_event:stream") assert len(messages) == 200
async def test_max_len_truncating(redis_event_transport: RedisEventTransport, redis_client, caplog): """Make sure the event stream gets truncated Note that truncation is approximate """ caplog.set_level(logging.WARNING) redis_event_transport.max_stream_length = 100 for x in range(0, 200): await redis_event_transport.send_event( EventMessage(api_name="my.api", event_name="my_event", kwargs={"field": "value"}), options={}, ) messages = await redis_client.xrange("my.api.my_event:stream") assert len(messages) >= 100 assert len(messages) < 150
def send_event(self, client, event_name_, **kwargs) -> Coroutine: """Send an event to the bus Note that we bypass using BusClient directly, otherwise we would trigger this plugin again thereby causing an infinite loop. """ kwargs.setdefault("timestamp", datetime.utcnow().timestamp()) kwargs.setdefault("service_name", self.service_name) kwargs.setdefault("process_name", self.process_name) kwargs = deform_to_bus(kwargs) event_transport = client.transport_registry.get_event_transport("internal.metrics") return event_transport.send_event( EventMessage(api_name="internal.metrics", event_name=event_name_, kwargs=kwargs), options={}, bus_client=client, )
async def test_send_event_per_api_stream(redis_event_transport: RedisEventTransport, redis_client): redis_event_transport.stream_use = StreamUse.PER_API await redis_event_transport.send_event( EventMessage(api_name="my.api", event_name="my_event", kwargs={"field": "value"}, id="123"), options={}, bus_client=None, ) messages = await redis_client.xrange("my.api.*:stream") assert len(messages) == 1 assert messages[0][1] == { b"api_name": b"my.api", b"event_name": b"my_event", b"id": b"123", b"version": b"1", b":field": b'"value"', }
async def test_execute_events(dummy_bus: BusNode, event_consumer, get_dummy_events, mocker): mocker.patch.object(dummy_bus.bus_client.event_transport, '_get_fake_messages', return_value=[ EventMessage(api_name='example.test', event_name='my_event', kwargs={'f': 123}) ]) # Setup the bus and do the call manually_set_plugins(plugins={'metrics': MetricsPlugin()}) registry.add(TestApi()) # The dummy transport will fire an every every 0.1 seconds await asyncio.sleep(0.15) event_messages = get_dummy_events() assert len(event_messages) == 2 # before_rpc_execution assert event_messages[0].api_name == 'internal.metrics' assert event_messages[0].event_name == 'event_received' assert event_messages[0].kwargs.pop('timestamp') assert event_messages[0].kwargs.pop('process_name') assert event_messages[0].kwargs == { 'api_name': 'example.test', 'event_name': 'my_event', 'event_id': 'event_id', 'kwargs': { 'f': 123 } } # after_rpc_execution assert event_messages[1].api_name == 'internal.metrics' assert event_messages[1].event_name == 'event_processed' assert event_messages[1].kwargs.pop('timestamp') assert event_messages[1].kwargs.pop('process_name') assert event_messages[1].kwargs == { 'api_name': 'example.test', 'event_name': 'my_event', 'event_id': 'event_id', 'kwargs': { 'f': 123 } }
async def before_server_start(self, *, bus_client: BusClient, loop): await bus_client.event_transport.send_event( EventMessage(api_name='internal.state', event_name='server_started', kwargs=self.get_state_kwargs(bus_client)), options={}, ) if self.do_ping: logger.info('Ping messages will be sent every {} seconds'.format( self.ping_interval)) asyncio.ensure_future(handle_aio_exceptions( self._send_ping(bus_client)), loop=loop) else: logger.warning( 'Ping events have been disabled. This will reduce log volume and bus traffic, but ' 'may result in this Lightbus server not appearing in the Lightbus admin interface.' )
async def before_worker_start(self, *, client: "BusClient"): event_transport = client.transport_registry.get_event_transport("internal.metrics") await event_transport.send_event( EventMessage( api_name="internal.state", event_name="worker_started", kwargs=self.get_state_kwargs(client), ), options={}, ) if self.ping_enabled: logger.info("Ping messages will be sent every {} seconds".format(self.ping_interval)) self._ping_task = asyncio.ensure_future(self._send_ping(client)) else: logger.warning( "Ping events have been disabled. This will reduce log volume and bus traffic, but " "may result in this Lightbus worker not appearing in the Lightbus admin interface." )
async def test_history_get_subset_multiple_batches( redis_event_transport: RedisEventTransport, redis_client ): message = EventMessage(native_id="", api_name="my_api", event_name="my_event") data = ByFieldMessageSerializer()(message) await redis_client.xadd("my_api.my_event:stream", data, message_id=b"1-0") await redis_client.xadd("my_api.my_event:stream", data, message_id=b"2-0") await redis_client.xadd("my_api.my_event:stream", data, message_id=b"3-0") await redis_client.xadd("my_api.my_event:stream", data, message_id=b"4-0") await redis_client.xadd("my_api.my_event:stream", data, message_id=b"5-0") messages = redis_event_transport.history( "my_api", "my_event", batch_size=2, start=datetime.fromtimestamp(0.002), stop=datetime.fromtimestamp(0.004), ) message_ids = {m.native_id async for m in messages} assert message_ids == {"2-0", "3-0", "4-0"}
async def fetch_events(self) -> Tuple[Sequence[EventMessage], Any]: pool = await self.get_redis_pool() with await pool as redis: if not self._streams: logger.debug('Event backend has been given no events to consume. Event backend will sleep.') self._task = asyncio.ensure_future(asyncio.sleep(3600 * 24 * 365)) else: logger.info(LBullets( 'Consuming events from', items={ '{} ({})'.format(*v) for v in self._streams.items() } )) # TODO: Count/timeout self._task = asyncio.ensure_future( redis.xread( streams=list(self._streams.keys()), latest_ids=list(self._streams.values()), count=10, # TODO: Make configurable, add timeout too ) ) try: stream_messages = await self._task or [] except asyncio.CancelledError as e: if self._reload: # Streams to listen on have changed. # Bail out and let this method get called again, # at which point we'll pickup the new streams. logger.debug('Event transport reloading.') stream_messages = [] self._reload = False else: raise event_messages = [] latest_ids = {} for stream, message_id, fields in stream_messages: stream = decode(stream, 'utf8') message_id = decode(message_id, 'utf8') decoded_fields = decode_message_fields(fields) # Keep track of which event ID we are up to. We will store these # in consumption_complete(), once we know the events have definitely # been consumed. latest_ids[stream] = message_id # Unfortunately, these is an edge-case when BOTH: # 1. We are consuming events from 'now' (i.e. event ID '$'), the default # 2. There is an unhandled error when processing the FIRST batch of events # In which case, the next iteration would start again from '$', in which # case we would loose events. Therefore 'subtract one' from the message ID # and store that immediately. Subtracting one is imprecise, as there is a SLIM # chance we could grab another event in the process. However, if events are # being consumed from 'now' then the developer presumably doesn't care about # a high level of precision. if self._streams[stream] == '$': self._streams[stream] = redis_stream_id_subtract_one(message_id) event_messages.append( EventMessage.from_dict(decoded_fields) ) logger.debug(LBullets( L("⬅ Received event {} on stream {}", Bold(message_id), Bold(stream)), items=decoded_fields )) return event_messages, latest_ids
def _get_fake_messages(self): return [ EventMessage(api_name='my_company.auth', event_name='user_registered', kwargs={'example': 'value'}) ]