async def _publish_messages(self, messages: List[str]) -> None: data = LoggerRabbitMessage( node_id=self._node_id, user_id=self._user_id, project_id=self._project_id, messages=messages, ) assert self._logs_exchange # nosec await self._logs_exchange.publish( aio_pika.Message(body=data.json().encode()), routing_key="")
async def log_message_parser(app: web.Application, data: bytes) -> None: rabbit_message = LoggerRabbitMessage.parse_raw(data) socket_messages: List[SocketMessageDict] = [{ "event_type": SOCKET_IO_LOG_EVENT, "data": { "messages": rabbit_message.messages, "node_id": f"{rabbit_message.node_id}", }, }] await send_messages(app, f"{rabbit_message.user_id}", socket_messages)
async def _task_log_change_handler(self, event: str) -> None: task_log_event = TaskLogEvent.parse_raw(event) logger.debug("received task log update: %s", task_log_event) *_, user_id, project_id, node_id = parse_dask_job_id(task_log_event.job_id) message = LoggerRabbitMessage( user_id=user_id, project_id=project_id, node_id=node_id, messages=[task_log_event.log], ) await self.rabbitmq_client.publish_message(message)
async def _publish_in_rabbit( user_id: int, project_id: UUIDStr, node_uuid: UUIDStr, num_messages: int, rabbit_exchanges: RabbitExchanges, ) -> Tuple[LogMessages, ProgressMessages, InstrumMessages]: log_messages = [ LoggerRabbitMessage( user_id=user_id, project_id=project_id, node_id=node_uuid, messages=[f"log number {n}"], ) for n in range(num_messages) ] progress_messages = [ ProgressRabbitMessage( user_id=user_id, project_id=project_id, node_id=node_uuid, progress=float(n) / float(num_messages), ) for n in range(num_messages) ] # indicate container is started instrumentation_start_message = ( instrumentation_stop_message) = InstrumentationRabbitMessage( metrics="service_started", user_id=user_id, project_id=project_id, node_id=node_uuid, service_uuid=node_uuid, service_type="COMPUTATIONAL", service_key="some/service/awesome/key", service_tag="some-awesome-tag", ) instrumentation_stop_message.metrics = "service_stopped" instrumentation_stop_message.result = RunningState.SUCCESS instrumentation_messages = [ instrumentation_start_message, instrumentation_stop_message, ] await rabbit_exchanges.instrumentation.publish( aio_pika.Message( body=instrumentation_start_message.json().encode(), content_type="text/json", ), routing_key="", ) for n in range(num_messages): await rabbit_exchanges.logs.publish( aio_pika.Message(body=log_messages[n].json().encode(), content_type="text/json"), routing_key="", ) await rabbit_exchanges.progress.publish( aio_pika.Message(body=progress_messages[n].json().encode(), content_type="text/json"), routing_key="", ) # indicate container is stopped await rabbit_exchanges.instrumentation.publish( aio_pika.Message( body=instrumentation_stop_message.json().encode(), content_type="text/json", ), routing_key="", ) return (log_messages, progress_messages, instrumentation_messages)
async def rabbit_message_handler(message: aio_pika.IncomingMessage): incoming_data.append(LoggerRabbitMessage.parse_raw(message.body))
async def test_rabbitmq( rabbit_service: RabbitSettings, rabbit_queue: aio_pika.Queue, mocker: MockerFixture, user_id: UserID, project_id: ProjectID, node_id: NodeID, app: FastAPI, ): rabbit = RabbitMQ(app) assert rabbit mock_close_connection_cb = mocker.patch( "simcore_service_dynamic_sidecar.core.rabbitmq._close_callback") mock_close_channel_cb = mocker.patch( "simcore_service_dynamic_sidecar.core.rabbitmq._channel_close_callback" ) incoming_data: List[LoggerRabbitMessage] = [] async def rabbit_message_handler(message: aio_pika.IncomingMessage): incoming_data.append(LoggerRabbitMessage.parse_raw(message.body)) await rabbit_queue.consume(rabbit_message_handler, exclusive=True, no_ack=True) await rabbit.connect() assert rabbit._connection.ready # pylint: disable=protected-access log_msg: str = "I am logging" log_messages: List[str] = ["I", "am a logger", "man..."] log_more_messages: List[str] = [f"msg{1}" for i in range(10)] await rabbit.post_log_message(log_msg) await rabbit.post_log_message(log_messages) # make sure the first 2 messages are # sent in the same chunk await asyncio.sleep(SLEEP_BETWEEN_SENDS * 1.1) await rabbit.post_log_message(log_more_messages) # wait for all the messages to be delivered, # need to make sure all messages are delivered await asyncio.sleep(SLEEP_BETWEEN_SENDS * 1.1) # if this fails the above sleep did not work assert len( incoming_data) == 2, f"missing incoming data: {pformat(incoming_data)}" assert incoming_data[0] == LoggerRabbitMessage( messages=[log_msg] + log_messages, node_id=node_id, project_id=project_id, user_id=user_id, ) assert incoming_data[1] == LoggerRabbitMessage( messages=log_more_messages, node_id=node_id, project_id=project_id, user_id=user_id, ) # ensure closes correctly await rabbit.close() mock_close_connection_cb.assert_called_once() mock_close_channel_cb.assert_called_once()