def test_multiple_instance(self): """ Test Clients with no instance """ message_bus_1 = MessageBus() message_bus_2 = MessageBus() message_bus_3 = MessageBus() admin = MessageBusAdmin(admin_id='admin', message_bus=message_bus_1) self.assertTrue(TestMessage._msg_type in admin.list_message_types()) messages = [] producer = MessageProducer(producer_id='p1', \ message_type=TestMessage._msg_type, method='sync', \ message_bus=message_bus_2) self.assertIsNotNone(producer, "Producer not found") for i in range(0, TestMessage._msg_count): messages.append("This is message" + str(i)) producer.send(messages) consumer = MessageConsumer(consumer_id='msys', consumer_group='connn', \ message_types=[TestMessage._msg_type], auto_ack=True, \ offset='latest', message_bus=message_bus_3) count = 0 while True: try: message = consumer.receive() if isinstance(message, bytes): count += 1 consumer.ack() except Exception: self.assertEqual(count, TestMessage._msg_count) break
def reset(config_path: str): """Remove/Delete all the data/logs that was created by user/testing.""" from cortx.utils.message_bus.error import MessageBusError try: # use gconf to deregister IEM from cortx.utils.message_bus import MessageBusAdmin, MessageBus from cortx.utils.message_bus import MessageProducer Conf.load(GCONF_INDEX, config_path, skip_reload=True) message_bus_backend = Conf.get('config', MSG_BUS_BACKEND_KEY) message_server_endpoints = Conf.get('config',\ f'{EXTERNAL_KEY}>{message_bus_backend}>endpoints') MessageBus.init(message_server_endpoints) mb = MessageBusAdmin(admin_id='reset') message_types_list = mb.list_message_types() if message_types_list: for message_type in message_types_list: producer = MessageProducer(producer_id=message_type,\ message_type=message_type, method='sync') producer.delete() except MessageBusError as e: raise SetupError(e.rc, "Can not reset Message Bus. %s", e) except Exception as e: raise SetupError( errors.ERR_OP_FAILED, "Internal error, can not \ reset Message Bus. %s", e) return 0
def init(cls, component: str, source: str, cluster_id: str, \ message_server_endpoints: str, **message_server_kwargs): """ Set the Event Message context Parameters: component Component that generates the IEM. For e.g. 'S3', 'SSPL' source Single character that indicates the type of component. For e.g. H-Hardware, S-Software, F-Firmware, O-OS """ cls._component = component cls._source = source cls._site_id = 1 cls._rack_id = 1 cls._node_id = Conf.machine_id cls._cluster_id = cluster_id if cls._component is None: Log.error("Invalid component type: %s" % cls._component ) raise EventMessageError(errno.EINVAL, "Invalid component type: %s", \ cls._component) if cls._source not in cls._SOURCE.keys(): Log.error("Invalid source type: %s" % cls._source) raise EventMessageError(errno.EINVAL, "Invalid source type: %s", \ cls._source) MessageBus.init(message_server_endpoints, **message_server_kwargs) cls._producer = MessageProducer(producer_id='event_producer', \ message_type='IEM', method='sync') Log.info("IEM Producer initialized for component %s and source %s" % \ (cls._component, cls._source))
def cleanup(pre_factory: bool, config_path: str): """Remove/Delete all the data that was created after post install.""" # delete message_types from cortx.utils.message_bus.error import MessageBusError try: # use gconf to clean and delete all message type in messagebus from cortx.utils.message_bus import MessageBus, MessageBusAdmin Conf.load(GCONF_INDEX, config_path, skip_reload=True) message_bus_backend = Conf.get(GCONF_INDEX, MSG_BUS_BACKEND_KEY) message_server_endpoints = Conf.get(GCONF_INDEX,\ f'{EXTERNAL_KEY}>{message_bus_backend}>endpoints') MessageBus.init(message_server_endpoints) mb = MessageBusAdmin(admin_id='cleanup') message_types_list = mb.list_message_types() if message_types_list: mb.deregister_message_type(message_types_list) except MessageBusError as e: raise SetupError(e.rc, "Can not cleanup Message Bus. %s", e) except Exception as e: raise SetupError( errors.ERR_OP_FAILED, "Can not cleanup Message \ Bus. %s", e) return 0
def test_same_instance(self): """ Test Singleton functionality """ message_bus_1 = MessageBus() message_bus_2 = MessageBus() # Check for same instance self.assertEqual(message_bus_1, message_bus_2)
def init(): """ Initialize utils MessageBus Library with kafka endpoints once per service. In future utils will throw error if init done multiple times. If any new service will come which uses MessageBus then init should be done there. """ message_server_endpoints = Conf.get( const.HA_GLOBAL_INDEX, f"kafka_config{const._DELIM}endpoints") utils_message_bus.init(message_server_endpoints)
def __init__(self, client_type: str, **client_conf: dict): self._message_bus = MessageBus() if 'message_bus' not in \ client_conf.keys() or client_conf['message_bus'] is None else \ client_conf['message_bus'] self._message_bus.init_client(client_type, **client_conf) self._client_conf = client_conf Log.debug(f"MessageBusClient: initialized with arguments" \ f" client_type: {client_type}, kwargs: {client_conf}")
def deregister_message_type(self, message_types: list): """ Deregisters a list of message types. Parameters: message_types This is essentially equivalent to the list of queue topic name. For e.g. ["Alert"] """ client_id = self._get_conf('client_id') MessageBus.deregister_message_type(client_id, message_types)
def register_message_type(self, message_types: list, partitions: int): """ Registers a list of message types. Parameters: message_types This is essentially equivalent to the list of queue topic name. For e.g. ["Alert"] partitions Integer that represents number of partitions to be created. """ client_id = self._get_conf('client_id') MessageBus.register_message_type(client_id, message_types, partitions)
def add_concurrency(self, message_type: str, concurrency_count: int): """ To achieve concurrency for a message type. Parameters: message_type This is essentially equivalent to queue/topic name. For e.g. "Alert" concurrency_count Integer to achieve concurrency among consumers """ client_id = self._get_conf('client_id') MessageBus.add_concurrency(client_id, message_type,\ concurrency_count)
def send(self, messages: list): """ Sends list of messages to the Message Bus. Parameters: messages A list of messages sent to Message Bus """ message_type = self._get_conf('message_type') method = self._get_conf('method') client_id = self._get_conf('client_id') messages = self._get_str_message_list(messages) MessageBus.send(client_id, message_type, method, messages)
def setUpClass(cls,\ cluster_conf_path: str = 'yaml:///etc/cortx/cluster.conf'): """Register the test message_type.""" if TestMessage._cluster_conf_path: cls.cluster_conf_path = TestMessage._cluster_conf_path else: cls.cluster_conf_path = cluster_conf_path Conf.load('config', cls.cluster_conf_path, skip_reload=True) message_server_endpoints = Conf.get('config',\ 'cortx>external>kafka>endpoints') Log.init('message_bus', '/var/log', level='INFO', \ backup_count=5, file_size_in_mb=5) MessageBus.init(message_server_endpoints=message_server_endpoints) cls._admin = MessageBusAdmin(admin_id='register') cls._admin.register_message_type(message_types= \ [TestMessage._message_type], partitions=1)
class TestMessage(unittest.TestCase): """ Test MessageBus related functionality """ message_bus = MessageBus() def test_send(self): """ Test Send Message """ messages = [] producer = MessageProducer(TestMessage.message_bus, \ producer_id='p1', message_type='big') self.assertIsNotNone(producer, "Producer not found") for i in range(0, 100): messages.append("This is message" + str(i)) self.assertEqual(len(messages), 100) self.assertIsInstance(messages, list) producer.send(messages) def test_count(self): """ Test Unread Message Count """ consumer = MessageConsumer(TestMessage.message_bus, \ consumer_id='sensors', consumer_group='test_group', \ message_types=['big'], auto_ack=True, offset='latest') self.assertIsNotNone(consumer, "Consumer not found") unread_count = consumer.get_unread_count() self.assertEqual(unread_count, 100)
class TestMessage(unittest.TestCase): """Test MessageBus related functionality.""" message_bus = MessageBus() def test_send(self): """Test Send Message.""" messages = [] producer = MessageProducer(TestMessage.message_bus, \ producer_id='sspl_sensor', message_type='Alert') self.assertIsNotNone(producer, "Producer not found") for i in range(0, 1000): messages.append("This is message" + str(i)) self.assertEqual(len(messages), 1000) self.assertIsInstance(messages, list) producer.send(messages) def test_receive(self): """Test Receive Message.""" consumer = MessageConsumer(TestMessage.message_bus, \ consumer_id='sspl_sensors', consumer_group='sspl', \ message_types=['Alert'], auto_ack=False, offset='latest') self.assertIsNotNone(consumer, "Consumer not found") count = 0 while True: try: message = consumer.receive() count += 1 self.assertIsNotNone(message, "Message not found") consumer.ack() except Exception as e: self.assertEqual(count, 1000) break
def validate(self): """Check for below requirement. 1. Validate input configs 2. Validate sspl conf is created 3. Check if message bus is accessible """ if os.geteuid() != 0: msg = "Run this command with root privileges." logger.error(msg) raise SetupError(errno.EINVAL, msg) # Validate input/provisioner configs machine_id = Utility.get_machine_id() Utility.get_config_value(consts.PRVSNR_CONFIG_INDEX, "server_node>%s>cluster_id" % machine_id) self.node_type = Utility.get_config_value( consts.PRVSNR_CONFIG_INDEX, "server_node>%s>type" % machine_id) enclosure_id = Utility.get_config_value( consts.PRVSNR_CONFIG_INDEX, "server_node>%s>storage>enclosure_id" % machine_id) self.enclosure_type = Utility.get_config_value( consts.PRVSNR_CONFIG_INDEX, "storage_enclosure>%s>type" % enclosure_id) # Validate sspl conf is created if not os.path.isfile(consts.file_store_config_path): msg = "Missing configuration - %s !! Create and rerun." % ( consts.file_store_config_path) logger.error(msg) raise SetupError(errno.EINVAL, msg) # Validate message bus is accessible self.mb = MessageBus()
def subscribe(cls, component: str, message_server_endpoints: str, \ **message_server_kwargs): """ Subscribe to IEM alerts Parameters: component Component that generates the IEM. For e.g. 'S3', 'SSPL' """ if component is None: Log.error("Invalid component type: %s" % component) raise EventMessageError(errno.EINVAL, "Invalid component type: %s", \ component) MessageBus.init(message_server_endpoints, **message_server_kwargs) cls._consumer = MessageConsumer(consumer_id='event_consumer', \ consumer_group=component, message_types=['IEM'], \ auto_ack=True, offset='earliest') Log.info("IEM Consumer initialized for component: %s" % component)
def set_message_type_expire(self, message_type: str, **kwargs): """Set expiration time for given message type.""" client_id = self._get_conf('client_id') status = MessageBus.set_message_type_expire(client_id, message_type,\ **kwargs) Log.info(f"Successfully updated {message_type} with new"+\ " configuration.") return status
def connect(self): """Connect to Message Bus.""" try: self._message_bus = MessageBus() except Exception as exception: msg = ("msg_bus init except:%s %s") % (exception, traceback.format_exc()) return False, msg return True, None
def receive(self, timeout: float = None) -> list: """ Receives messages from the Message Bus. Parameters: timeout Time in seconds to wait for the message. """ client_id = self._get_conf('client_id') return MessageBus.receive(client_id, timeout)
def setUpClass(cls, \ cluster_conf_path: str = 'yaml:///etc/cortx/cluster.conf'): """Register the test message_type.""" if TestKVPayloadMessage._cluster_conf_path: cls.cluster_conf_path = TestKVPayloadMessage._cluster_conf_path else: cls.cluster_conf_path = cluster_conf_path Conf.load('config', cls.cluster_conf_path, skip_reload=True) message_server_endpoints = Conf.get('config',\ 'cortx>external>kafka>endpoints') Log.init('message_bus', '/var/log', level='INFO', \ backup_count=5, file_size_in_mb=5) MessageBus.init(message_server_endpoints=message_server_endpoints) cls._admin = MessageBusAdmin(admin_id='register') cls._admin.register_message_type(message_types= \ [TestKVPayloadMessage._message_type], partitions=1) cls._consumer = MessageConsumer(consumer_id='kv_consumer', \ consumer_group='kv', message_types=[TestKVPayloadMessage.\ _message_type], auto_ack=True, offset='earliest') cls._producer = MessageProducer(producer_id='kv_producer', \ message_type=TestKVPayloadMessage._message_type, method='sync')
def init(config_path: str): """Perform initialization.""" # Create message_type for Event Message from cortx.utils.message_bus import MessageBus, MessageBusAdmin from cortx.utils.message_bus.error import MessageBusError try: # Read the config values # use gconf to create IEM topic Conf.load(GCONF_INDEX, config_path, skip_reload=True) message_bus_backend = Conf.get(GCONF_INDEX, MSG_BUS_BACKEND_KEY) message_server_endpoints = Conf.get(GCONF_INDEX,\ f'{EXTERNAL_KEY}>{message_bus_backend}>endpoints') MessageBus.init(message_server_endpoints) admin = MessageBusAdmin(admin_id='register') admin.register_message_type(message_types=['IEM',\ 'audit_messages'], partitions=1) except MessageBusError as e: if 'TOPIC_ALREADY_EXISTS' not in e.desc: raise SetupError(e.rc, "Unable to create message_type. %s", e) return 0
def __init__(self, message_server_endpoints, message_server_port=28300,\ cluster_id=None): super().__init__() MessageBus.init(message_server_endpoints=message_server_endpoints) from cortx.utils.iem_framework import IemRequestHandler from cortx.utils.message_bus import MessageBusRequestHandler from cortx.utils.audit_log import AuditLogRequestHandler IemRequestHandler.cluster_id = cluster_id IemRequestHandler.message_server_endpoints = message_server_endpoints self.app.add_routes([web.post('/EventMessage/event', IemRequestHandler.send), \ web.get('/EventMessage/event', IemRequestHandler.receive), \ web.post('/MessageBus/message/{message_type}', \ MessageBusRequestHandler.send), \ web.get('/MessageBus/message/{message_type}', \ MessageBusRequestHandler.receive), web.post('/AuditLog/message/', \ AuditLogRequestHandler.send), web.post('/AuditLog/webhook/', \ AuditLogRequestHandler.send_webhook_info) ]) super().run_app(web, message_server_port)
def test_send(self): """ Test Send Message. """ messages = [] message_bus = MessageBus() producer = MessageProducer(message_bus, producer_id='sel', \ message_type='Sel', method='async') self.assertIsNotNone(producer, "Producer not found") for i in range(0, 10): messages.append("This is message" + str(i)) self.assertIsInstance(messages, list) producer.send(messages)
class TestMessage(unittest.TestCase): """ Test MessageBus related functionality """ message_bus = MessageBus() _message_type = 'test_topic' _partition = 1 def test_list_message_type(self): """ Test list message type API """ admin = MessageBusAdmin(TestMessage.message_bus, admin_id='admin') admin.register_message_type(message_types=[TestMessage._message_type], \ partitions=TestMessage._partition) message_type_list = admin.list_message_types() self.assertTrue(TestMessage._message_type in message_type_list)
def test_receive(self): """ Test Receive Message. """ message_bus = MessageBus() consumer = MessageConsumer(message_bus, consumer_id='sel', \ consumer_group='sel', message_type=['Sel'], auto_ack=False, \ offset='latest') self.assertIsNotNone(consumer, "Consumer not found") while True: try: message = consumer.receive() self.assertIsNotNone(message, "Message not found") consumer.ack() except Exception as e: break
def __init__(self): from cortx.utils.message_bus import MessageBus from cortx.utils.conf_store import Conf Conf.load("index", "json:///etc/cortx/message_bus.conf") self._server = Conf.get("index", 'message_broker>cluster[0]>server') self._msg_bus = MessageBus() # Create a test topic cmd = "/opt/kafka/bin/kafka-topics.sh --create" + \ " --topic mytest --bootstrap-server " + self._server + ":9092" try: cmd_proc = SimpleProcess(cmd) res_op, res_err, res_rc = cmd_proc.run() if res_rc != 0: raise SetupError(errno.EINVAL, "Unable to test the config") except Exception as e: raise SetupError(errno.EINVAL, \ "Unable to test the config, %s", e)
class TestMessage(unittest.TestCase): """ Test MessageBus related functionality. """ message_bus = MessageBus() def test_consumer_one(self): """ Test Receive Message for consumer group 1 """ consumer = MessageConsumer(TestMessage.message_bus, \ consumer_id='sspl_sensor', consumer_group='c1', \ message_type=['test_type'], auto_ack=False, offset='latest') self.assertIsNotNone(consumer, "Consumer not found") count = 0 while True: try: message = consumer.receive() count += 1 self.assertIsNotNone(message, "Message not found") consumer.ack() except Exception as e: self.assertEqual(count, 10) break def test_consumer_two(self): """ Test Receive Message for consumer group 2 """ consumer = MessageConsumer(TestMessage.message_bus, \ consumer_id='sspl_sensor1', consumer_group='c3', \ message_type=['test_type'], auto_ack=False, offset='latest') self.assertIsNotNone(consumer, "Consumer not found") count = 0 while True: try: message = consumer.receive() count += 1 self.assertIsNotNone(message, "Message not found") consumer.ack() except Exception as e: self.assertEqual(count, 10) break
def __init__(self, event_list: List[SubscribeEvent], group_id: str = COMPONENT_ID): """ event_list - list of events to subscribe to. group_id - the group_id to pass to KafkaConsumer. """ logging.debug('Inside EventListener') self.event_manager = EventManager.get_instance() topic = self._subscribe(event_list) if topic is None: raise RuntimeError('Failed to subscribe to events') message_bus = MessageBus() self.consumer = MessageConsumer( message_bus=message_bus, consumer_id=COMPONENT_ID, consumer_group=group_id, message_types=[topic], # Why is it 'str' in cortx-py-utils?? auto_ack=str(False), offset='earliest')
def create_message_types(self): # Skip this step if sspl is being configured for node # replacement scenario as rabbitmq cluster is # already configured on the healthy node # Configure rabbitmq if not os.path.exists(consts.REPLACEMENT_NODE_ENV_VAR_FILE): message_types = [ Conf.get(consts.SSPL_CONFIG_INDEX, "RABBITMQINGRESSPROCESSOR" ">message_type"), Conf.get(consts.SSPL_CONFIG_INDEX, "RABBITMQEGRESSPROCESSOR" ">message_type") ] mb = MessageBus() mbadmin = MessageBusAdmin(mb, admin_id='admin') try: mbadmin.register_message_type(message_types=message_types, partitions=1) except MessageBusError as e: if self.topic_already_exists in e.desc: # Topic already exists pass else: raise e
def test_send_invalid_message(self): """Send invalid message format.""" message = MessageBus() with self.assertRaises(MessageBusError): TestKVPayloadMessage._producer.send([message])