def reset(config_path: str): """Remove/Delete all the data/logs that was created by user/testing.""" from cortx.utils.message_bus.error import MessageBusError try: # use gconf to deregister IEM from cortx.utils.message_bus import MessageBusAdmin, MessageBus from cortx.utils.message_bus import MessageProducer Conf.load(GCONF_INDEX, config_path, skip_reload=True) message_bus_backend = Conf.get('config', MSG_BUS_BACKEND_KEY) message_server_endpoints = Conf.get('config',\ f'{EXTERNAL_KEY}>{message_bus_backend}>endpoints') MessageBus.init(message_server_endpoints) mb = MessageBusAdmin(admin_id='reset') message_types_list = mb.list_message_types() if message_types_list: for message_type in message_types_list: producer = MessageProducer(producer_id=message_type,\ message_type=message_type, method='sync') producer.delete() except MessageBusError as e: raise SetupError(e.rc, "Can not reset Message Bus. %s", e) except Exception as e: raise SetupError( errors.ERR_OP_FAILED, "Internal error, can not \ reset Message Bus. %s", e) return 0
def send_msg(self, message): """ Sends a message """ from cortx.utils.message_bus import MessageProducer producer = MessageProducer(producer_id='setup', \ message_type='mytest', method='sync') producer.send(message) time.sleep(1)
def test_multiple_instance(self): """ Test Clients with no instance """ message_bus_1 = MessageBus() message_bus_2 = MessageBus() message_bus_3 = MessageBus() admin = MessageBusAdmin(admin_id='admin', message_bus=message_bus_1) self.assertTrue(TestMessage._msg_type in admin.list_message_types()) messages = [] producer = MessageProducer(producer_id='p1', \ message_type=TestMessage._msg_type, method='sync', \ message_bus=message_bus_2) self.assertIsNotNone(producer, "Producer not found") for i in range(0, TestMessage._msg_count): messages.append("This is message" + str(i)) producer.send(messages) consumer = MessageConsumer(consumer_id='msys', consumer_group='connn', \ message_types=[TestMessage._msg_type], auto_ack=True, \ offset='latest', message_bus=message_bus_3) count = 0 while True: try: message = consumer.receive() if isinstance(message, bytes): count += 1 consumer.ack() except Exception: self.assertEqual(count, TestMessage._msg_count) break
def test_delete_messages(self): """ Test Delete """ producer = MessageProducer(producer_id="p1", \ message_type=TestMessage._msg_type, method="sync") producer.delete() unread_count = producer.get_unread_count \ (consumer_group=TestMessage._consumer_group) self.assertEqual(unread_count, 0) self.assertFalse(unread_count > 0)
def test_018_multiple_admins(self): """Test multiple instances of admin interface.""" message_types_list = TestMessageBus._admin.list_message_types() message_types_list.remove(TestMessageBus._message_type) message_types_list.remove('__consumer_offsets') if message_types_list: for message_type in message_types_list: producer = MessageProducer(producer_id=message_type, \ message_type=message_type, method='sync') producer.delete()
def create_MsgProducer_obj(self): self._producer = None try: self._producer = MessageProducer(producer_id=self._producer_id, message_type=self._message_type, method=self._method) except Exception as err: logger.error( 'Instance creation for MessageProducer class failed due to %s' % err)
def __init__(self): self._request_shutdown = False self._msg_sent_succesfull = True # Configure messaging Exchange to transmit messages self._connection = None self._read_config() self._producer = MessageProducer( producer_id=self._producer_id, message_type=self._message_type, method=self._method, )
def setup_producer(self, prod_id, msg_type, method): """Setup producer.""" try: self._producer = MessageProducer(producer_id=prod_id, message_type=msg_type, method=method) except Exception as exception: msg = ("msg_bus setup producer except:%s %s") % ( exception, traceback.format_exc()) return False, msg return True, None
def setup_producer(self, prod_id, msg_type, method): """Setup producer.""" if not self._message_bus: raise Exception("Non Existent Message Bus") try: self._producer = MessageProducer(self._message_bus, \ producer_id=prod_id, message_type=msg_type, method=method) except Exception as exception: msg = ("msg_bus setup producer except:%s %s") % ( exception, traceback.format_exc()) return False, msg return True, None
def test_send(self): """ Test Send Message """ messages = [] producer = MessageProducer(TestMessage.message_bus, \ producer_id='p1', message_type='big') self.assertIsNotNone(producer, "Producer not found") for i in range(0, 100): messages.append("This is message" + str(i)) self.assertEqual(len(messages), 100) self.assertIsInstance(messages, list) producer.send(messages)
def test_send(self): """ Test Send Message. """ messages = [] message_bus = MessageBus() producer = MessageProducer(message_bus, producer_id='sel', \ message_type='Sel', method='async') self.assertIsNotNone(producer, "Producer not found") for i in range(0, 10): messages.append("This is message" + str(i)) self.assertIsInstance(messages, list) producer.send(messages)
async def message_bus_rest(request): if request.method == 'POST': try: message_type = request.match_info['message_type'] payload = await request.json() messages = payload['messages'] producer = MessageProducer(producer_id='rest_producer', \ message_type=message_type, method='sync') producer.send(messages) except MessageBusError as e: status_code = e.rc error_message = e.desc response_obj = {'error_code': status_code, 'exception': ['MessageBusError', {'message' : error_message}]} except Exception as e: exception_key = type(e).__name__ exception = RestServerError(exception_key).http_error() status_code = exception[0] error_message = exception[1] response_obj = {'error_code': status_code, 'exception': [exception_key, {'message' : error_message}]} raise MessageBusError(status_code, error_message) from e else: status_code = 200 # No exception, Success response_obj = {'status_code': status_code, 'status': 'success'} finally: return web.Response(text=json.dumps(response_obj) , status=status_code) if request.method == 'GET': try: message_types = str(request.match_info['message_type']).split('&') consumer_group = request.rel_url.query['consumer_group'] consumer = MessageConsumer(consumer_id='rest_consumer', \ consumer_group=consumer_group, message_types=message_types, \ auto_ack=True, offset='latest') message = consumer.receive() except MessageBusError as e: status_code = e.rc error_message = e.desc response_obj = {'error_code': status_code, 'exception': ['MessageBusError', {'message' : error_message}]} except Exception as e: exception_key = type(e).__name__ exception = RestServerError(exception_key).http_error() status_code = exception[0] error_message = exception[1] response_obj = {'error_code': status_code, 'exception': [exception_key, {'message' : error_message}]} raise MessageBusError(status_code, error_message) from e else: status_code = 200 # No exception, Success response_obj = {'messages': str(message)} finally: return web.Response(text=json.dumps(response_obj), status=status_code)
def __init__(self, msg_topic): self.topic = msg_topic admin = MessageBusAdmin("ha_admin") # Register the topic only once. try: admin.register_message_type([self.topic], 1) print(f'Message topic={self.topic} registered on Message Bus.') except Exception: print(f'Message topic={self.topic} is already registered!!') self.producer = MessageProducer(producer_id="system_health", message_type=self.topic, method="sync") print(f'Producer created for topic={self.topic}')
def __init__(self, producer_id: str, message_type: str, partitions: int): """ Register message types with message bus. Args: producer_id (str): producer id. message_types (str): Message type. partitions (int, optional): No. of partitions. Defaults to 1. Raises: MessageBusError: Message bus error. """ self.producer = MessageProducer( producer_id=producer_id, message_type=message_type, method=MessageBusProducer.PRODUCER_METHOD)
def initialize(self, conf_reader, msgQlist, products): """initialize configuration reader and internal msg queues""" # Initialize ScheduledMonitorThread super(EgressAccumulatedMsgsProcessor, self).initialize(conf_reader) super(EgressAccumulatedMsgsProcessor, self).initialize_msgQ(msgQlist) self.store_queue = StoreQueue() self._read_config() producer_initialized.wait() self._producer = MessageProducer(producer_id="acuumulated processor", message_type=self._message_type, method=self._method)
def test_producer_unread_count(self): """ Test unread message count from producer side """ producer = MessageProducer(producer_id="p1", \ message_type=TestMessage._msg_type, method="sync") messages = [] for i in range(0, TestMessage._msg_count): messages.append("This is message" + str(i)) self.assertEqual(len(messages), TestMessage._msg_count) self.assertIsInstance(messages, list) producer.send(messages) unread_count = producer.get_unread_count( consumer_group=TestMessage._consumer_group) self.assertEqual(unread_count, TestMessage._msg_count) self.assertFalse(unread_count != 100)
def init(cls, component: str, source: str, cluster_id: str, \ message_server_endpoints: str, **message_server_kwargs): """ Set the Event Message context Parameters: component Component that generates the IEM. For e.g. 'S3', 'SSPL' source Single character that indicates the type of component. For e.g. H-Hardware, S-Software, F-Firmware, O-OS """ cls._component = component cls._source = source cls._site_id = 1 cls._rack_id = 1 cls._node_id = Conf.machine_id cls._cluster_id = cluster_id if cls._component is None: Log.error("Invalid component type: %s" % cls._component ) raise EventMessageError(errno.EINVAL, "Invalid component type: %s", \ cls._component) if cls._source not in cls._SOURCE.keys(): Log.error("Invalid source type: %s" % cls._source) raise EventMessageError(errno.EINVAL, "Invalid source type: %s", \ cls._source) MessageBus.init(message_server_endpoints, **message_server_kwargs) cls._producer = MessageProducer(producer_id='event_producer', \ message_type='IEM', method='sync') Log.info("IEM Producer initialized for component %s and source %s" % \ (cls._component, cls._source))
def init(cls, component: str, source: str): """ Set the Event Message context Parameters: component Component that generates the IEM. For e.g. 'S3', 'SSPL' source Single character that indicates the type of component. For e.g. H-Hardware, S-Software, F-Firmware, O-OS """ cls._component = component cls._source = source try: Conf.load('cluster', cls._conf_file, skip_reload=True) ids = Conf.get('cluster', 'server_node') cls._site_id = ids['site_id'] cls._rack_id = ids['rack_id'] cls._node_id = ids['node_id'] cls._cluster_id = ids['cluster_id'] except Exception as e: raise EventMessageError(errno.EINVAL, "Invalid config in %s. %s", \ cls._conf_file, e) if cls._component is None: raise EventMessageError(errno.EINVAL, "Invalid component type: %s", \ cls._component) if cls._source not in cls._SOURCE.keys(): raise EventMessageError(errno.EINVAL, "Invalid source type: %s", \ cls._source) cls._producer = MessageProducer(producer_id='event_producer', \ message_type='IEM', method='sync')
def test_002_unknown_message_type(self): """Test invalid message type.""" with self.assertRaises(MessageBusError): MessageProducer(producer_id='send', message_type='', method='sync') with self.assertRaises(MessageBusError): MessageConsumer(consumer_id='receive', consumer_group='test', \ message_types=[''], auto_ack=False, offset='earliest')
def initialize(self, conf_reader, msgQlist, product): """initialize configuration reader and internal msg queues""" # Initialize ScheduledMonitorThread super(EgressProcessor, self).initialize(conf_reader) # Initialize internal message queues for this module super(EgressProcessor, self).initialize_msgQ(msgQlist) self.store_queue = StoreQueue() # Flag denoting that a shutdown message has been placed # into our message queue from the main sspl_ll_d handler self._request_shutdown = False self._read_config() self._producer = MessageProducer(producer_id=self._producer_id, message_type=self._message_type, method=self._method) producer_initialized.set()
def setUpClass(cls): """Register the test message_type.""" cls._admin.register_message_type(message_types= \ [TestMessageBus._message_type], partitions=1) cls._producer = MessageProducer(producer_id='send', \ message_type=TestMessageBus._message_type, method='sync') cls._consumer = MessageConsumer(consumer_id='receive', \ consumer_group='test', message_types=[TestMessageBus._message_type], \ auto_ack=False, offset='earliest')
def setUpClass(cls): """Register the test message_type.""" cls._admin.register_message_type(message_types= \ [TestKVPayloadMessage._message_type], partitions=1) cls._consumer = MessageConsumer(consumer_id='kv_consumer', consumer_group='kv', message_types=[TestKVPayloadMessage.\ _message_type], auto_ack=True, offset='earliest') cls._producer = MessageProducer(producer_id='kv_producer', \ message_type=TestKVPayloadMessage._message_type, method='sync')
class MessageBusProducer: PRODUCER_METHOD = "sync" def __init__(self, producer_id: str, message_type: str, partitions: int): """ Register message types with message bus. Args: producer_id (str): producer id. message_types (str): Message type. partitions (int, optional): No. of partitions. Defaults to 1. Raises: MessageBusError: Message bus error. """ self.producer = MessageProducer( producer_id=producer_id, message_type=message_type, method=MessageBusProducer.PRODUCER_METHOD) def publish(self, message: any): """ Produce message to message bus. Args: message (any): Message. If msg is dict it will be dumped as json. If msg is string then it will be send directly. If message is list, it should have all string element, all items will be published. """ if isinstance(message, dict): self.producer.send([json.dumps(message)]) elif isinstance(message, str): self.producer.send([message]) elif isinstance(message, list): self.producer.send(message) else: raise Exception(f"Invalid type of message {message}")
async def send(request): Log.debug(f"Received POST request for message type " \ f"{request.match_info['message_type']}. Processing message") try: message_type = request.match_info['message_type'] payload = await request.json() messages = payload['messages'] producer = MessageProducer(producer_id='rest_producer', \ message_type=message_type, method='sync') producer.send(messages) except MessageBusError as e: status_code = e.rc error_message = e.desc Log.error(f"Unable to send message for message_type: " \ f"{message_type}, status code: {status_code}," \ f" error: {error_message}") response_obj = {'error_code': status_code, 'exception': \ ['MessageBusError', {'message': error_message}]} except Exception as e: exception_key = type(e).__name__ exception = RestServerError(exception_key).http_error() status_code = exception[0] error_message = exception[1] Log.error(f"Internal error while sending messages for " \ f"message_type: {message_type}, status code: " \ f"{status_code}, error: {error_message}") response_obj = {'error_code': status_code, 'exception': \ [exception_key, {'message': error_message}]} raise MessageBusError(status_code, error_message) from e else: status_code = 200 # No exception, Success Log.debug(f"Sending messages for message_type " \ f"{message_type} using POST method finished with status " \ f"code: {status_code}") response_obj = {'status_code': status_code, 'status': 'success'} finally: return web.Response(text=json.dumps(response_obj), \ status=status_code)
def init(cls, component: str, source: str): """ Set the Event Message context Parameters: component Component that generates the IEM. For e.g. 'S3', 'SSPL' source Single character that indicates the type of component. For e.g. H-Hardware, S-Software, F-Firmware, O-OS """ cls._component = component cls._source = source Conf.load('config_file', 'json:///etc/cortx/cortx.conf', skip_reload=True) # if Log.logger is already initialized by some parent process # the same file will be used to log all the messagebus related # logs, else standard iem.log will be used. if not Log.logger: LOG_DIR='/var/log' iem_log_dir = os.path.join(LOG_DIR, 'cortx/utils/iem') log_level = Conf.get('config_file', 'utils>log_level', 'INFO') Log.init('iem', iem_log_dir, level=log_level, \ backup_count=5, file_size_in_mb=5) try: Conf.load('cluster', cls._conf_file, skip_reload=True) ids = Conf.get('cluster', 'server_node') cls._site_id = ids['site_id'] cls._rack_id = ids['rack_id'] cls._node_id = ids['node_id'] cls._cluster_id = ids['cluster_id'] except Exception as e: Log.error("Invalid config in %s." % cls._conf_file) raise EventMessageError(errno.EINVAL, "Invalid config in %s. %s", \ cls._conf_file, e) if cls._component is None: Log.error("Invalid component type: %s" % cls._component ) raise EventMessageError(errno.EINVAL, "Invalid component type: %s", \ cls._component) if cls._source not in cls._SOURCE.keys(): Log.error("Invalid source type: %s" % cls._source) raise EventMessageError(errno.EINVAL, "Invalid source type: %s", \ cls._source) cls._producer = MessageProducer(producer_id='event_producer', \ message_type='IEM', method='sync') Log.info("IEM Producer initialized for component %s and source %s" % \ (cls._component, cls._source))
def initialize(self, conf_reader, msgQlist, product): """initialize configuration reader and internal msg queues""" # Initialize ScheduledMonitorThread super(EgressProcessorTests, self).initialize(conf_reader) # Initialize internal message queues for this module super(EgressProcessorTests, self).initialize_msgQ(msgQlist) # Flag denoting that a shutdown message has been placed # into our message queue from the main sspl_ll_d handler self._request_shutdown = False self._msg_sent_succesfull = True self._product = product # Configure messaging Exchange to transmit messages self._connection = None self._read_config() self._producer = MessageProducer(producer_id=self._producer_id, message_type=self._message_type, method=self._method) producer_initialized.set()
class EventProducer: def __init__(self, msg_topic): self.topic = msg_topic admin = MessageBusAdmin("ha_admin") # Register the topic only once. try: admin.register_message_type([self.topic], 1) print(f'Message topic={self.topic} registered on Message Bus.') except Exception: print(f'Message topic={self.topic} is already registered!!') self.producer = MessageProducer(producer_id="system_health", message_type=self.topic, method="sync") print(f'Producer created for topic={self.topic}') def send_event(self, event): print('Sending event...') if isinstance(event, myEvent): event_to_send = event.get_json_msg() else: event_to_send = json.dumps(event) self.producer.send([event_to_send]) print(f'Sent event={event} to message bus.')
def setUpClass(cls, \ cluster_conf_path: str = 'yaml:///etc/cortx/cluster.conf'): """Register the test message_type.""" if TestKVPayloadMessage._cluster_conf_path: cls.cluster_conf_path = TestKVPayloadMessage._cluster_conf_path else: cls.cluster_conf_path = cluster_conf_path Conf.load('config', cls.cluster_conf_path, skip_reload=True) message_server_endpoints = Conf.get('config',\ 'cortx>external>kafka>endpoints') Log.init('message_bus', '/var/log', level='INFO', \ backup_count=5, file_size_in_mb=5) MessageBus.init(message_server_endpoints=message_server_endpoints) cls._admin = MessageBusAdmin(admin_id='register') cls._admin.register_message_type(message_types= \ [TestKVPayloadMessage._message_type], partitions=1) cls._consumer = MessageConsumer(consumer_id='kv_consumer', \ consumer_group='kv', message_types=[TestKVPayloadMessage.\ _message_type], auto_ack=True, offset='earliest') cls._producer = MessageProducer(producer_id='kv_producer', \ message_type=TestKVPayloadMessage._message_type, method='sync')
def init(cls, component: str, source: str, receiver: bool = False): """ Set the Event Message context """ cls._component = component cls._source = source cls() if cls._component is None: raise EventMessageError(errno.EINVAL, "Invalid component type: %s", \ cls._component) if cls._source not in cls._SOURCE.keys(): raise EventMessageError(errno.EINVAL, "Invalid source type: %s", \ cls._source) if receiver: cls._client = MessageConsumer(consumer_id='event_consumer', \ consumer_group=cls._component, message_types=['IEM'], \ auto_ack=True, offset='earliest') else: cls._client = MessageProducer(producer_id='event_producer', \ message_type='IEM', method='sync')
class S3CortxMsgBus: def __init__(self): """Init.""" self._producer = None self._consumer = None def setup_producer(self, prod_id, msg_type, method): """Setup producer.""" try: self._producer = MessageProducer(producer_id=prod_id, message_type=msg_type, method=method) except Exception as exception: msg = ("msg_bus setup producer except:%s %s") % ( exception, traceback.format_exc()) return False, msg return True, None def send(self, messages): """Send the constructed message.""" try: self._producer.send(messages) except Exception as exception: msg = ("msg_bus send except:%s %s") % ( exception, traceback.format_exc()) return False, msg return True, None def purge(self): """Purge/Delete all the messages.""" self._producer.delete() def setup_consumer(self, cons_id, group, msg_type, auto_ack, offset): """Setup the consumer.""" try: self._consumer = MessageConsumer(consumer_id=cons_id, \ consumer_group=group, message_types=[msg_type], auto_ack=auto_ack, offset=offset) except Exception as exception: msg = ("msg_bus setup_consumer except:%s %s") % ( exception, traceback.format_exc()) return False, msg return True, None def receive(self, daemon_mode): """Receive the incoming message.""" try: if daemon_mode: #timeout=0 makes it as blocking indefinitely message = self._consumer.receive(timeout=0) else: #timeout = 0.5 sec, by default, which is non-blocking message = self._consumer.receive() except Exception as exception: msg = ("msg_bus receive except:%s %s") % ( exception, traceback.format_exc()) return False, msg return True, message def ack(self): """Ack the received message.""" try: self._consumer.ack() except Exception as exception: msg = ("msg_bus ack except:%s %s") % ( exception, traceback.format_exc()) return False, msg return True, None def count(self, consumer_group): """Get the count of unread messages.""" unread_count = 0 try: unread_count = self._producer.get_unread_count(consumer_group) except: return 0 return unread_count @staticmethod def create_topic(admin_id: str, message_types: list, partitions: int): """create topic.""" mbadmin = MessageBusAdmin(admin_id = admin_id) try: mbadmin.register_message_type(message_types = message_types, partitions = partitions) except Exception as e: if "TOPIC_ALREADY_EXISTS" not in str(e): raise(e) @staticmethod def add_concurrency(admin_id: str, message_type: str, concurrency_count: int): """Increase partition count for given topic.""" mbadmin = MessageBusAdmin(admin_id = admin_id) mbadmin.add_concurrency(message_type = message_type, concurrency_count = concurrency_count) @staticmethod def delete_topic(admin_id: str, message_types: list): """Delete given topic""" mbadmin = MessageBusAdmin(admin_id = admin_id) mbadmin.deregister_message_type(message_types = message_types) @staticmethod def list_topics(admin_id: str): """list all available topics""" mbadmin = MessageBusAdmin(admin_id = admin_id) return mbadmin.list_message_types() @staticmethod def is_topic_exist(admin_id: str, topic_name: str): """retuns true if topic exist else false""" mbadmin = MessageBusAdmin(admin_id = admin_id) if topic_name in mbadmin.list_message_types(): return True return False