def reset(config_path: str):
"""Remove/Delete all the data/logs that was created by user/testing."""
from cortx.utils.message_bus.error import MessageBusError
try:
# use gconf to deregister IEM
from cortx.utils.message_bus import MessageBusAdmin, MessageBus
from cortx.utils.message_bus import MessageProducer
Conf.load(GCONF_INDEX, config_path, skip_reload=True)
message_bus_backend = Conf.get('config', MSG_BUS_BACKEND_KEY)
message_server_endpoints = Conf.get('config',\
f'{EXTERNAL_KEY}>{message_bus_backend}>endpoints')
MessageBus.init(message_server_endpoints)
mb = MessageBusAdmin(admin_id='reset')
message_types_list = mb.list_message_types()
if message_types_list:
for message_type in message_types_list:
producer = MessageProducer(producer_id=message_type,\
message_type=message_type, method='sync')
producer.delete()
except MessageBusError as e:
raise SetupError(e.rc, "Can not reset Message Bus. %s", e)
except Exception as e:
raise SetupError(
errors.ERR_OP_FAILED, "Internal error, can not \
reset Message Bus. %s", e)
return 0
def send_msg(self, message):
""" Sends a message """
from cortx.utils.message_bus import MessageProducer
producer = MessageProducer(producer_id='setup', \
message_type='mytest', method='sync')
producer.send(message)
time.sleep(1)
def test_multiple_instance(self):
""" Test Clients with no instance """
message_bus_1 = MessageBus()
message_bus_2 = MessageBus()
message_bus_3 = MessageBus()
admin = MessageBusAdmin(admin_id='admin', message_bus=message_bus_1)
self.assertTrue(TestMessage._msg_type in admin.list_message_types())
messages = []
producer = MessageProducer(producer_id='p1', \
message_type=TestMessage._msg_type, method='sync', \
message_bus=message_bus_2)
self.assertIsNotNone(producer, "Producer not found")
for i in range(0, TestMessage._msg_count):
messages.append("This is message" + str(i))
producer.send(messages)
consumer = MessageConsumer(consumer_id='msys', consumer_group='connn', \
message_types=[TestMessage._msg_type], auto_ack=True, \
offset='latest', message_bus=message_bus_3)
count = 0
while True:
try:
message = consumer.receive()
if isinstance(message, bytes):
count += 1
consumer.ack()
except Exception:
self.assertEqual(count, TestMessage._msg_count)
break
def test_delete_messages(self):
""" Test Delete """
producer = MessageProducer(producer_id="p1", \
message_type=TestMessage._msg_type, method="sync")
producer.delete()
unread_count = producer.get_unread_count \
(consumer_group=TestMessage._consumer_group)
self.assertEqual(unread_count, 0)
self.assertFalse(unread_count > 0)
def test_018_multiple_admins(self):
"""Test multiple instances of admin interface."""
message_types_list = TestMessageBus._admin.list_message_types()
message_types_list.remove(TestMessageBus._message_type)
message_types_list.remove('__consumer_offsets')
if message_types_list:
for message_type in message_types_list:
producer = MessageProducer(producer_id=message_type, \
message_type=message_type, method='sync')
producer.delete()
def create_MsgProducer_obj(self):
self._producer = None
try:
self._producer = MessageProducer(producer_id=self._producer_id,
message_type=self._message_type,
method=self._method)
except Exception as err:
logger.error(
'Instance creation for MessageProducer class failed due to %s'
% err)
def __init__(self):
self._request_shutdown = False
self._msg_sent_succesfull = True
# Configure messaging Exchange to transmit messages
self._connection = None
self._read_config()
self._producer = MessageProducer(
producer_id=self._producer_id,
message_type=self._message_type,
method=self._method,
)
def setup_producer(self, prod_id, msg_type, method):
"""Setup producer."""
try:
self._producer = MessageProducer(producer_id=prod_id,
message_type=msg_type,
method=method)
except Exception as exception:
msg = ("msg_bus setup producer except:%s %s") % (
exception, traceback.format_exc())
return False, msg
return True, None
def setup_producer(self, prod_id, msg_type, method):
"""Setup producer."""
if not self._message_bus:
raise Exception("Non Existent Message Bus")
try:
self._producer = MessageProducer(self._message_bus, \
producer_id=prod_id, message_type=msg_type, method=method)
except Exception as exception:
msg = ("msg_bus setup producer except:%s %s") % (
exception, traceback.format_exc())
return False, msg
return True, None
def test_send(self):
""" Test Send Message """
messages = []
producer = MessageProducer(TestMessage.message_bus, \
producer_id='p1', message_type='big')
self.assertIsNotNone(producer, "Producer not found")
for i in range(0, 100):
messages.append("This is message" + str(i))
self.assertEqual(len(messages), 100)
self.assertIsInstance(messages, list)
producer.send(messages)
def test_send(self):
""" Test Send Message. """
messages = []
message_bus = MessageBus()
producer = MessageProducer(message_bus, producer_id='sel', \
message_type='Sel', method='async')
self.assertIsNotNone(producer, "Producer not found")
for i in range(0, 10):
messages.append("This is message" + str(i))
self.assertIsInstance(messages, list)
producer.send(messages)
async def message_bus_rest(request):
if request.method == 'POST':
try:
message_type = request.match_info['message_type']
payload = await request.json()
messages = payload['messages']
producer = MessageProducer(producer_id='rest_producer', \
message_type=message_type, method='sync')
producer.send(messages)
except MessageBusError as e:
status_code = e.rc
error_message = e.desc
response_obj = {'error_code': status_code, 'exception': ['MessageBusError', {'message' : error_message}]}
except Exception as e:
exception_key = type(e).__name__
exception = RestServerError(exception_key).http_error()
status_code = exception[0]
error_message = exception[1]
response_obj = {'error_code': status_code, 'exception': [exception_key, {'message' : error_message}]}
raise MessageBusError(status_code, error_message) from e
else:
status_code = 200 # No exception, Success
response_obj = {'status_code': status_code, 'status': 'success'}
finally:
return web.Response(text=json.dumps(response_obj) , status=status_code)
if request.method == 'GET':
try:
message_types = str(request.match_info['message_type']).split('&')
consumer_group = request.rel_url.query['consumer_group']
consumer = MessageConsumer(consumer_id='rest_consumer', \
consumer_group=consumer_group, message_types=message_types, \
auto_ack=True, offset='latest')
message = consumer.receive()
except MessageBusError as e:
status_code = e.rc
error_message = e.desc
response_obj = {'error_code': status_code, 'exception': ['MessageBusError', {'message' : error_message}]}
except Exception as e:
exception_key = type(e).__name__
exception = RestServerError(exception_key).http_error()
status_code = exception[0]
error_message = exception[1]
response_obj = {'error_code': status_code, 'exception': [exception_key, {'message' : error_message}]}
raise MessageBusError(status_code, error_message) from e
else:
status_code = 200 # No exception, Success
response_obj = {'messages': str(message)}
finally:
return web.Response(text=json.dumps(response_obj), status=status_code)
def __init__(self, msg_topic):
self.topic = msg_topic
admin = MessageBusAdmin("ha_admin")
# Register the topic only once.
try:
admin.register_message_type([self.topic], 1)
print(f'Message topic={self.topic} registered on Message Bus.')
except Exception:
print(f'Message topic={self.topic} is already registered!!')
self.producer = MessageProducer(producer_id="system_health",
message_type=self.topic,
method="sync")
print(f'Producer created for topic={self.topic}')
def __init__(self, producer_id: str, message_type: str, partitions: int):
"""
Register message types with message bus.
Args:
producer_id (str): producer id.
message_types (str): Message type.
partitions (int, optional): No. of partitions. Defaults to 1.
Raises:
MessageBusError: Message bus error.
"""
self.producer = MessageProducer(
producer_id=producer_id,
message_type=message_type,
method=MessageBusProducer.PRODUCER_METHOD)
def initialize(self, conf_reader, msgQlist, products):
"""initialize configuration reader and internal msg queues"""
# Initialize ScheduledMonitorThread
super(EgressAccumulatedMsgsProcessor, self).initialize(conf_reader)
super(EgressAccumulatedMsgsProcessor, self).initialize_msgQ(msgQlist)
self.store_queue = StoreQueue()
self._read_config()
producer_initialized.wait()
self._producer = MessageProducer(producer_id="acuumulated processor",
message_type=self._message_type,
method=self._method)
def test_producer_unread_count(self):
""" Test unread message count from producer side """
producer = MessageProducer(producer_id="p1", \
message_type=TestMessage._msg_type, method="sync")
messages = []
for i in range(0, TestMessage._msg_count):
messages.append("This is message" + str(i))
self.assertEqual(len(messages), TestMessage._msg_count)
self.assertIsInstance(messages, list)
producer.send(messages)
unread_count = producer.get_unread_count(
consumer_group=TestMessage._consumer_group)
self.assertEqual(unread_count, TestMessage._msg_count)
self.assertFalse(unread_count != 100)
def init(cls, component: str, source: str, cluster_id: str, \
message_server_endpoints: str, **message_server_kwargs):
"""
Set the Event Message context
Parameters:
component Component that generates the IEM. For e.g. 'S3', 'SSPL'
source Single character that indicates the type of component.
For e.g. H-Hardware, S-Software, F-Firmware, O-OS
"""
cls._component = component
cls._source = source
cls._site_id = 1
cls._rack_id = 1
cls._node_id = Conf.machine_id
cls._cluster_id = cluster_id
if cls._component is None:
Log.error("Invalid component type: %s" % cls._component )
raise EventMessageError(errno.EINVAL, "Invalid component type: %s", \
cls._component)
if cls._source not in cls._SOURCE.keys():
Log.error("Invalid source type: %s" % cls._source)
raise EventMessageError(errno.EINVAL, "Invalid source type: %s", \
cls._source)
MessageBus.init(message_server_endpoints, **message_server_kwargs)
cls._producer = MessageProducer(producer_id='event_producer', \
message_type='IEM', method='sync')
Log.info("IEM Producer initialized for component %s and source %s" % \
(cls._component, cls._source))
def init(cls, component: str, source: str):
"""
Set the Event Message context
Parameters:
component Component that generates the IEM. For e.g. 'S3', 'SSPL'
source Single character that indicates the type of component.
For e.g. H-Hardware, S-Software, F-Firmware, O-OS
"""
cls._component = component
cls._source = source
try:
Conf.load('cluster', cls._conf_file, skip_reload=True)
ids = Conf.get('cluster', 'server_node')
cls._site_id = ids['site_id']
cls._rack_id = ids['rack_id']
cls._node_id = ids['node_id']
cls._cluster_id = ids['cluster_id']
except Exception as e:
raise EventMessageError(errno.EINVAL, "Invalid config in %s. %s", \
cls._conf_file, e)
if cls._component is None:
raise EventMessageError(errno.EINVAL, "Invalid component type: %s", \
cls._component)
if cls._source not in cls._SOURCE.keys():
raise EventMessageError(errno.EINVAL, "Invalid source type: %s", \
cls._source)
cls._producer = MessageProducer(producer_id='event_producer', \
message_type='IEM', method='sync')
def test_002_unknown_message_type(self):
"""Test invalid message type."""
with self.assertRaises(MessageBusError):
MessageProducer(producer_id='send', message_type='', method='sync')
with self.assertRaises(MessageBusError):
MessageConsumer(consumer_id='receive', consumer_group='test', \
message_types=[''], auto_ack=False, offset='earliest')
def initialize(self, conf_reader, msgQlist, product):
"""initialize configuration reader and internal msg queues"""
# Initialize ScheduledMonitorThread
super(EgressProcessor, self).initialize(conf_reader)
# Initialize internal message queues for this module
super(EgressProcessor, self).initialize_msgQ(msgQlist)
self.store_queue = StoreQueue()
# Flag denoting that a shutdown message has been placed
# into our message queue from the main sspl_ll_d handler
self._request_shutdown = False
self._read_config()
self._producer = MessageProducer(producer_id=self._producer_id,
message_type=self._message_type,
method=self._method)
producer_initialized.set()
def setUpClass(cls):
"""Register the test message_type."""
cls._admin.register_message_type(message_types= \
[TestMessageBus._message_type], partitions=1)
cls._producer = MessageProducer(producer_id='send', \
message_type=TestMessageBus._message_type, method='sync')
cls._consumer = MessageConsumer(consumer_id='receive', \
consumer_group='test', message_types=[TestMessageBus._message_type], \
auto_ack=False, offset='earliest')
def setUpClass(cls):
"""Register the test message_type."""
cls._admin.register_message_type(message_types= \
[TestKVPayloadMessage._message_type], partitions=1)
cls._consumer = MessageConsumer(consumer_id='kv_consumer',
consumer_group='kv', message_types=[TestKVPayloadMessage.\
_message_type], auto_ack=True, offset='earliest')
cls._producer = MessageProducer(producer_id='kv_producer', \
message_type=TestKVPayloadMessage._message_type, method='sync')
class MessageBusProducer:
PRODUCER_METHOD = "sync"
def __init__(self, producer_id: str, message_type: str, partitions: int):
"""
Register message types with message bus.
Args:
producer_id (str): producer id.
message_types (str): Message type.
partitions (int, optional): No. of partitions. Defaults to 1.
Raises:
MessageBusError: Message bus error.
"""
self.producer = MessageProducer(
producer_id=producer_id,
message_type=message_type,
method=MessageBusProducer.PRODUCER_METHOD)
def publish(self, message: any):
"""
Produce message to message bus.
Args:
message (any): Message.
If msg is dict it will be dumped as json.
If msg is string then it will be send directly.
If message is list, it should have all string element, all items will be published.
"""
if isinstance(message, dict):
self.producer.send([json.dumps(message)])
elif isinstance(message, str):
self.producer.send([message])
elif isinstance(message, list):
self.producer.send(message)
else:
raise Exception(f"Invalid type of message {message}")
async def send(request):
Log.debug(f"Received POST request for message type " \
f"{request.match_info['message_type']}. Processing message")
try:
message_type = request.match_info['message_type']
payload = await request.json()
messages = payload['messages']
producer = MessageProducer(producer_id='rest_producer', \
message_type=message_type, method='sync')
producer.send(messages)
except MessageBusError as e:
status_code = e.rc
error_message = e.desc
Log.error(f"Unable to send message for message_type: " \
f"{message_type}, status code: {status_code}," \
f" error: {error_message}")
response_obj = {'error_code': status_code, 'exception': \
['MessageBusError', {'message': error_message}]}
except Exception as e:
exception_key = type(e).__name__
exception = RestServerError(exception_key).http_error()
status_code = exception[0]
error_message = exception[1]
Log.error(f"Internal error while sending messages for " \
f"message_type: {message_type}, status code: " \
f"{status_code}, error: {error_message}")
response_obj = {'error_code': status_code, 'exception': \
[exception_key, {'message': error_message}]}
raise MessageBusError(status_code, error_message) from e
else:
status_code = 200 # No exception, Success
Log.debug(f"Sending messages for message_type " \
f"{message_type} using POST method finished with status " \
f"code: {status_code}")
response_obj = {'status_code': status_code, 'status': 'success'}
finally:
return web.Response(text=json.dumps(response_obj), \
status=status_code)
def init(cls, component: str, source: str):
"""
Set the Event Message context
Parameters:
component Component that generates the IEM. For e.g. 'S3', 'SSPL'
source Single character that indicates the type of component.
For e.g. H-Hardware, S-Software, F-Firmware, O-OS
"""
cls._component = component
cls._source = source
Conf.load('config_file', 'json:///etc/cortx/cortx.conf',
skip_reload=True)
# if Log.logger is already initialized by some parent process
# the same file will be used to log all the messagebus related
# logs, else standard iem.log will be used.
if not Log.logger:
LOG_DIR='/var/log'
iem_log_dir = os.path.join(LOG_DIR, 'cortx/utils/iem')
log_level = Conf.get('config_file', 'utils>log_level', 'INFO')
Log.init('iem', iem_log_dir, level=log_level, \
backup_count=5, file_size_in_mb=5)
try:
Conf.load('cluster', cls._conf_file, skip_reload=True)
ids = Conf.get('cluster', 'server_node')
cls._site_id = ids['site_id']
cls._rack_id = ids['rack_id']
cls._node_id = ids['node_id']
cls._cluster_id = ids['cluster_id']
except Exception as e:
Log.error("Invalid config in %s." % cls._conf_file)
raise EventMessageError(errno.EINVAL, "Invalid config in %s. %s", \
cls._conf_file, e)
if cls._component is None:
Log.error("Invalid component type: %s" % cls._component )
raise EventMessageError(errno.EINVAL, "Invalid component type: %s", \
cls._component)
if cls._source not in cls._SOURCE.keys():
Log.error("Invalid source type: %s" % cls._source)
raise EventMessageError(errno.EINVAL, "Invalid source type: %s", \
cls._source)
cls._producer = MessageProducer(producer_id='event_producer', \
message_type='IEM', method='sync')
Log.info("IEM Producer initialized for component %s and source %s" % \
(cls._component, cls._source))
def initialize(self, conf_reader, msgQlist, product):
"""initialize configuration reader and internal msg queues"""
# Initialize ScheduledMonitorThread
super(EgressProcessorTests, self).initialize(conf_reader)
# Initialize internal message queues for this module
super(EgressProcessorTests, self).initialize_msgQ(msgQlist)
# Flag denoting that a shutdown message has been placed
# into our message queue from the main sspl_ll_d handler
self._request_shutdown = False
self._msg_sent_succesfull = True
self._product = product
# Configure messaging Exchange to transmit messages
self._connection = None
self._read_config()
self._producer = MessageProducer(producer_id=self._producer_id,
message_type=self._message_type,
method=self._method)
producer_initialized.set()
class EventProducer:
def __init__(self, msg_topic):
self.topic = msg_topic
admin = MessageBusAdmin("ha_admin")
# Register the topic only once.
try:
admin.register_message_type([self.topic], 1)
print(f'Message topic={self.topic} registered on Message Bus.')
except Exception:
print(f'Message topic={self.topic} is already registered!!')
self.producer = MessageProducer(producer_id="system_health",
message_type=self.topic,
method="sync")
print(f'Producer created for topic={self.topic}')
def send_event(self, event):
print('Sending event...')
if isinstance(event, myEvent):
event_to_send = event.get_json_msg()
else:
event_to_send = json.dumps(event)
self.producer.send([event_to_send])
print(f'Sent event={event} to message bus.')
def setUpClass(cls, \
cluster_conf_path: str = 'yaml:///etc/cortx/cluster.conf'):
"""Register the test message_type."""
if TestKVPayloadMessage._cluster_conf_path:
cls.cluster_conf_path = TestKVPayloadMessage._cluster_conf_path
else:
cls.cluster_conf_path = cluster_conf_path
Conf.load('config', cls.cluster_conf_path, skip_reload=True)
message_server_endpoints = Conf.get('config',\
'cortx>external>kafka>endpoints')
Log.init('message_bus', '/var/log', level='INFO', \
backup_count=5, file_size_in_mb=5)
MessageBus.init(message_server_endpoints=message_server_endpoints)
cls._admin = MessageBusAdmin(admin_id='register')
cls._admin.register_message_type(message_types= \
[TestKVPayloadMessage._message_type], partitions=1)
cls._consumer = MessageConsumer(consumer_id='kv_consumer', \
consumer_group='kv', message_types=[TestKVPayloadMessage.\
_message_type], auto_ack=True, offset='earliest')
cls._producer = MessageProducer(producer_id='kv_producer', \
message_type=TestKVPayloadMessage._message_type, method='sync')
def init(cls, component: str, source: str, receiver: bool = False):
""" Set the Event Message context """
cls._component = component
cls._source = source
cls()
if cls._component is None:
raise EventMessageError(errno.EINVAL, "Invalid component type: %s", \
cls._component)
if cls._source not in cls._SOURCE.keys():
raise EventMessageError(errno.EINVAL, "Invalid source type: %s", \
cls._source)
if receiver:
cls._client = MessageConsumer(consumer_id='event_consumer', \
consumer_group=cls._component, message_types=['IEM'], \
auto_ack=True, offset='earliest')
else:
cls._client = MessageProducer(producer_id='event_producer', \
message_type='IEM', method='sync')
class S3CortxMsgBus:
def __init__(self):
"""Init."""
self._producer = None
self._consumer = None
def setup_producer(self, prod_id, msg_type, method):
"""Setup producer."""
try:
self._producer = MessageProducer(producer_id=prod_id,
message_type=msg_type,
method=method)
except Exception as exception:
msg = ("msg_bus setup producer except:%s %s") % (
exception, traceback.format_exc())
return False, msg
return True, None
def send(self, messages):
"""Send the constructed message."""
try:
self._producer.send(messages)
except Exception as exception:
msg = ("msg_bus send except:%s %s") % (
exception, traceback.format_exc())
return False, msg
return True, None
def purge(self):
"""Purge/Delete all the messages."""
self._producer.delete()
def setup_consumer(self, cons_id, group, msg_type, auto_ack, offset):
"""Setup the consumer."""
try:
self._consumer = MessageConsumer(consumer_id=cons_id, \
consumer_group=group, message_types=[msg_type], auto_ack=auto_ack, offset=offset)
except Exception as exception:
msg = ("msg_bus setup_consumer except:%s %s") % (
exception, traceback.format_exc())
return False, msg
return True, None
def receive(self, daemon_mode):
"""Receive the incoming message."""
try:
if daemon_mode:
#timeout=0 makes it as blocking indefinitely
message = self._consumer.receive(timeout=0)
else:
#timeout = 0.5 sec, by default, which is non-blocking
message = self._consumer.receive()
except Exception as exception:
msg = ("msg_bus receive except:%s %s") % (
exception, traceback.format_exc())
return False, msg
return True, message
def ack(self):
"""Ack the received message."""
try:
self._consumer.ack()
except Exception as exception:
msg = ("msg_bus ack except:%s %s") % (
exception, traceback.format_exc())
return False, msg
return True, None
def count(self, consumer_group):
"""Get the count of unread messages."""
unread_count = 0
try:
unread_count = self._producer.get_unread_count(consumer_group)
except:
return 0
return unread_count
@staticmethod
def create_topic(admin_id: str, message_types: list, partitions: int):
"""create topic."""
mbadmin = MessageBusAdmin(admin_id = admin_id)
try:
mbadmin.register_message_type(message_types = message_types,
partitions = partitions)
except Exception as e:
if "TOPIC_ALREADY_EXISTS" not in str(e):
raise(e)
@staticmethod
def add_concurrency(admin_id: str, message_type: str, concurrency_count: int):
"""Increase partition count for given topic."""
mbadmin = MessageBusAdmin(admin_id = admin_id)
mbadmin.add_concurrency(message_type = message_type,
concurrency_count = concurrency_count)
@staticmethod
def delete_topic(admin_id: str, message_types: list):
"""Delete given topic"""
mbadmin = MessageBusAdmin(admin_id = admin_id)
mbadmin.deregister_message_type(message_types = message_types)
@staticmethod
def list_topics(admin_id: str):
"""list all available topics"""
mbadmin = MessageBusAdmin(admin_id = admin_id)
return mbadmin.list_message_types()
@staticmethod
def is_topic_exist(admin_id: str, topic_name: str):
"""retuns true if topic exist else false"""
mbadmin = MessageBusAdmin(admin_id = admin_id)
if topic_name in mbadmin.list_message_types():
return True
return False
