def test_multiple_instance(self):
""" Test Clients with no instance """
message_bus_1 = MessageBus()
message_bus_2 = MessageBus()
message_bus_3 = MessageBus()
admin = MessageBusAdmin(admin_id='admin', message_bus=message_bus_1)
self.assertTrue(TestMessage._msg_type in admin.list_message_types())
messages = []
producer = MessageProducer(producer_id='p1', \
message_type=TestMessage._msg_type, method='sync', \
message_bus=message_bus_2)
self.assertIsNotNone(producer, "Producer not found")
for i in range(0, TestMessage._msg_count):
messages.append("This is message" + str(i))
producer.send(messages)
consumer = MessageConsumer(consumer_id='msys', consumer_group='connn', \
message_types=[TestMessage._msg_type], auto_ack=True, \
offset='latest', message_bus=message_bus_3)
count = 0
while True:
try:
message = consumer.receive()
if isinstance(message, bytes):
count += 1
consumer.ack()
except Exception:
self.assertEqual(count, TestMessage._msg_count)
break
def reset(config_path: str):
"""Remove/Delete all the data/logs that was created by user/testing."""
from cortx.utils.message_bus.error import MessageBusError
try:
# use gconf to deregister IEM
from cortx.utils.message_bus import MessageBusAdmin, MessageBus
from cortx.utils.message_bus import MessageProducer
Conf.load(GCONF_INDEX, config_path, skip_reload=True)
message_bus_backend = Conf.get('config', MSG_BUS_BACKEND_KEY)
message_server_endpoints = Conf.get('config',\
f'{EXTERNAL_KEY}>{message_bus_backend}>endpoints')
MessageBus.init(message_server_endpoints)
mb = MessageBusAdmin(admin_id='reset')
message_types_list = mb.list_message_types()
if message_types_list:
for message_type in message_types_list:
producer = MessageProducer(producer_id=message_type,\
message_type=message_type, method='sync')
producer.delete()
except MessageBusError as e:
raise SetupError(e.rc, "Can not reset Message Bus. %s", e)
except Exception as e:
raise SetupError(
errors.ERR_OP_FAILED, "Internal error, can not \
reset Message Bus. %s", e)
return 0
def init(cls, component: str, source: str, cluster_id: str, \
message_server_endpoints: str, **message_server_kwargs):
"""
Set the Event Message context
Parameters:
component Component that generates the IEM. For e.g. 'S3', 'SSPL'
source Single character that indicates the type of component.
For e.g. H-Hardware, S-Software, F-Firmware, O-OS
"""
cls._component = component
cls._source = source
cls._site_id = 1
cls._rack_id = 1
cls._node_id = Conf.machine_id
cls._cluster_id = cluster_id
if cls._component is None:
Log.error("Invalid component type: %s" % cls._component )
raise EventMessageError(errno.EINVAL, "Invalid component type: %s", \
cls._component)
if cls._source not in cls._SOURCE.keys():
Log.error("Invalid source type: %s" % cls._source)
raise EventMessageError(errno.EINVAL, "Invalid source type: %s", \
cls._source)
MessageBus.init(message_server_endpoints, **message_server_kwargs)
cls._producer = MessageProducer(producer_id='event_producer', \
message_type='IEM', method='sync')
Log.info("IEM Producer initialized for component %s and source %s" % \
(cls._component, cls._source))
def cleanup(pre_factory: bool, config_path: str):
"""Remove/Delete all the data that was created after post install."""
# delete message_types
from cortx.utils.message_bus.error import MessageBusError
try:
# use gconf to clean and delete all message type in messagebus
from cortx.utils.message_bus import MessageBus, MessageBusAdmin
Conf.load(GCONF_INDEX, config_path, skip_reload=True)
message_bus_backend = Conf.get(GCONF_INDEX, MSG_BUS_BACKEND_KEY)
message_server_endpoints = Conf.get(GCONF_INDEX,\
f'{EXTERNAL_KEY}>{message_bus_backend}>endpoints')
MessageBus.init(message_server_endpoints)
mb = MessageBusAdmin(admin_id='cleanup')
message_types_list = mb.list_message_types()
if message_types_list:
mb.deregister_message_type(message_types_list)
except MessageBusError as e:
raise SetupError(e.rc, "Can not cleanup Message Bus. %s", e)
except Exception as e:
raise SetupError(
errors.ERR_OP_FAILED, "Can not cleanup Message \
Bus. %s", e)
return 0
def test_same_instance(self):
""" Test Singleton functionality """
message_bus_1 = MessageBus()
message_bus_2 = MessageBus()
# Check for same instance
self.assertEqual(message_bus_1, message_bus_2)
def init():
"""
Initialize utils MessageBus Library with kafka endpoints once per service. In future utils will throw error if
init done multiple times. If any new service will come which uses MessageBus then init should be done there.
"""
message_server_endpoints = Conf.get(
const.HA_GLOBAL_INDEX, f"kafka_config{const._DELIM}endpoints")
utils_message_bus.init(message_server_endpoints)
def __init__(self, client_type: str, **client_conf: dict):
self._message_bus = MessageBus() if 'message_bus' not in \
client_conf.keys() or client_conf['message_bus'] is None else \
client_conf['message_bus']
self._message_bus.init_client(client_type, **client_conf)
self._client_conf = client_conf
Log.debug(f"MessageBusClient: initialized with arguments" \
f" client_type: {client_type}, kwargs: {client_conf}")
def deregister_message_type(self, message_types: list):
"""
Deregisters a list of message types.
Parameters:
message_types This is essentially equivalent to the list of queue
topic name. For e.g. ["Alert"]
"""
client_id = self._get_conf('client_id')
MessageBus.deregister_message_type(client_id, message_types)
def register_message_type(self, message_types: list, partitions: int):
"""
Registers a list of message types.
Parameters:
message_types This is essentially equivalent to the list of queue
topic name. For e.g. ["Alert"]
partitions Integer that represents number of partitions to be
created.
"""
client_id = self._get_conf('client_id')
MessageBus.register_message_type(client_id, message_types, partitions)
def add_concurrency(self, message_type: str, concurrency_count: int):
"""
To achieve concurrency for a message type.
Parameters:
message_type This is essentially equivalent to queue/topic name.
For e.g. "Alert"
concurrency_count Integer to achieve concurrency among consumers
"""
client_id = self._get_conf('client_id')
MessageBus.add_concurrency(client_id, message_type,\
concurrency_count)
def send(self, messages: list):
"""
Sends list of messages to the Message Bus.
Parameters:
messages A list of messages sent to Message Bus
"""
message_type = self._get_conf('message_type')
method = self._get_conf('method')
client_id = self._get_conf('client_id')
messages = self._get_str_message_list(messages)
MessageBus.send(client_id, message_type, method, messages)
def setUpClass(cls,\
cluster_conf_path: str = 'yaml:///etc/cortx/cluster.conf'):
"""Register the test message_type."""
if TestMessage._cluster_conf_path:
cls.cluster_conf_path = TestMessage._cluster_conf_path
else:
cls.cluster_conf_path = cluster_conf_path
Conf.load('config', cls.cluster_conf_path, skip_reload=True)
message_server_endpoints = Conf.get('config',\
'cortx>external>kafka>endpoints')
Log.init('message_bus', '/var/log', level='INFO', \
backup_count=5, file_size_in_mb=5)
MessageBus.init(message_server_endpoints=message_server_endpoints)
cls._admin = MessageBusAdmin(admin_id='register')
cls._admin.register_message_type(message_types= \
[TestMessage._message_type], partitions=1)
class TestMessage(unittest.TestCase):
""" Test MessageBus related functionality """
message_bus = MessageBus()
def test_send(self):
""" Test Send Message """
messages = []
producer = MessageProducer(TestMessage.message_bus, \
producer_id='p1', message_type='big')
self.assertIsNotNone(producer, "Producer not found")
for i in range(0, 100):
messages.append("This is message" + str(i))
self.assertEqual(len(messages), 100)
self.assertIsInstance(messages, list)
producer.send(messages)
def test_count(self):
""" Test Unread Message Count """
consumer = MessageConsumer(TestMessage.message_bus, \
consumer_id='sensors', consumer_group='test_group', \
message_types=['big'], auto_ack=True, offset='latest')
self.assertIsNotNone(consumer, "Consumer not found")
unread_count = consumer.get_unread_count()
self.assertEqual(unread_count, 100)
class TestMessage(unittest.TestCase):
"""Test MessageBus related functionality."""
message_bus = MessageBus()
def test_send(self):
"""Test Send Message."""
messages = []
producer = MessageProducer(TestMessage.message_bus, \
producer_id='sspl_sensor', message_type='Alert')
self.assertIsNotNone(producer, "Producer not found")
for i in range(0, 1000):
messages.append("This is message" + str(i))
self.assertEqual(len(messages), 1000)
self.assertIsInstance(messages, list)
producer.send(messages)
def test_receive(self):
"""Test Receive Message."""
consumer = MessageConsumer(TestMessage.message_bus, \
consumer_id='sspl_sensors', consumer_group='sspl', \
message_types=['Alert'], auto_ack=False, offset='latest')
self.assertIsNotNone(consumer, "Consumer not found")
count = 0
while True:
try:
message = consumer.receive()
count += 1
self.assertIsNotNone(message, "Message not found")
consumer.ack()
except Exception as e:
self.assertEqual(count, 1000)
break
def validate(self):
"""Check for below requirement.
1. Validate input configs
2. Validate sspl conf is created
3. Check if message bus is accessible
"""
if os.geteuid() != 0:
msg = "Run this command with root privileges."
logger.error(msg)
raise SetupError(errno.EINVAL, msg)
# Validate input/provisioner configs
machine_id = Utility.get_machine_id()
Utility.get_config_value(consts.PRVSNR_CONFIG_INDEX,
"server_node>%s>cluster_id" % machine_id)
self.node_type = Utility.get_config_value(
consts.PRVSNR_CONFIG_INDEX, "server_node>%s>type" % machine_id)
enclosure_id = Utility.get_config_value(
consts.PRVSNR_CONFIG_INDEX,
"server_node>%s>storage>enclosure_id" % machine_id)
self.enclosure_type = Utility.get_config_value(
consts.PRVSNR_CONFIG_INDEX,
"storage_enclosure>%s>type" % enclosure_id)
# Validate sspl conf is created
if not os.path.isfile(consts.file_store_config_path):
msg = "Missing configuration - %s !! Create and rerun." % (
consts.file_store_config_path)
logger.error(msg)
raise SetupError(errno.EINVAL, msg)
# Validate message bus is accessible
self.mb = MessageBus()
def subscribe(cls, component: str, message_server_endpoints: str, \
**message_server_kwargs):
"""
Subscribe to IEM alerts
Parameters:
component Component that generates the IEM. For e.g. 'S3', 'SSPL'
"""
if component is None:
Log.error("Invalid component type: %s" % component)
raise EventMessageError(errno.EINVAL, "Invalid component type: %s", \
component)
MessageBus.init(message_server_endpoints, **message_server_kwargs)
cls._consumer = MessageConsumer(consumer_id='event_consumer', \
consumer_group=component, message_types=['IEM'], \
auto_ack=True, offset='earliest')
Log.info("IEM Consumer initialized for component: %s" % component)
def set_message_type_expire(self, message_type: str, **kwargs):
"""Set expiration time for given message type."""
client_id = self._get_conf('client_id')
status = MessageBus.set_message_type_expire(client_id, message_type,\
**kwargs)
Log.info(f"Successfully updated {message_type} with new"+\
" configuration.")
return status
def connect(self):
"""Connect to Message Bus."""
try:
self._message_bus = MessageBus()
except Exception as exception:
msg = ("msg_bus init except:%s %s") % (exception,
traceback.format_exc())
return False, msg
return True, None
def receive(self, timeout: float = None) -> list:
"""
Receives messages from the Message Bus.
Parameters:
timeout Time in seconds to wait for the message.
"""
client_id = self._get_conf('client_id')
return MessageBus.receive(client_id, timeout)
def setUpClass(cls, \
cluster_conf_path: str = 'yaml:///etc/cortx/cluster.conf'):
"""Register the test message_type."""
if TestKVPayloadMessage._cluster_conf_path:
cls.cluster_conf_path = TestKVPayloadMessage._cluster_conf_path
else:
cls.cluster_conf_path = cluster_conf_path
Conf.load('config', cls.cluster_conf_path, skip_reload=True)
message_server_endpoints = Conf.get('config',\
'cortx>external>kafka>endpoints')
Log.init('message_bus', '/var/log', level='INFO', \
backup_count=5, file_size_in_mb=5)
MessageBus.init(message_server_endpoints=message_server_endpoints)
cls._admin = MessageBusAdmin(admin_id='register')
cls._admin.register_message_type(message_types= \
[TestKVPayloadMessage._message_type], partitions=1)
cls._consumer = MessageConsumer(consumer_id='kv_consumer', \
consumer_group='kv', message_types=[TestKVPayloadMessage.\
_message_type], auto_ack=True, offset='earliest')
cls._producer = MessageProducer(producer_id='kv_producer', \
message_type=TestKVPayloadMessage._message_type, method='sync')
def init(config_path: str):
"""Perform initialization."""
# Create message_type for Event Message
from cortx.utils.message_bus import MessageBus, MessageBusAdmin
from cortx.utils.message_bus.error import MessageBusError
try:
# Read the config values
# use gconf to create IEM topic
Conf.load(GCONF_INDEX, config_path, skip_reload=True)
message_bus_backend = Conf.get(GCONF_INDEX, MSG_BUS_BACKEND_KEY)
message_server_endpoints = Conf.get(GCONF_INDEX,\
f'{EXTERNAL_KEY}>{message_bus_backend}>endpoints')
MessageBus.init(message_server_endpoints)
admin = MessageBusAdmin(admin_id='register')
admin.register_message_type(message_types=['IEM',\
'audit_messages'], partitions=1)
except MessageBusError as e:
if 'TOPIC_ALREADY_EXISTS' not in e.desc:
raise SetupError(e.rc, "Unable to create message_type. %s", e)
return 0
def __init__(self, message_server_endpoints, message_server_port=28300,\
cluster_id=None):
super().__init__()
MessageBus.init(message_server_endpoints=message_server_endpoints)
from cortx.utils.iem_framework import IemRequestHandler
from cortx.utils.message_bus import MessageBusRequestHandler
from cortx.utils.audit_log import AuditLogRequestHandler
IemRequestHandler.cluster_id = cluster_id
IemRequestHandler.message_server_endpoints = message_server_endpoints
self.app.add_routes([web.post('/EventMessage/event', IemRequestHandler.send), \
web.get('/EventMessage/event', IemRequestHandler.receive), \
web.post('/MessageBus/message/{message_type}', \
MessageBusRequestHandler.send), \
web.get('/MessageBus/message/{message_type}', \
MessageBusRequestHandler.receive),
web.post('/AuditLog/message/', \
AuditLogRequestHandler.send),
web.post('/AuditLog/webhook/', \
AuditLogRequestHandler.send_webhook_info)
])
super().run_app(web, message_server_port)
def test_send(self):
""" Test Send Message. """
messages = []
message_bus = MessageBus()
producer = MessageProducer(message_bus, producer_id='sel', \
message_type='Sel', method='async')
self.assertIsNotNone(producer, "Producer not found")
for i in range(0, 10):
messages.append("This is message" + str(i))
self.assertIsInstance(messages, list)
producer.send(messages)
class TestMessage(unittest.TestCase):
""" Test MessageBus related functionality """
message_bus = MessageBus()
_message_type = 'test_topic'
_partition = 1
def test_list_message_type(self):
""" Test list message type API """
admin = MessageBusAdmin(TestMessage.message_bus, admin_id='admin')
admin.register_message_type(message_types=[TestMessage._message_type], \
partitions=TestMessage._partition)
message_type_list = admin.list_message_types()
self.assertTrue(TestMessage._message_type in message_type_list)
def test_receive(self):
""" Test Receive Message. """
message_bus = MessageBus()
consumer = MessageConsumer(message_bus, consumer_id='sel', \
consumer_group='sel', message_type=['Sel'], auto_ack=False, \
offset='latest')
self.assertIsNotNone(consumer, "Consumer not found")
while True:
try:
message = consumer.receive()
self.assertIsNotNone(message, "Message not found")
consumer.ack()
except Exception as e:
break
def __init__(self):
from cortx.utils.message_bus import MessageBus
from cortx.utils.conf_store import Conf
Conf.load("index", "json:///etc/cortx/message_bus.conf")
self._server = Conf.get("index", 'message_broker>cluster[0]>server')
self._msg_bus = MessageBus()
# Create a test topic
cmd = "/opt/kafka/bin/kafka-topics.sh --create" + \
" --topic mytest --bootstrap-server " + self._server + ":9092"
try:
cmd_proc = SimpleProcess(cmd)
res_op, res_err, res_rc = cmd_proc.run()
if res_rc != 0:
raise SetupError(errno.EINVAL, "Unable to test the config")
except Exception as e:
raise SetupError(errno.EINVAL, \
"Unable to test the config, %s", e)
class TestMessage(unittest.TestCase):
""" Test MessageBus related functionality. """
message_bus = MessageBus()
def test_consumer_one(self):
""" Test Receive Message for consumer group 1 """
consumer = MessageConsumer(TestMessage.message_bus, \
consumer_id='sspl_sensor', consumer_group='c1', \
message_type=['test_type'], auto_ack=False, offset='latest')
self.assertIsNotNone(consumer, "Consumer not found")
count = 0
while True:
try:
message = consumer.receive()
count += 1
self.assertIsNotNone(message, "Message not found")
consumer.ack()
except Exception as e:
self.assertEqual(count, 10)
break
def test_consumer_two(self):
""" Test Receive Message for consumer group 2 """
consumer = MessageConsumer(TestMessage.message_bus, \
consumer_id='sspl_sensor1', consumer_group='c3', \
message_type=['test_type'], auto_ack=False, offset='latest')
self.assertIsNotNone(consumer, "Consumer not found")
count = 0
while True:
try:
message = consumer.receive()
count += 1
self.assertIsNotNone(message, "Message not found")
consumer.ack()
except Exception as e:
self.assertEqual(count, 10)
break
def __init__(self,
event_list: List[SubscribeEvent],
group_id: str = COMPONENT_ID):
"""
event_list - list of events to subscribe to.
group_id - the group_id to pass to KafkaConsumer.
"""
logging.debug('Inside EventListener')
self.event_manager = EventManager.get_instance()
topic = self._subscribe(event_list)
if topic is None:
raise RuntimeError('Failed to subscribe to events')
message_bus = MessageBus()
self.consumer = MessageConsumer(
message_bus=message_bus,
consumer_id=COMPONENT_ID,
consumer_group=group_id,
message_types=[topic],
# Why is it 'str' in cortx-py-utils??
auto_ack=str(False),
offset='earliest')
def create_message_types(self):
# Skip this step if sspl is being configured for node
# replacement scenario as rabbitmq cluster is
# already configured on the healthy node
# Configure rabbitmq
if not os.path.exists(consts.REPLACEMENT_NODE_ENV_VAR_FILE):
message_types = [
Conf.get(consts.SSPL_CONFIG_INDEX, "RABBITMQINGRESSPROCESSOR"
">message_type"),
Conf.get(consts.SSPL_CONFIG_INDEX, "RABBITMQEGRESSPROCESSOR"
">message_type")
]
mb = MessageBus()
mbadmin = MessageBusAdmin(mb, admin_id='admin')
try:
mbadmin.register_message_type(message_types=message_types,
partitions=1)
except MessageBusError as e:
if self.topic_already_exists in e.desc:
# Topic already exists
pass
else:
raise e
def test_send_invalid_message(self):
"""Send invalid message format."""
message = MessageBus()
with self.assertRaises(MessageBusError):
TestKVPayloadMessage._producer.send([message])
