def reset(config_path: str):
"""Remove/Delete all the data/logs that was created by user/testing."""
from cortx.utils.message_bus.error import MessageBusError
try:
# use gconf to deregister IEM
from cortx.utils.message_bus import MessageBusAdmin, MessageBus
from cortx.utils.message_bus import MessageProducer
Conf.load(GCONF_INDEX, config_path, skip_reload=True)
message_bus_backend = Conf.get('config', MSG_BUS_BACKEND_KEY)
message_server_endpoints = Conf.get('config',\
f'{EXTERNAL_KEY}>{message_bus_backend}>endpoints')
MessageBus.init(message_server_endpoints)
mb = MessageBusAdmin(admin_id='reset')
message_types_list = mb.list_message_types()
if message_types_list:
for message_type in message_types_list:
producer = MessageProducer(producer_id=message_type,\
message_type=message_type, method='sync')
producer.delete()
except MessageBusError as e:
raise SetupError(e.rc, "Can not reset Message Bus. %s", e)
except Exception as e:
raise SetupError(
errors.ERR_OP_FAILED, "Internal error, can not \
reset Message Bus. %s", e)
return 0
def test_multiple_instance(self):
""" Test Clients with no instance """
message_bus_1 = MessageBus()
message_bus_2 = MessageBus()
message_bus_3 = MessageBus()
admin = MessageBusAdmin(admin_id='admin', message_bus=message_bus_1)
self.assertTrue(TestMessage._msg_type in admin.list_message_types())
messages = []
producer = MessageProducer(producer_id='p1', \
message_type=TestMessage._msg_type, method='sync', \
message_bus=message_bus_2)
self.assertIsNotNone(producer, "Producer not found")
for i in range(0, TestMessage._msg_count):
messages.append("This is message" + str(i))
producer.send(messages)
consumer = MessageConsumer(consumer_id='msys', consumer_group='connn', \
message_types=[TestMessage._msg_type], auto_ack=True, \
offset='latest', message_bus=message_bus_3)
count = 0
while True:
try:
message = consumer.receive()
if isinstance(message, bytes):
count += 1
consumer.ack()
except Exception:
self.assertEqual(count, TestMessage._msg_count)
break
def cleanup():
"""Remove/Delete all the data that was created after post install."""
conf_file = '/etc/cortx/message_bus.conf'
if os.path.exists(conf_file):
# delete message_types
try:
from cortx.utils.message_bus import MessageBusAdmin
mb = MessageBusAdmin(admin_id='cleanup')
message_types_list = mb.list_message_types()
if message_types_list:
mb.deregister_message_type(message_types_list)
except MessageBusError as e:
raise SetupError(e.rc, "Can not cleanup Message Bus. %s", e)
except Exception as e:
raise SetupError(
errors.ERR_OP_FAILED, "Can not cleanup Message \
Bus. %s", e)
config_files = ['/etc/cortx/message_bus.conf', \
'/etc/cortx/cluster.conf']
for each_file in config_files:
if os.path.exists(each_file):
# delete data/config stored
try:
os.remove(each_file)
except OSError as e:
raise SetupError(
e.errno, "Error deleting config file %s, \
%s", each_file, e)
return 0
def reset():
""" Remove/Delete all the data that was created after post install """
conf_file = '/etc/cortx/message_bus.conf'
if os.path.exists(conf_file):
# delete message_types
from cortx.utils.message_bus import MessageBusAdmin
try:
mb = MessageBusAdmin(admin_id='reset')
message_types_list = mb.list_message_types()
if message_types_list:
mb.deregister_message_type(message_types_list)
except MessageBusError as e:
raise SetupError(e.rc, "Can not reset Message Bus. %s", e)
except Exception as e:
raise SetupError(errors.ERR_OP_FAILED, "Can not reset Message Bus. \
%s", e)
# Stop MessageBus Service
cmd = SimpleProcess("systemctl stop cortx_message_bus")
_, stderr, res_rc = cmd.run()
if res_rc != 0:
raise SetupError(res_rc, "Unable to stop MessageBus Service. \
%s", stderr.decode('utf-8'))
return 0
def reset():
"""Remove/Delete all the data/logs that was created by user/testing."""
try:
from cortx.utils.message_bus import MessageBusAdmin
from cortx.utils.message_bus.message_bus_client import MessageProducer
mb = MessageBusAdmin(admin_id='reset')
message_types_list = mb.list_message_types()
if message_types_list:
for message_type in message_types_list:
producer = MessageProducer(producer_id=message_type, \
message_type=message_type, method='sync')
producer.delete()
except MessageBusError as e:
raise SetupError(e.rc, "Can not reset Message Bus. %s", e)
except Exception as e:
raise SetupError(
errors.ERR_OP_FAILED, "Internal error, can not \
reset Message Bus. %s", e)
# Clear the logs
utils_log_path = '/var/log/cortx/utils/'
if os.path.exists(utils_log_path):
cmd = "find %s -type f -name '*.log' -exec truncate -s 0 {} +" % utils_log_path
cmd_proc = SimpleProcess(cmd)
_, stderr, rc = cmd_proc.run()
if rc != 0:
raise SetupError(errors.ERR_OP_FAILED, \
"Can not reset log files. %s", stderr)
return 0
def is_topic_exist(admin_id: str, topic_name: str):
"""retuns true if topic exist else false"""
if S3CortxMsgBus._message_bus:
mbadmin = MessageBusAdmin(S3CortxMsgBus._message_bus, admin_id)
if topic_name in mbadmin.list_message_types():
return True
return False
def cleanup(pre_factory: bool, config_path: str):
"""Remove/Delete all the data that was created after post install."""
# delete message_types
from cortx.utils.message_bus.error import MessageBusError
try:
# use gconf to clean and delete all message type in messagebus
from cortx.utils.message_bus import MessageBus, MessageBusAdmin
Conf.load(GCONF_INDEX, config_path, skip_reload=True)
message_bus_backend = Conf.get(GCONF_INDEX, MSG_BUS_BACKEND_KEY)
message_server_endpoints = Conf.get(GCONF_INDEX,\
f'{EXTERNAL_KEY}>{message_bus_backend}>endpoints')
MessageBus.init(message_server_endpoints)
mb = MessageBusAdmin(admin_id='cleanup')
message_types_list = mb.list_message_types()
if message_types_list:
mb.deregister_message_type(message_types_list)
except MessageBusError as e:
raise SetupError(e.rc, "Can not cleanup Message Bus. %s", e)
except Exception as e:
raise SetupError(
errors.ERR_OP_FAILED, "Can not cleanup Message \
Bus. %s", e)
return 0
def init():
""" Perform initialization """
# Create message_type for Event Message
from cortx.utils.message_bus import MessageBusAdmin
try:
admin = MessageBusAdmin(admin_id='register')
admin.register_message_type(message_types=['IEM'], partitions=1)
except MessageBusError as e:
raise SetupError(e.rc, "Unable to create message_type. %s", e)
# start MessageBus service and check status
start_cmd = SimpleProcess("systemctl start cortx_message_bus")
_, start_err, start_rc = start_cmd.run()
if start_rc != 0:
raise SetupError(
start_rc, "Unable to start MessageBus Service \
%s", start_err.decode('utf-8'))
status_cmd = SimpleProcess("systemctl status cortx_message_bus")
_, status_err, status_rc = status_cmd.run()
if status_rc != 0:
raise SetupError(
status_rc, "MessageBus Service is either failed \
inactive. %s", status_err.decode('utf-8'))
return 0
def test_list_message_type(self):
""" Test list message type API """
admin = MessageBusAdmin(TestMessage.message_bus, admin_id='admin')
admin.register_message_type(message_types=[TestMessage._message_type], \
partitions=TestMessage._partition)
message_type_list = admin.list_message_types()
self.assertTrue(TestMessage._message_type in message_type_list)
def cleanup(pre_factory: bool):
"""Remove/Delete all the data that was created after post install."""
conf_file = '/etc/cortx/message_bus.conf'
if os.path.exists(conf_file):
# delete message_types
try:
from cortx.utils.message_bus import MessageBusAdmin
mb = MessageBusAdmin(admin_id='cleanup')
message_types_list = mb.list_message_types()
if message_types_list:
mb.deregister_message_type(message_types_list)
except MessageBusError as e:
raise SetupError(e.rc, "Can not cleanup Message Bus. %s", e)
except Exception as e:
raise SetupError(
errors.ERR_OP_FAILED, "Can not cleanup Message \
Bus. %s", e)
config_files = ['/etc/cortx/message_bus.conf', \
'/etc/cortx/cluster.conf']
Utils._delete_files(config_files)
if pre_factory:
# deleting all log files as part of pre-factory cleanup
cortx_utils_log_regex = '/var/log/cortx/utils/**/*.log'
log_files = glob.glob(cortx_utils_log_regex, recursive=True)
Utils._delete_files(log_files)
return 0
def delete_topic(admin_id: str, message_types: list):
"""Delete given topic"""
try:
if S3CortxMsgBus._message_bus:
mbadmin = MessageBusAdmin(S3CortxMsgBus._message_bus, admin_id)
mbadmin.deregister_message_type(message_types=message_types)
except:
raise Exception("Failed to delete topic")
def deregister(message_type: str):
"""
Deregister message type to message bus.
Args:
message_type (str): Message type.
"""
admin = MessageBusAdmin(admin_id=MessageBus.ADMIN_ID)
if message_type in admin.list_message_types():
admin.deregister_message_type(message_types=[message_type])
def create_topic(admin_id: str, message_types: list, partitions: int):
"""create topic."""
try:
if S3CortxMsgBus._message_bus:
mbadmin = MessageBusAdmin(S3CortxMsgBus._message_bus, admin_id)
mbadmin.register_message_type(message_types=message_types,
partitions=partitions)
except:
raise Exception("Failed to create topic")
def create_topic(admin_id: str, message_types: list, partitions: int):
"""create topic."""
mbadmin = MessageBusAdmin(admin_id = admin_id)
try:
mbadmin.register_message_type(message_types = message_types,
partitions = partitions)
except Exception as e:
if "TOPIC_ALREADY_EXISTS" not in str(e):
raise(e)
def increase_partitions(admin_id: str, message_types: list,
partitions: int):
"""Increase partition count for given topic."""
try:
if S3CortxMsgBus._message_bus:
mbadmin = MessageBusAdmin(S3CortxMsgBus._message_bus, admin_id)
mbadmin.increase_parallelism(message_types=message_types,
partitions=partitions)
except:
raise Exception("Failed to increase partition")
def __init__(self, msg_topic):
self.topic = msg_topic
admin = MessageBusAdmin("ha_admin")
# Register the topic only once.
try:
admin.register_message_type([self.topic], 1)
print(f'Message topic={self.topic} registered on Message Bus.')
except Exception:
print(f'Message topic={self.topic} is already registered!!')
self.producer = MessageProducer(producer_id="system_health",
message_type=self.topic,
method="sync")
print(f'Producer created for topic={self.topic}')
def __init__(self):
from cortx.utils.message_bus import MessageBusAdmin
# Create a test message_type
try:
admin = MessageBusAdmin(admin_id='messageadmin')
admin.register_message_type(message_types=['mytest'], partitions=1)
list_message_type = admin.list_message_types()
if 'mytest' not in list_message_type:
raise SetupError(errors.ERR_OP_FAILED, "Failed to test the config." \
"message_type 'mytest' creation failed.")
except Exception as e:
raise SetupError(errors.ERR_OP_FAILED,
"Failed to test the config, %s", e)
def __del__(self):
# Delete the test message_type
from cortx.utils.message_bus import MessageBusAdmin
# deregister_message_type
try:
admin = MessageBusAdmin(admin_id='messageadmin')
admin.deregister_message_type(message_types=['mytest'])
list_message_type = admin.list_message_types()
if 'mytest' in list_message_type:
raise SetupError(errors.ERR_OP_FAILED, "Failed to test the" \
" config. Deregister message_type: mytest failed")
except Exception as e:
raise SetupError(errors.ERR_OP_FAILED, \
"Failed to test the config, %s", e)
def register(message_type: str, partitions: int = 1):
"""
Register message type to message bus.
Args:
message_type (str): Message type.
partitions (int): Number of partition.
"""
admin = MessageBusAdmin(admin_id=MessageBus.ADMIN_ID)
try:
if message_type not in admin.list_message_types():
admin.register_message_type(message_types=[message_type],
partitions=partitions)
except Exception as e:
if "TOPIC_ALREADY_EXISTS" not in str(e):
raise (e)
def setUpClass(cls,\
cluster_conf_path: str = 'yaml:///etc/cortx/cluster.conf'):
"""Register the test message_type."""
if TestMessage._cluster_conf_path:
cls.cluster_conf_path = TestMessage._cluster_conf_path
else:
cls.cluster_conf_path = cluster_conf_path
Conf.load('config', cls.cluster_conf_path, skip_reload=True)
message_server_endpoints = Conf.get('config',\
'cortx>external>kafka>endpoints')
Log.init('message_bus', '/var/log', level='INFO', \
backup_count=5, file_size_in_mb=5)
MessageBus.init(message_server_endpoints=message_server_endpoints)
cls._admin = MessageBusAdmin(admin_id='register')
cls._admin.register_message_type(message_types= \
[TestMessage._message_type], partitions=1)
def reset():
"""Remove/Delete all the data/logs that was created by user/testing."""
import time
_purge_retry = 20
try:
from cortx.utils.message_bus import MessageBusAdmin
from cortx.utils.message_bus.message_bus_client import MessageProducer
mb = MessageBusAdmin(admin_id='reset')
message_types_list = mb.list_message_types()
if message_types_list:
for message_type in message_types_list:
producer = MessageProducer(producer_id=message_type, \
message_type=message_type, method='sync')
for retry_count in range(1, (_purge_retry + 2)):
if retry_count > _purge_retry:
Log.error(f"MessageBusError: {errors.ERR_OP_FAILED} " \
f" Unable to delete messages for message type" \
f" {message_type} after {retry_count} retries")
raise MessageBusError(errors.ERR_OP_FAILED,\
"Unable to delete messages for message type" + \
"%s after %d retries", message_type, \
retry_count)
rc = producer.delete()
if rc == 0:
break
time.sleep(2 * retry_count)
except MessageBusError as e:
raise SetupError(e.rc, "Can not reset Message Bus. %s", e)
except Exception as e:
raise SetupError(
errors.ERR_OP_FAILED, "Internal error, can not \
reset Message Bus. %s", e)
# Clear the logs
utils_log_path = '/var/log/cortx/utils/'
if os.path.exists(utils_log_path):
cmd = "find %s -type f -name '*.log' -exec truncate -s 0 {} +" % utils_log_path
cmd_proc = SimpleProcess(cmd)
_, stderr, rc = cmd_proc.run()
if rc != 0:
raise SetupError(errors.ERR_OP_FAILED, \
"Can not reset log files. %s", stderr)
return 0
def create_message_types(self):
# Skip this step if sspl is being configured for node
# replacement scenario as cluster is
# already configured on the healthy node
# Configure message bus
if not os.path.exists(consts.REPLACEMENT_NODE_ENV_VAR_FILE):
message_types = [Conf.get(consts.SSPL_CONFIG_INDEX,
"INGRESSPROCESSOR"
">message_type"),
Conf.get(consts.SSPL_CONFIG_INDEX,
"EGRESSPROCESSOR"
">message_type")]
mbadmin = MessageBusAdmin(admin_id="admin")
try:
mbadmin.register_message_type(message_types=message_types,
partitions=1)
except MessageBusError as e:
if self.topic_already_exists not in e.desc:
# if topic not already exists, raise exception
raise e
def init(config_path: str):
"""Perform initialization."""
# Create message_type for Event Message
from cortx.utils.message_bus import MessageBus, MessageBusAdmin
from cortx.utils.message_bus.error import MessageBusError
try:
# Read the config values
# use gconf to create IEM topic
Conf.load(GCONF_INDEX, config_path, skip_reload=True)
message_bus_backend = Conf.get(GCONF_INDEX, MSG_BUS_BACKEND_KEY)
message_server_endpoints = Conf.get(GCONF_INDEX,\
f'{EXTERNAL_KEY}>{message_bus_backend}>endpoints')
MessageBus.init(message_server_endpoints)
admin = MessageBusAdmin(admin_id='register')
admin.register_message_type(message_types=['IEM',\
'audit_messages'], partitions=1)
except MessageBusError as e:
if 'TOPIC_ALREADY_EXISTS' not in e.desc:
raise SetupError(e.rc, "Unable to create message_type. %s", e)
return 0
def setUpClass(cls, \
cluster_conf_path: str = 'yaml:///etc/cortx/cluster.conf'):
"""Register the test message_type."""
if TestKVPayloadMessage._cluster_conf_path:
cls.cluster_conf_path = TestKVPayloadMessage._cluster_conf_path
else:
cls.cluster_conf_path = cluster_conf_path
Conf.load('config', cls.cluster_conf_path, skip_reload=True)
message_server_endpoints = Conf.get('config',\
'cortx>external>kafka>endpoints')
Log.init('message_bus', '/var/log', level='INFO', \
backup_count=5, file_size_in_mb=5)
MessageBus.init(message_server_endpoints=message_server_endpoints)
cls._admin = MessageBusAdmin(admin_id='register')
cls._admin.register_message_type(message_types= \
[TestKVPayloadMessage._message_type], partitions=1)
cls._consumer = MessageConsumer(consumer_id='kv_consumer', \
consumer_group='kv', message_types=[TestKVPayloadMessage.\
_message_type], auto_ack=True, offset='earliest')
cls._producer = MessageProducer(producer_id='kv_producer', \
message_type=TestKVPayloadMessage._message_type, method='sync')
def create_message_types(self):
# Skip this step if sspl is being configured for node
# replacement scenario as rabbitmq cluster is
# already configured on the healthy node
# Configure rabbitmq
if not os.path.exists(consts.REPLACEMENT_NODE_ENV_VAR_FILE):
message_types = [
Conf.get(consts.SSPL_CONFIG_INDEX, "RABBITMQINGRESSPROCESSOR"
">message_type"),
Conf.get(consts.SSPL_CONFIG_INDEX, "RABBITMQEGRESSPROCESSOR"
">message_type")
]
mb = MessageBus()
mbadmin = MessageBusAdmin(mb, admin_id='admin')
try:
mbadmin.register_message_type(message_types=message_types,
partitions=1)
except MessageBusError as e:
if self.topic_already_exists in e.desc:
# Topic already exists
pass
else:
raise e
class TestMessage(unittest.TestCase):
"""Test MessageBus rest server functionality."""
_base_url = 'http://127.0.0.1:28300/MessageBus/message/'
_admin = MessageBusAdmin(admin_id='register')
_message_type = 'test'
_consumer_group = 'receive'
@classmethod
def setUpClass(cls):
"""Register the test message_type."""
cls._admin.register_message_type(message_types= \
[TestMessage._message_type], partitions=1)
def test_post(self):
"""Test send message."""
url = self._base_url + self._message_type
data = json.dumps({'messages': ['hello', 'how are you']})
headers = {'content-type': 'application/json'}
response = requests.post(url=url, data=data, headers=headers)
self.assertEqual(response.json()['status'], 'success')
self.assertEqual(response.status_code, 200)
def test_get(self):
"""Test receive message."""
response = requests.get(self._base_url + self._message_type +
'?consumer_group=' + self._consumer_group)
self.assertEqual(response.status_code, 200)
@classmethod
def tearDownClass(cls):
"""Deregister the test message_type."""
cls._admin.deregister_message_type(message_types= \
[TestMessage._message_type])
message_type_list = TestMessage._admin.list_message_types()
cls.assertTrue(cls, TestMessage._message_type not in \
message_type_list)
def add_concurrency(admin_id: str, message_type: str, concurrency_count: int):
"""Increase partition count for given topic."""
mbadmin = MessageBusAdmin(admin_id = admin_id)
mbadmin.add_concurrency(message_type = message_type,
concurrency_count = concurrency_count)
def delete_topic(admin_id: str, message_types: list):
"""Delete given topic"""
mbadmin = MessageBusAdmin(admin_id = admin_id)
mbadmin.deregister_message_type(message_types = message_types)
def is_topic_exist(admin_id: str, topic_name: str):
"""retuns true if topic exist else false"""
mbadmin = MessageBusAdmin(admin_id = admin_id)
if topic_name in mbadmin.list_message_types():
return True
return False
def list_topics(admin_id: str):
"""list all available topics"""
mbadmin = MessageBusAdmin(admin_id = admin_id)
return mbadmin.list_message_types()