def __init__(self, msg_topic):
self.consumer = MessageConsumer(consumer_id='hare',
consumer_group='test_consumer',
message_types=[msg_topic],
auto_ack=str(True),
offset='earliest')
print(f'Consumer created for reading topic={msg_topic}')
def test_multiple_instance(self):
""" Test Clients with no instance """
message_bus_1 = MessageBus()
message_bus_2 = MessageBus()
message_bus_3 = MessageBus()
admin = MessageBusAdmin(admin_id='admin', message_bus=message_bus_1)
self.assertTrue(TestMessage._msg_type in admin.list_message_types())
messages = []
producer = MessageProducer(producer_id='p1', \
message_type=TestMessage._msg_type, method='sync', \
message_bus=message_bus_2)
self.assertIsNotNone(producer, "Producer not found")
for i in range(0, TestMessage._msg_count):
messages.append("This is message" + str(i))
producer.send(messages)
consumer = MessageConsumer(consumer_id='msys', consumer_group='connn', \
message_types=[TestMessage._msg_type], auto_ack=True, \
offset='latest', message_bus=message_bus_3)
count = 0
while True:
try:
message = consumer.receive()
if isinstance(message, bytes):
count += 1
consumer.ack()
except Exception:
self.assertEqual(count, TestMessage._msg_count)
break
class TestIngressProcessor:
"""Handles incoming messages via messaging for automated tests."""
MODULE_NAME = "IngressProcessorTests"
PRIORITY = 1
# Section and keys in configuration file
PROCESSOR = MODULE_NAME.upper()
CONSUMER_ID = "consumer_id"
CONSUMER_GROUP = "consumer_group"
MESSAGE_TYPE = "message_type"
OFFSET = "offset"
SYSTEM_INFORMATION_KEY = "SYSTEM_INFORMATION"
JSON_ACTUATOR_SCHEMA = "SSPL-LL_Actuator_Response.json"
JSON_SENSOR_SCHEMA = "SSPL-LL_Sensor_Response.json"
def __init__(self):
self._read_config()
self._consumer = MessageConsumer(
consumer_id=self._consumer_id,
consumer_group=self._consumer_group,
message_types=[self._message_type],
auto_ack=True,
offset=self._offset,
)
def message_reader(self):
logger.info("Started reading messages")
try:
while True:
message = self._consumer.receive()
if message:
logger.info(
f"IngressProcessorTests, Message Received: {message}")
self._consumer.ack()
yield json.loads(message)["message"]
else:
time.sleep(0.2)
except Exception as e:
logger.error("IngressProcessorTests, Exception: %s" % str(e))
logger.error(traceback.format_exc())
def _read_config(self):
"""Configure the messaging exchange with defaults available."""
# Make methods locally available
self._node_id = Conf.get(SSPL_TEST_CONF, NODE_ID_KEY, "SN01")
self._consumer_id = Conf.get(SSPL_TEST_CONF,
f"{self.PROCESSOR}>{self.CONSUMER_ID}",
"sspl_actuator")
self._consumer_group = Conf.get(
SSPL_TEST_CONF, f"{self.PROCESSOR}>{self.CONSUMER_GROUP}",
"cortx_monitor")
self._message_type = Conf.get(SSPL_TEST_CONF,
f"{self.PROCESSOR}>{self.MESSAGE_TYPE}",
"Requests")
self._offset = Conf.get(SSPL_TEST_CONF,
f"{self.PROCESSOR}>{self.OFFSET}", "earliest")
def create_MsgConsumer_obj(self):
self._consumer = None
try:
self._consumer = MessageConsumer(consumer_id=self._consumer_id,
consumer_group=self._consumer_group,
message_types=[self._message_type],
auto_ack=False, offset=self._offset)
except Exception as err:
logger.error('Instance creation for MessageConsumer class failed due to %s' % err)
def test_count(self):
""" Test Unread Message Count """
consumer = MessageConsumer(TestMessage.message_bus, \
consumer_id='sensors', consumer_group='test_group', \
message_types=['big'], auto_ack=True, offset='latest')
self.assertIsNotNone(consumer, "Consumer not found")
unread_count = consumer.get_unread_count()
self.assertEqual(unread_count, 100)
def __init__(self):
self._read_config()
self._consumer = MessageConsumer(
consumer_id=self._consumer_id,
consumer_group=self._consumer_group,
message_types=[self._message_type],
auto_ack=True,
offset=self._offset,
)
def setup_consumer(self, cons_id, group, msg_type, auto_ack, offset):
"""Setup the consumer."""
try:
self._consumer = MessageConsumer(consumer_id=cons_id, \
consumer_group=group, message_types=[msg_type], auto_ack=auto_ack, offset=offset)
except Exception as exception:
msg = ("msg_bus setup_consumer except:%s %s") % (
exception, traceback.format_exc())
return False, msg
return True, None
def setup_consumer(self, cons_id, group, msg_type, auto_ack, offset):
"""Setup the consumer."""
if not S3CortxMsgBus._message_bus:
raise Exception("Non Existent Message Bus")
try:
self._consumer = MessageConsumer(S3CortxMsgBus._message_bus, consumer_id=cons_id, \
consumer_group=group, message_types=[msg_type], auto_ack=auto_ack, offset=offset)
except Exception as exception:
msg = ("msg_bus setup_consumer except:%s %s") % (
exception, traceback.format_exc())
return False, msg
return True, None
async def message_bus_rest(request):
if request.method == 'POST':
try:
message_type = request.match_info['message_type']
payload = await request.json()
messages = payload['messages']
producer = MessageProducer(producer_id='rest_producer', \
message_type=message_type, method='sync')
producer.send(messages)
except MessageBusError as e:
status_code = e.rc
error_message = e.desc
response_obj = {'error_code': status_code, 'exception': ['MessageBusError', {'message' : error_message}]}
except Exception as e:
exception_key = type(e).__name__
exception = RestServerError(exception_key).http_error()
status_code = exception[0]
error_message = exception[1]
response_obj = {'error_code': status_code, 'exception': [exception_key, {'message' : error_message}]}
raise MessageBusError(status_code, error_message) from e
else:
status_code = 200 # No exception, Success
response_obj = {'status_code': status_code, 'status': 'success'}
finally:
return web.Response(text=json.dumps(response_obj) , status=status_code)
if request.method == 'GET':
try:
message_types = str(request.match_info['message_type']).split('&')
consumer_group = request.rel_url.query['consumer_group']
consumer = MessageConsumer(consumer_id='rest_consumer', \
consumer_group=consumer_group, message_types=message_types, \
auto_ack=True, offset='latest')
message = consumer.receive()
except MessageBusError as e:
status_code = e.rc
error_message = e.desc
response_obj = {'error_code': status_code, 'exception': ['MessageBusError', {'message' : error_message}]}
except Exception as e:
exception_key = type(e).__name__
exception = RestServerError(exception_key).http_error()
status_code = exception[0]
error_message = exception[1]
response_obj = {'error_code': status_code, 'exception': [exception_key, {'message' : error_message}]}
raise MessageBusError(status_code, error_message) from e
else:
status_code = 200 # No exception, Success
response_obj = {'messages': str(message)}
finally:
return web.Response(text=json.dumps(response_obj), status=status_code)
def receive_msg(self):
""" Receives a message """
from cortx.utils.message_bus import MessageConsumer
consumer = MessageConsumer(consumer_id='setup', \
consumer_group='provisioner', message_types=['mytest'], auto_ack=False, \
offset='earliest')
while True:
try:
time.sleep(1)
message = consumer.receive()
if message != None:
return message
except Exception as e:
raise SetupError(errno.EINVAL, \
"Unable to test the config. %s", e)
def test_receive(self):
""" Test Receive Message. """
message_bus = MessageBus()
consumer = MessageConsumer(message_bus, consumer_id='sel', \
consumer_group='sel', message_type=['Sel'], auto_ack=False, \
offset='latest')
self.assertIsNotNone(consumer, "Consumer not found")
while True:
try:
message = consumer.receive()
self.assertIsNotNone(message, "Message not found")
consumer.ack()
except Exception as e:
break
def initialize(self, conf_reader, msgQlist, product):
"""initialize configuration reader and internal msg queues"""
# Initialize ScheduledMonitorThread
super(IngressProcessor, self).initialize(conf_reader)
# Initialize internal message queues for this module
super(IngressProcessor, self).initialize_msgQ(msgQlist)
self._read_config()
producer_initialized.wait()
self._consumer = MessageConsumer(consumer_id=self._consumer_id,
consumer_group=self._consumer_group,
message_types=[self._message_type],
auto_ack=False,
offset=self._offset)
def test_007_receive_different_consumer_group(self):
"""Test receive from different consumer_group."""
consumer_group = ['group_1', 'group2']
for cg in consumer_group:
consumer = MessageConsumer(consumer_id=cg, consumer_group=cg, \
message_types=[TestMessageBus._message_type], auto_ack=False, \
offset='earliest')
count = 0
while True:
message = consumer.receive()
if message is None:
break
self.assertIsNotNone(message, "Message not found")
count += 1
self.assertEqual(count, (TestMessageBus._bulk_count + 1))
class EventConsumer:
def __init__(self, msg_topic):
self.consumer = MessageConsumer(consumer_id='hare',
consumer_group='test_consumer',
message_types=[msg_topic],
auto_ack=str(True),
offset='earliest')
print(f'Consumer created for reading topic={msg_topic}')
def receive_events(self):
while True:
try:
message = self.consumer.receive()
except Exception:
print('Subscribed topic not available.')
break
if message:
data = self._parse(message)
print(f'Received event={data}')
else:
print('No more events in the bus. Consumer exiting.')
break
def _parse(self, message):
return json.loads(message.decode('utf-8'))
def test_002_unknown_message_type(self):
"""Test invalid message type."""
with self.assertRaises(MessageBusError):
MessageProducer(producer_id='send', message_type='', method='sync')
with self.assertRaises(MessageBusError):
MessageConsumer(consumer_id='receive', consumer_group='test', \
message_types=[''], auto_ack=False, offset='earliest')
def receive_msg(self):
""" Receives a message """
from cortx.utils.message_bus import MessageConsumer
consumer = MessageConsumer(consumer_id='setup', \
consumer_group='provisioner', message_types=['mytest'], auto_ack=False, \
offset='earliest')
while True:
try:
time.sleep(1)
message = consumer.receive()
if message is not None:
return message
except Exception as e:
raise SetupError(errors.ERR_OP_FAILED, "Failed to receive messages" \
"for message_type: mytest. Unable to test the config." \
" %s with consumer_id: setup, message_types: mytest " \
"consumer_group: provisioner", e)
def test_consumer_two(self):
""" Test Receive Message for consumer group 2 """
consumer = MessageConsumer(TestMessage.message_bus, \
consumer_id='sspl_sensor1', consumer_group='c3', \
message_type=['test_type'], auto_ack=False, offset='latest')
self.assertIsNotNone(consumer, "Consumer not found")
count = 0
while True:
try:
message = consumer.receive()
count += 1
self.assertIsNotNone(message, "Message not found")
consumer.ack()
except Exception as e:
self.assertEqual(count, 10)
break
def setUpClass(cls):
"""Register the test message_type."""
cls._admin.register_message_type(message_types= \
[TestMessageBus._message_type], partitions=1)
cls._producer = MessageProducer(producer_id='send', \
message_type=TestMessageBus._message_type, method='sync')
cls._consumer = MessageConsumer(consumer_id='receive', \
consumer_group='test', message_types=[TestMessageBus._message_type], \
auto_ack=False, offset='earliest')
def setUpClass(cls):
"""Register the test message_type."""
cls._admin.register_message_type(message_types= \
[TestKVPayloadMessage._message_type], partitions=1)
cls._consumer = MessageConsumer(consumer_id='kv_consumer',
consumer_group='kv', message_types=[TestKVPayloadMessage.\
_message_type], auto_ack=True, offset='earliest')
cls._producer = MessageProducer(producer_id='kv_producer', \
message_type=TestKVPayloadMessage._message_type, method='sync')
def test_receive(self):
""" Test Receive Message. """
consumer = MessageConsumer(consumer_id='sspl_sensors',
consumer_group='sspl',
message_types=['Sel'],
auto_ack=False,
offset='latest')
self.assertIsNotNone(consumer, "Consumer not found")
count = 0
while True:
try:
message = consumer.receive()
count += 1
self.assertIsNotNone(message, "Message not found")
consumer.ack()
except Exception as e:
self.assertEqual(count, 10)
break
def test_consumer_unread_count(self):
""" Test unread message count from consumer side """
read_count = 0
consumer = MessageConsumer(consumer_id="c1", \
consumer_group=TestMessage._consumer_group, \
message_types=[TestMessage._msg_type], auto_ack=True, \
offset="latest")
while True:
message = consumer.receive(timeout=0)
if message is not None:
read_count += 1
consumer.ack()
if read_count == TestMessage._receive_limit:
break
unread_count = consumer.get_unread_count \
(message_type=TestMessage._msg_type)
self.assertEqual(unread_count, (TestMessage._msg_count - \
TestMessage._receive_limit))
self.assertNotEqual(unread_count, TestMessage._msg_count)
async def receive(request):
Log.debug(f"Received GET request for message type " \
f"{request.match_info['message_type']}. Getting message")
try:
message_types = str(request.match_info['message_type']).split('&')
consumer_group = request.rel_url.query['consumer_group']
consumer = MessageConsumer(consumer_id='rest_consumer', \
consumer_group=consumer_group, message_types=message_types, \
auto_ack=True, offset='latest')
message = consumer.receive()
except MessageBusError as e:
status_code = e.rc
error_message = e.desc
Log.error(f"Unable to receive message for message_type: " \
f"{message_types} using consumer group {consumer_group}, " \
f"status code: {status_code}, error: {error_message}")
response_obj = {'error_code': status_code, 'exception': \
['MessageBusError', {'message': error_message}]}
except Exception as e:
exception_key = type(e).__name__
exception = RestServerError(exception_key).http_error()
status_code = exception[0]
error_message = exception[1]
Log.error(f"Internal error while receiving messages for " \
f"message_type: {message_types}, status code: " \
f"{status_code}, error: {error_message}")
response_obj = {'error_code': status_code, 'exception': \
[exception_key, {'message': error_message}]}
raise MessageBusError(status_code, error_message) from e
else:
status_code = 200 # No exception, Success
response_obj = {'messages': str(message)}
Log.debug(f"Received message - {message} for message_type "
f"{message_types} using GET method finished with " \
f"status code: {status_code}")
finally:
return web.Response(text=json.dumps(response_obj), \
status=status_code)
def __init__(self,
event_list: List[SubscribeEvent],
group_id: str = COMPONENT_ID):
"""
event_list - list of events to subscribe to.
group_id - the group_id to pass to KafkaConsumer.
"""
logging.debug('Inside EventListener')
self.event_manager = EventManager.get_instance()
topic = self._subscribe(event_list)
if topic is None:
raise RuntimeError('Failed to subscribe to events')
message_bus = MessageBus()
self.consumer = MessageConsumer(
message_bus=message_bus,
consumer_id=COMPONENT_ID,
consumer_group=group_id,
message_types=[topic],
# Why is it 'str' in cortx-py-utils??
auto_ack=str(False),
offset='earliest')
def subscribe(cls, component: str, **filters):
"""
Subscribe to IEM alerts
Parameters:
component Component that generates the IEM. For e.g. 'S3', 'SSPL'
"""
if component is None:
raise EventMessageError(errno.EINVAL, "Invalid component type: %s", \
component)
cls._consumer = MessageConsumer(consumer_id='event_consumer', \
consumer_group=component, message_types=['IEM'], \
auto_ack=True, offset='earliest')
def start(self):
"""
Start the consumer
"""
Log.info(f"Starting the daemon for {self.name}...")
self.consumer = MessageConsumer(consumer_id=str(self.consumer_id),
consumer_group=self.consumer_group,
message_types=[self.message_type],
auto_ack=self.auto_ack,
offset=self.offset)
self.consumer_thread = Thread(target=self.run, name=self.name)
self.consumer_thread.setDaemon(True)
self.consumer_thread.start()
Log.info(f"The daemon {self.name} started successfully.")
def subscribe(cls, component: str, message_server_endpoints: str, \
**message_server_kwargs):
"""
Subscribe to IEM alerts
Parameters:
component Component that generates the IEM. For e.g. 'S3', 'SSPL'
"""
if component is None:
Log.error("Invalid component type: %s" % component)
raise EventMessageError(errno.EINVAL, "Invalid component type: %s", \
component)
MessageBus.init(message_server_endpoints, **message_server_kwargs)
cls._consumer = MessageConsumer(consumer_id='event_consumer', \
consumer_group=component, message_types=['IEM'], \
auto_ack=True, offset='earliest')
Log.info("IEM Consumer initialized for component: %s" % component)
def setUpClass(cls, \
cluster_conf_path: str = 'yaml:///etc/cortx/cluster.conf'):
"""Register the test message_type."""
if TestKVPayloadMessage._cluster_conf_path:
cls.cluster_conf_path = TestKVPayloadMessage._cluster_conf_path
else:
cls.cluster_conf_path = cluster_conf_path
Conf.load('config', cls.cluster_conf_path, skip_reload=True)
message_server_endpoints = Conf.get('config',\
'cortx>external>kafka>endpoints')
Log.init('message_bus', '/var/log', level='INFO', \
backup_count=5, file_size_in_mb=5)
MessageBus.init(message_server_endpoints=message_server_endpoints)
cls._admin = MessageBusAdmin(admin_id='register')
cls._admin.register_message_type(message_types= \
[TestKVPayloadMessage._message_type], partitions=1)
cls._consumer = MessageConsumer(consumer_id='kv_consumer', \
consumer_group='kv', message_types=[TestKVPayloadMessage.\
_message_type], auto_ack=True, offset='earliest')
cls._producer = MessageProducer(producer_id='kv_producer', \
message_type=TestKVPayloadMessage._message_type, method='sync')
def init(cls, component: str, source: str, receiver: bool = False):
""" Set the Event Message context """
cls._component = component
cls._source = source
cls()
if cls._component is None:
raise EventMessageError(errno.EINVAL, "Invalid component type: %s", \
cls._component)
if cls._source not in cls._SOURCE.keys():
raise EventMessageError(errno.EINVAL, "Invalid source type: %s", \
cls._source)
if receiver:
cls._client = MessageConsumer(consumer_id='event_consumer', \
consumer_group=cls._component, message_types=['IEM'], \
auto_ack=True, offset='earliest')
else:
cls._client = MessageProducer(producer_id='event_producer', \
message_type='IEM', method='sync')
class IngressProcessor(ScheduledModuleThread, InternalMsgQ):
"""Handles incoming messages via message bus."""
MODULE_NAME = "IngressProcessor"
PRIORITY = 1
# Section and keys in configuration file
PROCESSOR = MODULE_NAME.upper()
CONSUMER_ID = "consumer_id"
CONSUMER_GROUP = "consumer_group"
MESSAGE_TYPE = "message_type"
OFFSET = "offset"
SYSTEM_INFORMATION_KEY = 'SYSTEM_INFORMATION'
CLUSTER_ID_KEY = 'cluster_id'
NODE_ID_KEY = 'node_id'
JSON_ACTUATOR_SCHEMA = "SSPL-LL_Actuator_Request.json"
JSON_SENSOR_SCHEMA = "SSPL-LL_Sensor_Request.json"
@staticmethod
def name():
""" @return: name of the module."""
return IngressProcessor.MODULE_NAME
def __init__(self):
super(IngressProcessor, self).__init__(self.MODULE_NAME, self.PRIORITY)
# Read in the actuator schema for validating messages
schema_file = os.path.join(RESOURCE_PATH + '/actuators',
self.JSON_ACTUATOR_SCHEMA)
self._actuator_schema = self._load_schema(schema_file)
# Read in the sensor schema for validating messages
schema_file = os.path.join(RESOURCE_PATH + '/sensors',
self.JSON_SENSOR_SCHEMA)
self._sensor_schema = self._load_schema(schema_file)
def _load_schema(self, schema_file):
"""Loads a schema from a file and validates
@param string schema_file location of schema on the file system
@return string Trimmed and validated schema
"""
with open(schema_file, 'r') as f:
schema = json.load(f)
# Validate the schema to conform to Draft 3 specification
Draft3Validator.check_schema(schema)
return schema
def initialize(self, conf_reader, msgQlist, product):
"""initialize configuration reader and internal msg queues"""
# Initialize ScheduledMonitorThread
super(IngressProcessor, self).initialize(conf_reader)
# Initialize internal message queues for this module
super(IngressProcessor, self).initialize_msgQ(msgQlist)
self._read_config()
producer_initialized.wait()
self._consumer = MessageConsumer(consumer_id=self._consumer_id,
consumer_group=self._consumer_group,
message_types=[self._message_type],
auto_ack=False,
offset=self._offset)
def run(self):
# self._set_debug(True)
# self._set_debug_persist(True)
# time.sleep(180)
logger.info(
"IngressProcessor, Initialization complete, accepting requests")
try:
while True:
message = self._consumer.receive()
if message:
logger.info(
f"IngressProcessor, Message Recieved: {message}")
self._process_msg(message)
self._consumer.ack()
else:
time.sleep(1)
except Exception as e:
if self.is_running() is True:
logger.info(
"IngressProcessor ungracefully breaking out of run loop, restarting."
)
logger.error("IngressProcessor, Exception: %s" % str(e))
self._scheduler.enter(10, self._priority, self.run, ())
else:
logger.info(
"IngressProcessor gracefully breaking out of run Loop, not restarting."
)
self._log_debug("Finished processing successfully")
def _process_msg(self, body):
"""Parses the incoming message and hands off to the appropriate module"""
ingressMsg = {}
uuid = None
try:
if isinstance(body, dict) is False:
ingressMsg = json.loads(body)
else:
ingressMsg = body
# Authenticate message using username and signature fields
username = ingressMsg.get("username")
signature = ingressMsg.get("signature")
message = ingressMsg.get("message")
uuid = ingressMsg.get("uuid")
msg_len = len(message) + 1
if uuid is None:
uuid = "N/A"
if use_security_lib and \
SSPL_SEC.sspl_verify_message(msg_len, str(message),
username, signature) != 0:
logger.warn(
"IngressProcessor, Authentication failed on message: %s" %
ingressMsg)
return
# Get the incoming message type
if message.get("actuator_request_type") is not None:
msgType = message.get("actuator_request_type")
# Validate against the actuator schema
validate(ingressMsg, self._actuator_schema)
elif message.get("sensor_request_type") is not None:
msgType = message.get("sensor_request_type")
# Validate against the sensor schema
validate(ingressMsg, self._sensor_schema)
else:
# We only handle incoming actuator and sensor requests, ignore
# everything else.
return
# Check for debugging being activated in the message header
self._check_debug(message)
self._log_debug("_process_msg, ingressMsg: %s" % ingressMsg)
self._send_to_msg_handler(msgType, message, uuid)
except Exception as ex:
logger.error(
"IngressProcessor, _process_msg unrecognized message: %r" %
ingressMsg)
ack_msg = AckResponseMsg("Error Processing Msg",
"Msg Handler Not Found", uuid).getJson()
self._write_internal_msgQ(EgressProcessor.name(), ack_msg)
def _send_to_msg_handler(self, msgType, message, uuid):
# Hand off to appropriate actuator message handler
if msgType.get("thread_controller") is not None:
self._write_internal_msgQ("ThreadController", message)
elif msgType.get("service_controller") is not None:
self._write_internal_msgQ("ServiceMsgHandler", message)
elif msgType.get("node_controller") is not None:
self._write_internal_msgQ("NodeControllerMsgHandler", message)
elif msgType.get("storage_enclosure") is not None:
self._write_internal_msgQ("RealStorActuatorMsgHandler", message)
# Hand off to appropriate sensor message handler
elif msgType.get("node_data") is not None:
self._write_internal_msgQ("NodeDataMsgHandler", message)
elif msgType.get("enclosure_alert") is not None:
self._write_internal_msgQ("RealStorEnclMsgHandler", message)
elif msgType.get("storage_enclosure") is not None:
self._write_internal_msgQ("RealStorActuatorMsgHandler", message)
# ... handle other incoming messages that have been validated
else:
# Send ack about not finding a msg handler
ack_msg = AckResponseMsg("Error Processing Message",
"Message Handler Not Found",
uuid).getJson()
self._write_internal_msgQ(EgressProcessor.name(), ack_msg)
def _read_config(self):
"""Read config for messaging bus."""
# Make methods locally available
self._node_id = Conf.get(SSPL_CONF,
f"{CLUSTER}>{SRVNODE}>{self.NODE_ID_KEY}",
'SN01')
self._consumer_id = Conf.get(SSPL_CONF,
f"{self.PROCESSOR}>{self.CONSUMER_ID}",
'sspl_actuator')
self._consumer_group = Conf.get(
SSPL_CONF, f"{self.PROCESSOR}>{self.CONSUMER_GROUP}",
'cortx_monitor')
self._message_type = Conf.get(SSPL_CONF,
f"{self.PROCESSOR}>{self.MESSAGE_TYPE}",
'requests')
self._offset = Conf.get(SSPL_CONF, f"{self.PROCESSOR}>{self.OFFSET}",
'earliest')
def shutdown(self):
"""Clean up scheduler queue and gracefully shutdown thread"""
# TODO: cleanup message bus connection if that
# functionality get added in messaging framework
super(IngressProcessor, self).shutdown()
class S3CortxMsgBus:
def __init__(self):
"""Init."""
self._producer = None
self._consumer = None
def setup_producer(self, prod_id, msg_type, method):
"""Setup producer."""
try:
self._producer = MessageProducer(producer_id=prod_id,
message_type=msg_type,
method=method)
except Exception as exception:
msg = ("msg_bus setup producer except:%s %s") % (
exception, traceback.format_exc())
return False, msg
return True, None
def send(self, messages):
"""Send the constructed message."""
try:
self._producer.send(messages)
except Exception as exception:
msg = ("msg_bus send except:%s %s") % (
exception, traceback.format_exc())
return False, msg
return True, None
def purge(self):
"""Purge/Delete all the messages."""
self._producer.delete()
def setup_consumer(self, cons_id, group, msg_type, auto_ack, offset):
"""Setup the consumer."""
try:
self._consumer = MessageConsumer(consumer_id=cons_id, \
consumer_group=group, message_types=[msg_type], auto_ack=auto_ack, offset=offset)
except Exception as exception:
msg = ("msg_bus setup_consumer except:%s %s") % (
exception, traceback.format_exc())
return False, msg
return True, None
def receive(self, daemon_mode):
"""Receive the incoming message."""
try:
if daemon_mode:
#timeout=0 makes it as blocking indefinitely
message = self._consumer.receive(timeout=0)
else:
#timeout = 0.5 sec, by default, which is non-blocking
message = self._consumer.receive()
except Exception as exception:
msg = ("msg_bus receive except:%s %s") % (
exception, traceback.format_exc())
return False, msg
return True, message
def ack(self):
"""Ack the received message."""
try:
self._consumer.ack()
except Exception as exception:
msg = ("msg_bus ack except:%s %s") % (
exception, traceback.format_exc())
return False, msg
return True, None
def count(self, consumer_group):
"""Get the count of unread messages."""
unread_count = 0
try:
unread_count = self._producer.get_unread_count(consumer_group)
except:
return 0
return unread_count
@staticmethod
def create_topic(admin_id: str, message_types: list, partitions: int):
"""create topic."""
mbadmin = MessageBusAdmin(admin_id = admin_id)
try:
mbadmin.register_message_type(message_types = message_types,
partitions = partitions)
except Exception as e:
if "TOPIC_ALREADY_EXISTS" not in str(e):
raise(e)
@staticmethod
def add_concurrency(admin_id: str, message_type: str, concurrency_count: int):
"""Increase partition count for given topic."""
mbadmin = MessageBusAdmin(admin_id = admin_id)
mbadmin.add_concurrency(message_type = message_type,
concurrency_count = concurrency_count)
@staticmethod
def delete_topic(admin_id: str, message_types: list):
"""Delete given topic"""
mbadmin = MessageBusAdmin(admin_id = admin_id)
mbadmin.deregister_message_type(message_types = message_types)
@staticmethod
def list_topics(admin_id: str):
"""list all available topics"""
mbadmin = MessageBusAdmin(admin_id = admin_id)
return mbadmin.list_message_types()
@staticmethod
def is_topic_exist(admin_id: str, topic_name: str):
"""retuns true if topic exist else false"""
mbadmin = MessageBusAdmin(admin_id = admin_id)
if topic_name in mbadmin.list_message_types():
return True
return False
