def send_msg(self, message): """ Sends a message """ from cortx.utils.message_bus import MessageProducer producer = MessageProducer(producer_id='setup', \ message_type='mytest', method='sync') producer.send(message) time.sleep(1)
def test_multiple_instance(self): """ Test Clients with no instance """ message_bus_1 = MessageBus() message_bus_2 = MessageBus() message_bus_3 = MessageBus() admin = MessageBusAdmin(admin_id='admin', message_bus=message_bus_1) self.assertTrue(TestMessage._msg_type in admin.list_message_types()) messages = [] producer = MessageProducer(producer_id='p1', \ message_type=TestMessage._msg_type, method='sync', \ message_bus=message_bus_2) self.assertIsNotNone(producer, "Producer not found") for i in range(0, TestMessage._msg_count): messages.append("This is message" + str(i)) producer.send(messages) consumer = MessageConsumer(consumer_id='msys', consumer_group='connn', \ message_types=[TestMessage._msg_type], auto_ack=True, \ offset='latest', message_bus=message_bus_3) count = 0 while True: try: message = consumer.receive() if isinstance(message, bytes): count += 1 consumer.ack() except Exception: self.assertEqual(count, TestMessage._msg_count) break
def test_send(self): """ Test Send Message. """ messages = [] message_bus = MessageBus() producer = MessageProducer(message_bus, producer_id='sel', \ message_type='Sel', method='async') self.assertIsNotNone(producer, "Producer not found") for i in range(0, 10): messages.append("This is message" + str(i)) self.assertIsInstance(messages, list) producer.send(messages)
def test_send(self): """ Test Send Message """ messages = [] producer = MessageProducer(TestMessage.message_bus, \ producer_id='p1', message_type='big') self.assertIsNotNone(producer, "Producer not found") for i in range(0, 100): messages.append("This is message" + str(i)) self.assertEqual(len(messages), 100) self.assertIsInstance(messages, list) producer.send(messages)
async def message_bus_rest(request): if request.method == 'POST': try: message_type = request.match_info['message_type'] payload = await request.json() messages = payload['messages'] producer = MessageProducer(producer_id='rest_producer', \ message_type=message_type, method='sync') producer.send(messages) except MessageBusError as e: status_code = e.rc error_message = e.desc response_obj = {'error_code': status_code, 'exception': ['MessageBusError', {'message' : error_message}]} except Exception as e: exception_key = type(e).__name__ exception = RestServerError(exception_key).http_error() status_code = exception[0] error_message = exception[1] response_obj = {'error_code': status_code, 'exception': [exception_key, {'message' : error_message}]} raise MessageBusError(status_code, error_message) from e else: status_code = 200 # No exception, Success response_obj = {'status_code': status_code, 'status': 'success'} finally: return web.Response(text=json.dumps(response_obj) , status=status_code) if request.method == 'GET': try: message_types = str(request.match_info['message_type']).split('&') consumer_group = request.rel_url.query['consumer_group'] consumer = MessageConsumer(consumer_id='rest_consumer', \ consumer_group=consumer_group, message_types=message_types, \ auto_ack=True, offset='latest') message = consumer.receive() except MessageBusError as e: status_code = e.rc error_message = e.desc response_obj = {'error_code': status_code, 'exception': ['MessageBusError', {'message' : error_message}]} except Exception as e: exception_key = type(e).__name__ exception = RestServerError(exception_key).http_error() status_code = exception[0] error_message = exception[1] response_obj = {'error_code': status_code, 'exception': [exception_key, {'message' : error_message}]} raise MessageBusError(status_code, error_message) from e else: status_code = 200 # No exception, Success response_obj = {'messages': str(message)} finally: return web.Response(text=json.dumps(response_obj), status=status_code)
def test_producer_unread_count(self): """ Test unread message count from producer side """ producer = MessageProducer(producer_id="p1", \ message_type=TestMessage._msg_type, method="sync") messages = [] for i in range(0, TestMessage._msg_count): messages.append("This is message" + str(i)) self.assertEqual(len(messages), TestMessage._msg_count) self.assertIsInstance(messages, list) producer.send(messages) unread_count = producer.get_unread_count( consumer_group=TestMessage._consumer_group) self.assertEqual(unread_count, TestMessage._msg_count) self.assertFalse(unread_count != 100)
class MessageBusProducer: PRODUCER_METHOD = "sync" def __init__(self, producer_id: str, message_type: str, partitions: int): """ Register message types with message bus. Args: producer_id (str): producer id. message_types (str): Message type. partitions (int, optional): No. of partitions. Defaults to 1. Raises: MessageBusError: Message bus error. """ self.producer = MessageProducer( producer_id=producer_id, message_type=message_type, method=MessageBusProducer.PRODUCER_METHOD) def publish(self, message: any): """ Produce message to message bus. Args: message (any): Message. If msg is dict it will be dumped as json. If msg is string then it will be send directly. If message is list, it should have all string element, all items will be published. """ if isinstance(message, dict): self.producer.send([json.dumps(message)]) elif isinstance(message, str): self.producer.send([message]) elif isinstance(message, list): self.producer.send(message) else: raise Exception(f"Invalid type of message {message}")
async def send(request): Log.debug(f"Received POST request for message type " \ f"{request.match_info['message_type']}. Processing message") try: message_type = request.match_info['message_type'] payload = await request.json() messages = payload['messages'] producer = MessageProducer(producer_id='rest_producer', \ message_type=message_type, method='sync') producer.send(messages) except MessageBusError as e: status_code = e.rc error_message = e.desc Log.error(f"Unable to send message for message_type: " \ f"{message_type}, status code: {status_code}," \ f" error: {error_message}") response_obj = {'error_code': status_code, 'exception': \ ['MessageBusError', {'message': error_message}]} except Exception as e: exception_key = type(e).__name__ exception = RestServerError(exception_key).http_error() status_code = exception[0] error_message = exception[1] Log.error(f"Internal error while sending messages for " \ f"message_type: {message_type}, status code: " \ f"{status_code}, error: {error_message}") response_obj = {'error_code': status_code, 'exception': \ [exception_key, {'message': error_message}]} raise MessageBusError(status_code, error_message) from e else: status_code = 200 # No exception, Success Log.debug(f"Sending messages for message_type " \ f"{message_type} using POST method finished with status " \ f"code: {status_code}") response_obj = {'status_code': status_code, 'status': 'success'} finally: return web.Response(text=json.dumps(response_obj), \ status=status_code)
class EventProducer: def __init__(self, msg_topic): self.topic = msg_topic admin = MessageBusAdmin("ha_admin") # Register the topic only once. try: admin.register_message_type([self.topic], 1) print(f'Message topic={self.topic} registered on Message Bus.') except Exception: print(f'Message topic={self.topic} is already registered!!') self.producer = MessageProducer(producer_id="system_health", message_type=self.topic, method="sync") print(f'Producer created for topic={self.topic}') def send_event(self, event): print('Sending event...') if isinstance(event, myEvent): event_to_send = event.get_json_msg() else: event_to_send = json.dumps(event) self.producer.send([event_to_send]) print(f'Sent event={event} to message bus.')
"description": "The cluster has lost srvnode-1 server. System is running in degraded mode. For more information refer the Troubleshooting guide. Extra Info: host=srvnode-1; status=standby;", "impact": "NA", "recommendation": "NA", "site_id": "1", "node_id": "0", "rack_id": "1", "cluster_id": "e766bd52-c19c-45b6-9c91-663fd8203c2e" }, "alert_type": "get", "severity": "warning", "specific_info": { "source": "Software", "component": "ha", "module": "Node", "event": "The cluster has lost one server. System is running in degraded mode.", "IEC": "WS0080010001" }, "alert_id": "16215817982ac7b67a94584377a060ffaca0b56cf8", "host_id": "srvnode-1.mgmt.public" } } }) ] producer = MessageProducer(producer_id="sspl-sensor", message_type="alerts", method="sync") producer.send(message_list) print("Message send")
class S3CortxMsgBus: def __init__(self): """Init.""" self._producer = None self._consumer = None def setup_producer(self, prod_id, msg_type, method): """Setup producer.""" try: self._producer = MessageProducer(producer_id=prod_id, message_type=msg_type, method=method) except Exception as exception: msg = ("msg_bus setup producer except:%s %s") % ( exception, traceback.format_exc()) return False, msg return True, None def send(self, messages): """Send the constructed message.""" try: self._producer.send(messages) except Exception as exception: msg = ("msg_bus send except:%s %s") % ( exception, traceback.format_exc()) return False, msg return True, None def purge(self): """Purge/Delete all the messages.""" self._producer.delete() def setup_consumer(self, cons_id, group, msg_type, auto_ack, offset): """Setup the consumer.""" try: self._consumer = MessageConsumer(consumer_id=cons_id, \ consumer_group=group, message_types=[msg_type], auto_ack=auto_ack, offset=offset) except Exception as exception: msg = ("msg_bus setup_consumer except:%s %s") % ( exception, traceback.format_exc()) return False, msg return True, None def receive(self, daemon_mode): """Receive the incoming message.""" try: if daemon_mode: #timeout=0 makes it as blocking indefinitely message = self._consumer.receive(timeout=0) else: #timeout = 0.5 sec, by default, which is non-blocking message = self._consumer.receive() except Exception as exception: msg = ("msg_bus receive except:%s %s") % ( exception, traceback.format_exc()) return False, msg return True, message def ack(self): """Ack the received message.""" try: self._consumer.ack() except Exception as exception: msg = ("msg_bus ack except:%s %s") % ( exception, traceback.format_exc()) return False, msg return True, None def count(self, consumer_group): """Get the count of unread messages.""" unread_count = 0 try: unread_count = self._producer.get_unread_count(consumer_group) except: return 0 return unread_count @staticmethod def create_topic(admin_id: str, message_types: list, partitions: int): """create topic.""" mbadmin = MessageBusAdmin(admin_id = admin_id) try: mbadmin.register_message_type(message_types = message_types, partitions = partitions) except Exception as e: if "TOPIC_ALREADY_EXISTS" not in str(e): raise(e) @staticmethod def add_concurrency(admin_id: str, message_type: str, concurrency_count: int): """Increase partition count for given topic.""" mbadmin = MessageBusAdmin(admin_id = admin_id) mbadmin.add_concurrency(message_type = message_type, concurrency_count = concurrency_count) @staticmethod def delete_topic(admin_id: str, message_types: list): """Delete given topic""" mbadmin = MessageBusAdmin(admin_id = admin_id) mbadmin.deregister_message_type(message_types = message_types) @staticmethod def list_topics(admin_id: str): """list all available topics""" mbadmin = MessageBusAdmin(admin_id = admin_id) return mbadmin.list_message_types() @staticmethod def is_topic_exist(admin_id: str, topic_name: str): """retuns true if topic exist else false""" mbadmin = MessageBusAdmin(admin_id = admin_id) if topic_name in mbadmin.list_message_types(): return True return False
class RabbitMQegressProcessor(ScheduledModuleThread, InternalMsgQ): """Handles outgoing messages via rabbitMQ over localhost""" MODULE_NAME = "RabbitMQegressProcessor" PRIORITY = 1 # Section and keys in configuration file RABBITMQPROCESSOR = MODULE_NAME.upper() SIGNATURE_USERNAME = '******' SIGNATURE_TOKEN = 'message_signature_token' SIGNATURE_EXPIRES = 'message_signature_expires' IEM_ROUTE_ADDR = 'iem_route_addr' PRODUCER_ID = 'producer_id' MESSAGE_TYPE = 'message_type' METHOD = 'method' @staticmethod def name(): """ @return: name of the module.""" return RabbitMQegressProcessor.MODULE_NAME def __init__(self): super(RabbitMQegressProcessor, self).__init__(self.MODULE_NAME, self.PRIORITY) def initialize(self, conf_reader, msgQlist, product): """initialize configuration reader and internal msg queues""" # Initialize ScheduledMonitorThread super(RabbitMQegressProcessor, self).initialize(conf_reader) # Initialize internal message queues for this module super(RabbitMQegressProcessor, self).initialize_msgQ(msgQlist) self.store_queue = StoreQueue() # Flag denoting that a shutdown message has been placed # into our message queue from the main sspl_ll_d handler self._request_shutdown = False self._read_config() self._producer = MessageProducer(message_bus, producer_id=self._producer_id, message_type=self._message_type, method=self._method) producer_initialized.set() def run(self): """Run the module periodically on its own thread. """ self._log_debug("Start accepting requests") # self._set_debug(True) # self._set_debug_persist(True) try: # Loop thru all messages in queue until and transmit while not self._is_my_msgQ_empty(): self._jsonMsg, self._event = self._read_my_msgQ() if self._jsonMsg is not None: self._transmit_msg_on_exchange() except Exception: # Log it and restart the whole process when a failure occurs logger.error("RabbitMQegressProcessor restarting") self._log_debug("Finished processing successfully") # Shutdown is requested by the sspl_ll_d shutdown handler # placing a 'shutdown' msg into our queue which allows us to # finish processing any other queued up messages. if self._request_shutdown is True: self.shutdown() else: self._scheduler.enter(1, self._priority, self.run, ()) def _read_config(self): """Configure the RabbitMQ exchange with defaults available""" try: self._signature_user = Conf.get( SSPL_CONF, f"{self.RABBITMQPROCESSOR}>{self.SIGNATURE_USERNAME}", 'sspl-ll') self._signature_token = Conf.get( SSPL_CONF, f"{self.RABBITMQPROCESSOR}>{self.SIGNATURE_TOKEN}", 'FAKETOKEN1234') self._signature_expires = Conf.get( SSPL_CONF, f"{self.RABBITMQPROCESSOR}>{self.SIGNATURE_EXPIRES}", "3600") self._producer_id = Conf.get( SSPL_CONF, f"{self.RABBITMQPROCESSOR}>{self.PRODUCER_ID}", "sspl-sensor") self._message_type = Conf.get( SSPL_CONF, f"{self.RABBITMQPROCESSOR}>{self.MESSAGE_TYPE}", "alerts") self._method = Conf.get(SSPL_CONF, f"{self.RABBITMQPROCESSOR}>{self.METHOD}", "sync") except Exception as ex: logger.error("RabbitMQegressProcessor, _read_config: %r" % ex) def _add_signature(self): """Adds the authentication signature to the message""" self._log_debug("_add_signature, jsonMsg: %s" % self._jsonMsg) self._jsonMsg["username"] = self._signature_user self._jsonMsg["expires"] = int(self._signature_expires) self._jsonMsg["time"] = str(int(time.time())) if use_security_lib: authn_token_len = len(self._signature_token) + 1 session_length = int(self._signature_expires) token = ctypes.create_string_buffer( SSPL_SEC.sspl_get_token_length()) SSPL_SEC.sspl_generate_session_token(self._signature_user, authn_token_len, self._signature_token, session_length, token) # Generate the signature msg_len = len(self._jsonMsg) + 1 sig = ctypes.create_string_buffer(SSPL_SEC.sspl_get_sig_length()) SSPL_SEC.sspl_sign_message(msg_len, str(self._jsonMsg), self._signature_user, token, sig) self._jsonMsg["signature"] = str(sig.raw, encoding='utf-8') else: self._jsonMsg["signature"] = "SecurityLibNotInstalled" def _transmit_msg_on_exchange(self): """Transmit json message onto RabbitMQ exchange""" self._log_debug("_transmit_msg_on_exchange, jsonMsg: %s" % self._jsonMsg) try: # Check for shut down message from sspl_ll_d and set a flag to shutdown # once our message queue is empty if self._jsonMsg.get("message").get( "actuator_response_type") is not None and \ self._jsonMsg.get("message").get( "actuator_response_type").get( "thread_controller") is not None and \ self._jsonMsg.get("message").get( "actuator_response_type").get("thread_controller").get( "thread_response") == \ "SSPL-LL is shutting down": logger.info( "RabbitMQegressProcessor, _transmit_msg_on_exchange, received" "global shutdown message from sspl_ll_d") self._request_shutdown = True # Publish json message to the correct channel # NOTE: We need to route ThreadController messages to ACK channel. # We can't modify schema as it will affect other modules too. As a # temporary solution we have added a extra check to see if actuator_response_type # is "thread_controller". # TODO: Find a proper way to solve this issue. Avoid changing # core egress processor code if self._jsonMsg.get("message").get( "actuator_response_type") is not None and \ (self._jsonMsg.get("message").get( "actuator_response_type").get("ack") is not None or self._jsonMsg.get("message").get( "actuator_response_type").get( "thread_controller") is not None): self._add_signature() self._producer.send([json.dumps(self._jsonMsg)]) logger.debug( "_transmit_msg_on_exchange, Successfully Sent: %s" % self._jsonMsg) # Routing requests for IEM msgs sent from the LoggingMsgHandler elif self._jsonMsg.get("message").get("IEM_routing") is not None: log_msg = self._jsonMsg.get("message").get("IEM_routing").get( "log_msg") if self._iem_route_addr != "": self._producer.send([json.dumps(self._jsonMsg)]) else: logger.warn( "RabbitMQegressProcessor, Attempted to route IEM without a valid 'iem_route_addr' set." ) logger.debug( "_transmit_msg_on_exchange, Successfully Sent: %s" % log_msg) else: self._add_signature() jsonMsg = json.dumps(self._jsonMsg) try: if self.store_queue.is_empty(): self._producer.send([jsonMsg]) logger.info(f"Published Alert: {jsonMsg}") else: logger.info("'Accumulated msg queue' is not Empty." + " Adding the msg to the end of the queue") self.store_queue.put(jsonMsg) except MessageBusError as e: logger.error( f"RabbitMQegressProcessor, _transmit_msg_on_exchange, error {e} in producing message,\ adding message to consul {self._jsonMsg}") self.store_queue.put(jsonMsg) except Exception as err: logger.error( f'RabbitMQegressProcessor, _transmit_msg_on_exchange, Unknown error {err} while publishing the message, adding to persistent store {self._jsonMsg}' ) self.store_queue.put(jsonMsg) # If event is added by sensors, set it if self._event: self._event.set() except Exception as ex: logger.error( f'RabbitMQegressProcessor, _transmit_msg_on_exchange, problem while publishing the message:{ex}, adding message to consul: {self._jsonMsg}' ) def shutdown(self): """Clean up scheduler queue and gracefully shutdown thread""" super(RabbitMQegressProcessor, self).shutdown()
class TestEgressProcessor: """Handles outgoing messages via messaging over localhost.""" MODULE_NAME = "EgressProcessorTests" PRIORITY = 1 # Section and keys in configuration file SYSTEM_INFORMATION = "SYSTEM_INFORMATION" RACK_ID = "rack_id" NODE_ID = "node_id" CLUSTER_ID = "cluster_id" SITE_ID = "site_id" PROCESSOR = MODULE_NAME.upper() SIGNATURE_USERNAME = "******" SIGNATURE_TOKEN = "message_signature_token" SIGNATURE_EXPIRES = "message_signature_expires" IEM_ROUTE_ADDR = "iem_route_addr" PRODUCER_ID = "producer_id" MESSAGE_TYPE = "message_type" METHOD = "method" def __init__(self): self._request_shutdown = False self._msg_sent_succesfull = True # Configure messaging Exchange to transmit messages self._connection = None self._read_config() self._producer = MessageProducer( producer_id=self._producer_id, message_type=self._message_type, method=self._method, ) def _read_config(self): """Configure the messaging exchange with defaults available.""" try: self._signature_user = Conf.get( SSPL_TEST_CONF, f"{self.PROCESSOR}>{self.SIGNATURE_USERNAME}", "sspl-ll") self._signature_token = Conf.get( SSPL_TEST_CONF, f"{self.PROCESSOR}>{self.SIGNATURE_TOKEN}", "FAKETOKEN1234", ) self._signature_expires = Conf.get( SSPL_TEST_CONF, f"{self.PROCESSOR}>{self.SIGNATURE_EXPIRES}", "3600") self._producer_id = Conf.get( SSPL_TEST_CONF, f"{self.PROCESSOR}>{self.PRODUCER_ID}", "sspl-sensor") self._message_type = Conf.get( SSPL_TEST_CONF, f"{self.PROCESSOR}>{self.MESSAGE_TYPE}", "Alerts") self._method = Conf.get(SSPL_TEST_CONF, f"{self.PROCESSOR}>{self.METHOD}", "Sync") except Exception as ex: logger.error("EgressProcessorTests, _read_config: %r" % ex) def publish(self, msg): """Transmit json message onto messaging exchange.""" msg = json.dumps(msg) self._producer.send([msg])
class S3CortxMsgBus: def __init__(self): """Init.""" self._message_bus = None self._producer = None self._consumer = None def connect(self): """Connect to Message Bus.""" try: self._message_bus = MessageBus() except Exception as exception: msg = ("msg_bus init except:%s %s") % (exception, traceback.format_exc()) return False, msg return True, None def setup_producer(self, prod_id, msg_type, method): """Setup producer.""" if not self._message_bus: raise Exception("Non Existent Message Bus") try: self._producer = MessageProducer(self._message_bus, \ producer_id=prod_id, message_type=msg_type, method=method) except Exception as exception: msg = ("msg_bus setup producer except:%s %s") % ( exception, traceback.format_exc()) return False, msg return True, None def send(self, messages): """Send the constructed message.""" try: self._producer.send(messages) except Exception as exception: msg = ("msg_bus send except:%s %s") % (exception, traceback.format_exc()) return False, msg return True, None def purge(self): """Purge/Delete all the messages.""" if not self._message_bus: raise Exception("Non Existent Message Bus, Cannot Purge") self._producer.delete() def setup_consumer(self, cons_id, group, msg_type, auto_ack, offset): """Setup the consumer.""" if not self._message_bus: raise Exception("Non Existent Message Bus") try: self._consumer = MessageConsumer(self._message_bus, consumer_id=cons_id, \ consumer_group=group, message_type=[msg_type], auto_ack=auto_ack, offset=offset) except Exception as exception: msg = ("msg_bus setup_consumer except:%s %s") % ( exception, traceback.format_exc()) return False, msg return True, None def receive(self): """Receive the incoming message.""" try: message = self._consumer.receive() except Exception as exception: msg = ("msg_bus receive except:%s %s") % (exception, traceback.format_exc()) return False, msg return True, message def ack(self): """Ack the received message.""" try: self._consumer.ack() except Exception as exception: msg = ("msg_bus ack except:%s %s") % (exception, traceback.format_exc()) return False, msg return True, None
class EgressProcessorTests(ScheduledModuleThread, InternalMsgQ): """Handles outgoing messages via messaging over localhost.""" MODULE_NAME = "EgressProcessorTests" PRIORITY = 1 # Section and keys in configuration file SYSTEM_INFORMATION = "SYSTEM_INFORMATION" RACK_ID = "rack_id" NODE_ID = "node_id" CLUSTER_ID = "cluster_id" SITE_ID = "site_id" PROCESSOR = MODULE_NAME.upper() SIGNATURE_USERNAME = '******' SIGNATURE_TOKEN = 'message_signature_token' SIGNATURE_EXPIRES = 'message_signature_expires' IEM_ROUTE_ADDR = 'iem_route_addr' PRODUCER_ID = 'producer_id' MESSAGE_TYPE = 'message_type' METHOD = 'method' @staticmethod def name(): """ @return: name of the module.""" return EgressProcessorTests.MODULE_NAME def __init__(self): super(EgressProcessorTests, self).__init__(self.MODULE_NAME, self.PRIORITY) def initialize(self, conf_reader, msgQlist, product): """initialize configuration reader and internal msg queues""" # Initialize ScheduledMonitorThread super(EgressProcessorTests, self).initialize(conf_reader) # Initialize internal message queues for this module super(EgressProcessorTests, self).initialize_msgQ(msgQlist) # Flag denoting that a shutdown message has been placed # into our message queue from the main sspl_ll_d handler self._request_shutdown = False self._msg_sent_succesfull = True self._product = product # Configure messaging Exchange to transmit messages self._connection = None self._read_config() self.create_MsgProducer_obj() producer_initialized.set() def create_MsgProducer_obj(self): self._producer = None try: self._producer = MessageProducer(producer_id=self._producer_id, message_type=self._message_type, method=self._method) except Exception as err: logger.error( 'Instance creation for MessageProducer class failed due to %s' % err) def run(self): """Run the module periodically on its own thread. """ self._log_debug("Start accepting requests") # self._set_debug(True) # self._set_debug_persist(True) try: # Loop thru all messages in queue until and transmit while not self._is_my_msgQ_empty(): # Only get a new msg if we've successfully processed the current one if self._msg_sent_succesfull: self._jsonMsg = self._read_my_msgQ() if self._jsonMsg is not None: self._transmit_msg_on_exchange() except Exception: # Log it and restart the whole process when a failure occurs logger.exception("EgressProcessorTests restarting") # Configure messaging Exchange to receive messages self._get_connection() self._get_ack_connection() self._log_debug("Finished processing successfully") # Shutdown is requested by the sspl_ll_d shutdown handler # placing a 'shutdown' msg into our queue which allows us to # finish processing any other queued up messages. if self._request_shutdown is True: self.shutdown() else: self._scheduler.enter(1, self._priority, self.run, ()) def _read_config(self): """Configure the messaging exchange with defaults available.""" try: self._signature_user = Conf.get( SSPL_TEST_CONF, f"{self.PROCESSOR}>{self.SIGNATURE_USERNAME}", 'sspl-ll') self._signature_token = Conf.get( SSPL_TEST_CONF, f"{self.PROCESSOR}>{self.SIGNATURE_TOKEN}", 'FAKETOKEN1234') self._signature_expires = Conf.get( SSPL_TEST_CONF, f"{self.PROCESSOR}>{self.SIGNATURE_EXPIRES}", "3600") self._producer_id = Conf.get( SSPL_TEST_CONF, f"{self.PROCESSOR}>{self.PRODUCER_ID}", "sspl-sensor") self._message_type = Conf.get( SSPL_TEST_CONF, f"{self.PROCESSOR}>{self.MESSAGE_TYPE}", "Alerts") self._method = Conf.get(SSPL_TEST_CONF, f"{self.PROCESSOR}>{self.METHOD}", "Sync") except Exception as ex: logger.error("EgressProcessorTests, _read_config: %r" % ex) def _add_signature(self): """Adds the authentication signature to the message""" self._log_debug("_add_signature, jsonMsg: %s" % self._jsonMsg) self._jsonMsg["username"] = self._signature_user self._jsonMsg["expires"] = int(self._signature_expires) self._jsonMsg["time"] = str(int(time.time())) if use_security_lib: authn_token_len = len(self._signature_token) + 1 session_length = int(self._signature_expires) token = ctypes.create_string_buffer( SSPL_SEC.sspl_get_token_length()) SSPL_SEC.sspl_generate_session_token(self._signature_user, authn_token_len, self._signature_token, session_length, token) # Generate the signature msg_len = len(self._jsonMsg) + 1 sig = ctypes.create_string_buffer(SSPL_SEC.sspl_get_sig_length()) SSPL_SEC.sspl_sign_message(msg_len, str(self._jsonMsg), self._signature_user, token, sig) self._jsonMsg["signature"] = str(sig.raw) else: self._jsonMsg["signature"] = "SecurityLibNotInstalled" def _transmit_msg_on_exchange(self): """Transmit json message onto messaging exchange.""" self._log_debug("_transmit_msg_on_exchange, jsonMsg: %s" % self._jsonMsg) try: # Check for shut down message from sspl_ll_d and set a flag to shutdown # once our message queue is empty if self._jsonMsg.get("message").get( "actuator_response_type") is not None and \ self._jsonMsg.get("message").get( "actuator_response_type").get( "thread_controller") is not None and \ self._jsonMsg.get("message").get( "actuator_response_type").get("thread_controller").get( "thread_response") == \ "SSPL-LL is shutting down": logger.info( "EgressProcessorTests, _transmit_msg_on_exchange, received" "global shutdown message from sspl_ll_d") self._request_shutdown = True # Publish json message to the correct channel # NOTE: We need to route ThreadController messages to ACK channel. # We can't modify schema as it will affect other modules too. As a # temporary solution we have added a extra check to see if actuator_response_type # is "thread_controller". # TODO: Find a proper way to solve this issue. Avoid changing # core egress processor code if self._jsonMsg.get("message").get( "actuator_response_type") is not None and \ (self._jsonMsg.get("message").get( "actuator_response_type").get("ack") is not None or self._jsonMsg.get("message").get( "actuator_response_type").get( "thread_controller") is not None): self._add_signature() jsonMsg = json.dumps(self._jsonMsg) if isinstance(self._producer, MessageProducer): self._producer.send([json.dumps(self._jsonMsg)]) else: self.create_MsgProducer_obj() elif self._jsonMsg.get("message") is not None: message = self._jsonMsg.get("message") if message.get("actuator_request_type") or \ message.get("sensor_request_type") is not None: logger.info("inside egress, test actuator") self._add_signature() jsonMsg = json.dumps(self._jsonMsg) if isinstance(self._producer, MessageProducer): self._producer.send([jsonMsg]) else: self.create_MsgProducer_obj() else: self._add_signature() jsonMsg = json.dumps(self._jsonMsg) self._producer.send([jsonMsg]) # No exceptions thrown so success self._log_debug( "_transmit_msg_on_exchange, Successfully Sent: %s" % self._jsonMsg) self._msg_sent_succesfull = True except Exception as ex: logger.exception( "EgressProcessorTests, _transmit_msg_on_exchange: %r" % ex) self._msg_sent_succesfull = False def shutdown(self): """Clean up scheduler queue and gracefully shutdown thread""" super(EgressProcessorTests, self).shutdown()
def send_response(msg): try: admin = MessageBusAdmin(admin_id="admin") admin.register_message_type(message_types=["alerts"], partitions=1) print("sending response now") except MessageBusError as e: print("\n\n\n\n" + e.desc + "\n\n\n\n") if "ALREADY_EXISTS" not in e.desc: raise e # Get uuid from request and replace in response success_dict = { "title": "SSPL Actuator Response", "description": "Seagate Storage Platform Library - Actuator Response", "username": "******", "signature": "None", "time": "1628060172", "expires": 3600, "message": { "sspl_ll_msg_header": { "schema_version": "1.0.0", "sspl_version": "2.0.0", "msg_version": "1.0.0", "uuid": "16476007-a739-4785-b5c7-f3de189cdf9d" }, "actuator_response_type": { "host_id": "ssc-vm-rhev4-0180.colo.seagate.com", "alert_type": "control:shutdown", "alert_id": "16280601712397710d3c4f40d38e1a77cee3a4c097", "severity": "informational", "info": { "resource_type": "enclosure:fru:controller", "resource_id": "*", "event_time": "1628060171", "site_id": "DC01", "node_id": "SN01", "rack_id": "RC01", "cluster_id": "CC01" }, "specific_info": { "message": "Shutdown request Successful", "command": "shutdown both" } } } } #fail_dict = {"title": "SSPL Actuator Response","description": "Seagate Storage Platform Library - Actuator Response", "username": "******","signature": "None","time": "1628060172","expires": 3600,"message": {"sspl_ll_msg_header": {"schema_version": "1.0.0","sspl_version": "2.0.0","msg_version": "1.0.0","uuid": "16476007-a739-4785-b5c7-f3de189cdf9d"},"actuator_response_type": {"host_id": "ssc-vm-rhev4-0180.colo.seagate.com","alert_type": "control:shutdown","alert_id": "16280601712397710d3c4f40d38e1a77cee3a4c097","severity": "warning","info": {"resource_type": "enclosure:fru:controller","resource_id": "*","event_time": "1628060171","site_id": "DC01","node_id": "SN01","rack_id": "RC01","cluster_id": "CC01"},"specific_info": {"message": "Shutdown request Failed","command": "shutdown both"}}}} req_json = json.loads(msg) # Send success /fail resp based on test reqmt resp = success_dict #resp = fail_dict resp["message"]["sspl_ll_msg_header"]["uuid"] = req_json["message"][ "sspl_ll_msg_header"]["uuid"] print(f"uuid is {resp['message']['sspl_ll_msg_header']['uuid']}") resp_msg = [json.dumps(resp)] producer = MessageProducer(producer_id="sspl-sensor", message_type="alerts", method="sync") producer.send(resp_msg) print(f"Response is sent: \n {resp_msg} \n")
class EgressAccumulatedMsgsProcessor(ScheduledModuleThread, InternalMsgQ): """Send any unsent message to message bus.""" SENSOR_NAME = "EgressAccumulatedMsgsProcessor" PRIORITY = 1 # TODO: read egress config from common place # Section and keys in configuration file # Section and keys in configuration file PROCESSOR = 'EgressProcessor' SIGNATURE_USERNAME = '******' SIGNATURE_TOKEN = 'message_signature_token' SIGNATURE_EXPIRES = 'message_signature_expires' IEM_ROUTE_ADDR = 'iem_route_addr' PRODUCER_ID = 'producer_id' MESSAGE_TYPE = 'message_type' METHOD = 'method' # 300 seconds for 5 mins MSG_TIMEOUT = 300 @staticmethod def name(): """@return: name of the monitoring module.""" return EgressAccumulatedMsgsProcessor.SENSOR_NAME def __init__(self): super(EgressAccumulatedMsgsProcessor, self).__init__(self.SENSOR_NAME, self.PRIORITY) def initialize(self, conf_reader, msgQlist, products): """initialize configuration reader and internal msg queues""" # Initialize ScheduledMonitorThread super(EgressAccumulatedMsgsProcessor, self).initialize(conf_reader) super(EgressAccumulatedMsgsProcessor, self).initialize_msgQ(msgQlist) self.store_queue = StoreQueue() self._read_config() producer_initialized.wait() self._producer = MessageProducer(producer_id="acuumulated processor", message_type=self._message_type, method=self._method) def read_data(self): """This method is part of interface. Currently it is not in use. """ return {} def run(self): """Run the sensor on its own thread""" logger.debug("Consul accumulated messages processing started") if not self._is_my_msgQ_empty(): # Check for shut down message from sspl_ll_d and set a flag to shutdown # once our message queue is empty self._jsonMsg, _ = self._read_my_msgQ() if self._jsonMsg.get("message").get( "actuator_response_type") is not None and \ self._jsonMsg.get("message").get( "actuator_response_type").get( "thread_controller") is not None and \ self._jsonMsg.get("message").get( "actuator_response_type").get("thread_controller").get( "thread_response") == \ "SSPL-LL is shutting down": logger.info("EgressAccumulatedMsgsProcessor, run, received" "global shutdown message from sspl_ll_d") self.shutdown() try: # TODO : Fix accumulated message processor when message bus changes are available to # error out in case of failure (EOS-17626) if not self.store_queue.is_empty(): logger.debug( "Found accumulated messages, trying to send again") while not self.store_queue.is_empty(): message = self.store_queue.get() dict_msg = json.loads(message) if "actuator_response_type" in dict_msg["message"]: event_time = dict_msg["message"] \ ["actuator_response_type"]["info"]["event_time"] time_diff = int(time.time()) - int(event_time) if time_diff > self.MSG_TIMEOUT: continue if "sensor_response_type" in dict_msg["message"]: logger.info(f"Publishing Accumulated Alert: {message}") self._producer.send([message]) except MessageBusError as e: logger.error("EgressAccumulatedMsgsProcessor, run, %r" % e) except Exception as e: logger.error(e) finally: logger.debug("Consul accumulated processing ended") self._scheduler.enter(30, self._priority, self.run, ()) def _read_config(self): """Read config for messaging bus.""" try: self._signature_user = Conf.get( SSPL_CONF, f"{self.PROCESSOR}>{self.SIGNATURE_USERNAME}", 'sspl-ll') self._signature_token = Conf.get( SSPL_CONF, f"{self.PROCESSOR}>{self.SIGNATURE_TOKEN}", 'FAKETOKEN1234') self._signature_expires = Conf.get( SSPL_CONF, f"{self.PROCESSOR}>{self.SIGNATURE_EXPIRES}", "3600") self._producer_id = Conf.get( SSPL_CONF, f"{self.PROCESSOR}>{self.PRODUCER_ID}", "sspl-sensor") self._message_type = Conf.get( SSPL_CONF, f"{self.PROCESSOR}>{self.MESSAGE_TYPE}", "alerts") self._method = Conf.get(SSPL_CONF, f"{self.PROCESSOR}>{self.METHOD}", "sync") except Exception as ex: logger.error("EgressProcessor, _read_config: %r" % ex) def shutdown(self): """Clean up scheduler queue and gracefully shutdown thread""" super(EgressAccumulatedMsgsProcessor, self).shutdown() self._connection.cleanup()