def when_i_send_the_enclosure_sas_port_message_to_request_the_current_actuator_type_data(step, resource_type): egressMsg = { "username": "******", "description": "Seagate Storage Platform Library - Low Level - Actuator Request", "title": "SSPL-LL Actuator Request", "expires": 3600, "signature": "None", "time": "2019-11-21 08:37:27.144640", "message": { "sspl_ll_debug": { "debug_component": "sensor", "debug_enabled": True }, "response_dest": {}, "sspl_ll_msg_header": { "msg_version": "1.0.0", "uuid": "2ba55744-8218-40c2-8c2c-ea7bddf79c09", "schema_version": "1.0.0", "sspl_version": "1.0.0" }, "actuator_request_type": { "storage_enclosure": { "enclosure_request": "ENCL: enclosure:interface:sas", "resource": "Expansion Port" } } } } world.sspl_modules[RabbitMQegressProcessor.name()]._write_internal_msgQ(RabbitMQegressProcessor.name(), egressMsg)
def then_i_get_the_stop_successful_json_response_message(step): """I get the JSON response msg with 'thread_response': 'Stop Successful' key value""" time.sleep(2) module_name = None #import pdb #pdb.set_trace() while not world.sspl_modules[RabbitMQegressProcessor.name()]._is_my_msgQ_empty(): ingressMsg = world.sspl_modules[RabbitMQegressProcessor.name()]._read_my_msgQ() time.sleep(10) print("Received: %s" % ingressMsg) try: # Verify module name and thread response module_name = ingressMsg.get("actuator_response_type").get("thread_controller").get("module_name") time.sleep(5) #name = module_name["RAIDsensor"] #print("module_name: %s" % name) thread_response = ingressMsg.get("actuator_response_type").get("thread_controller").get("thread_response") #response = module_name["Stop Successful"] time.sleep(4) #print("thread_response: %s" % response) break except Exception as exception: time.sleep(2) print(exception) assert(module_name is not None) assert(thread_response is not None)
def given_i_send_in_the_actuator_message_to_restart_raid_sensor(step): # Clear the message queue buffer out while not world.sspl_modules[RabbitMQingressProcessorTests.name()]._is_my_msgQ_empty(): world.sspl_modules[RabbitMQingressProcessorTests.name()]._read_my_msgQ() egressMsg = { "title": "SSPL Actuator Request", "description": "Seagate Storage Platform Library - Actuator Request", "username" : "JohnDoe", "signature" : "None", "time" : "2015-05-29 14:28:30.974749", "expires" : 500, "message" : { "sspl_ll_msg_header": { "schema_version": "1.0.0", "sspl_version": "1.0.0", "msg_version": "1.0.0" }, "actuator_request_type": { "thread_controller": { "module_name" : "RAIDsensor", "thread_request": "restart" } } } } world.sspl_modules[RabbitMQegressProcessor.name()]._write_internal_msgQ(RabbitMQegressProcessor.name(), egressMsg)
def when_i_send_in_the_actuator_message_to_action_the_service( step, action, service): egressMsg = { "title": "SSPL Actuator Request", "description": "Seagate Storage Platform Library - Actuator Request", "username": "******", "signature": "None", "time": "2015-05-29 14:28:30.974749", "expires": 500, "message": { "sspl_ll_msg_header": { "schema_version": "1.0.0", "sspl_version": "1.0.0", "msg_version": "1.0.0" }, "actuator_request_type": { "service_controller": { "service_name": service, "service_request": action } } } } world.sspl_modules[RabbitMQegressProcessor.name()]._write_internal_msgQ( RabbitMQegressProcessor.name(), egressMsg)
def when_i_send_in_the_controller_sensor_message_to_request_the_current_sensor_type_data( step, resource_type): egressMsg = { "title": "SSPL Actuator Request", "description": "Seagate Storage Platform Library - Actuator Request", "username": "******", "signature": "None", "time": "2015-05-29 14:28:30.974749", "expires": 500, "message": { "sspl_ll_msg_header": { "schema_version": "1.0.0", "sspl_version": "1.0.0", "msg_version": "1.0.0" }, "sspl_ll_debug": { "debug_component": "sensor", "debug_enabled": True }, "sensor_request_type": { "enclosure_alert": { "info": { "resource_type": resource_type } } } } } world.sspl_modules[RabbitMQegressProcessor.name()]._write_internal_msgQ( RabbitMQegressProcessor.name(), egressMsg)
def given_i_request_to_start_raid_sensor_and_then_i_request_a_thread_status(step): egressMsg = { "title": "SSPL Actuator Request", "description": "Seagate Storage Platform Library - Actuator Request", "username" : "JohnDoe", "signature" : "None", "time" : "2015-05-29 14:28:30.974749", "expires" : 500, "message" : { "sspl_ll_msg_header": { "schema_version": "1.0.0", "sspl_version": "1.0.0", "msg_version": "1.0.0" }, "actuator_request_type": { "thread_controller": { "module_name" : "RAIDsensor", "thread_request": "start" } } } } world.sspl_modules[RabbitMQegressProcessor.name()]._write_internal_msgQ(RabbitMQegressProcessor.name(), egressMsg) # Request the status for the stopped thread egressMsg = { "title": "SSPL Actuator Request", "description": "Seagate Storage Platform Library - Actuator Request", "username" : "JohnDoe", "signature" : "None", "time" : "2015-05-29 14:28:30.974749", "expires" : 500, "message" : { "sspl_ll_msg_header": { "schema_version": "1.0.0", "sspl_version": "1.0.0", "msg_version": "1.0.0" }, "actuator_request_type": { "thread_controller": { "module_name" : "RAIDsensor", "thread_request": "status" } } } } world.sspl_modules[RabbitMQegressProcessor.name()]._write_internal_msgQ(RabbitMQegressProcessor.name(), egressMsg)
def _execute_raid_request(self, node_request, actuator_instance, json_msg, uuid): """Performs a RAID request by calling perform_request method of a RAID actuator. """ # Perform the RAID request on the node and get the response raid_response = actuator_instance.perform_request(json_msg).strip() self._log_debug(f"_process_msg, raid_response: {raid_response}") json_msg = AckResponseMsg(node_request, raid_response, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) # Restart openhpid to update HPI data only if it is a H/W environment if self.setup in ["hw", "ssu"]: self._log_debug("restarting openhpid service to update HPI data") if "assemble" in json_msg.get("actuator_request_type").get( "node_controller").get("node_request").lower(): internal_json_msg = json.dumps({ "actuator_request_type": { "service_controller": { "service_name": "openhpid.service", "service_request": "restart" } } }) self._write_internal_msgQ(ServiceMsgHandler.name(), internal_json_msg)
def _send_to_msg_handler(self, msgType, message, uuid): # Hand off to appropriate actuator message handler if msgType.get("logging") is not None: self._write_internal_msgQ("LoggingMsgHandler", message) elif msgType.get("thread_controller") is not None: self._write_internal_msgQ("ThreadController", message) elif msgType.get("service_controller") is not None: self._write_internal_msgQ("ServiceMsgHandler", message) elif msgType.get("node_controller") is not None: self._write_internal_msgQ("NodeControllerMsgHandler", message) elif msgType.get("storage_enclosure") is not None: self._write_internal_msgQ("RealStorActuatorMsgHandler", message) # Hand off to appropriate sensor message handler elif msgType.get("node_data") is not None: self._write_internal_msgQ("NodeDataMsgHandler", message) elif msgType.get("enclosure_alert") is not None: self._write_internal_msgQ("RealStorEnclMsgHandler", message) elif msgType.get("storage_enclosure") is not None: self._write_internal_msgQ("RealStorActuatorMsgHandler", message) # ... handle other incoming messages that have been validated else: # Send ack about not finding a msg handler ack_msg = AckResponseMsg("Error Processing Message", "Message Handler Not Found", uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), ack_msg)
def _run_thread_capture_errors(curr_module, sspl_modules, msgQlist, conf_reader, product): """Run the given thread and log any errors that happen on it. Will stop all sspl_modules if one of them fails.""" try: # Each module is passed a reference list to message queues so it can transmit # internal messages to other modules as desired curr_module.start_thread(conf_reader, msgQlist, product) except BaseException as ex: logger.critical( "SSPL-LL encountered a fatal error, terminating service Error: %s" % ex) logger.exception(ex) # Populate an actuator response message and transmit back to HAlon error_msg = "SSPL-LL encountered an error, terminating service Error: " + \ ", Exception: " + logger.exception(ex) json_msg = ThreadControllerMsg(curr_module.name(), error_msg).getJson() if product.lower() in [x.lower() for x in enabled_products]: self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) elif product.lower() in [x.lower() for x in cs_legacy_products]: self._write_internal_msgQ(PlaneCntrlRMQegressProcessor.name(), json_msg) # Shut it down, error is non-recoverable for name, other_module in list(sspl_modules.items()): if other_module is not curr_module: other_module.shutdown()
def run(self): """Run the module periodically on its own thread.""" if (self._product.lower() in [x.lower() for x in enabled_products]) and \ not self._threads_initialized: if self._product.lower() in [x.lower() for x in cs_products]: # Wait for the dcs-collector to populate the /tmp/dcs/hpi directory while not os.path.isdir(self._hpi_base_dir): logger.info("ThreadController, dir not found: %s " % self._hpi_base_dir) logger.info("ThreadController, rechecking in %s secs" % self._start_delay) time.sleep(int(self._start_delay)) logger.debug("ThreadController._sspl_modules is {}".format( self._sspl_modules)) # Allow other threads to initialize continue_waiting = False for (n, m) in self._sspl_modules.items(): if not isinstance(m, SensorThread): continue thread_init_status = m.get_thread_init_status() logger.debug("Thread status for {} is {}".format( m.__class__, thread_init_status)) if thread_init_status == SensorThreadState.FAILED: m.shutdown() elif thread_init_status == SensorThreadState.WAITING: continue_waiting = True if continue_waiting: self._scheduler.enter(10, self._priority, self.run, ()) return # Notify external applications that've started up successfully startup_msg = "SSPL-LL service has started successfully" json_msg = ThreadControllerMsg(ThreadController.name(), startup_msg).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) self._threads_initialized = True #self._set_debug(True) #self._set_debug_persist(True) self._log_debug("Start accepting requests") try: # Block on message queue until it contains an entry jsonMsg, _ = self._read_my_msgQ() if jsonMsg is not None: self._process_msg(jsonMsg) # Keep processing until the message queue is empty while not self._is_my_msgQ_empty(): jsonMsg, _ = self._read_my_msgQ() if jsonMsg is not None: self._process_msg(jsonMsg) except Exception as ex: # Log it and restart the whole process when a failure occurs logger.exception("ThreadController restarting: %r" % ex) self._scheduler.enter(1, self._priority, self.run, ()) self._log_debug("Finished processing successfully")
def _send_ifdata_json_msg(self, sensor_type, resource_id, resource_type, state, severity, event=""): """A resuable method for transmitting IFDataMsg to RMQ and IEM logging""" ifDataMsg = IFdataMsg(self._node_sensor.host_id, self._node_sensor.local_time, self._node_sensor.if_data, resource_id, resource_type, self.site_id, self.node_id, self.cluster_id, self.rack_id, state, severity, event) # Add in uuid if it was present in the json request if self._uuid is not None: ifDataMsg.set_uuid(self._uuid) jsonMsg = ifDataMsg.getJson() self.if_sensor_data = jsonMsg self.os_sensor_type[sensor_type] = self.if_sensor_data # Send the event to logging msg handler to send IEM message to journald #internal_json_msg=json.dumps({ # 'actuator_request_type': { # 'logging': { # 'log_level': 'LOG_WARNING', # 'log_type': 'IEM', # 'log_msg': '{}'.format(jsonMsg)}}}) #self._write_internal_msgQ(LoggingMsgHandler.name(), internal_json_msg) # Transmit it out over rabbitMQ channel self._write_internal_msgQ(RabbitMQegressProcessor.name(), jsonMsg)
def _generate_local_mount_data(self): """Create & transmit a local_mount_data message as defined by the sensor response json schema""" # Notify the node sensor to update its data required for the local_mount_data message successful = self._node_sensor.read_data("local_mount_data", self._get_debug(), self._units) if not successful: logger.error( "NodeDataMsgHandler, _generate_local_mount_data was NOT successful." ) # Create the local mount data message and hand it over to the egress processor to transmit localMountDataMsg = LocalMountDataMsg( self._node_sensor.host_id, self._epoch_time, self._node_sensor.free_space, self._node_sensor.free_inodes, self._node_sensor.free_swap, self._node_sensor.total_space, self._node_sensor.total_swap, self._units) # Add in uuid if it was present in the json request if self._uuid is not None: localMountDataMsg.set_uuid(self._uuid) jsonMsg = localMountDataMsg.getJson() # Transmit it out over rabbitMQ channel self._write_internal_msgQ(RabbitMQegressProcessor.name(), jsonMsg)
def check_RabbitMQegressProcessor_is_running(self): """Used by the shutdown_handler to allow queued egress msgs to complete""" if self._product.lower() in [x.lower() for x in enabled_products]: return self._sspl_modules[ PlaneCntrlRMQegressProcessor.name()].is_running() elif self._product.lower() in [x.lower() for x in cs_legacy_products]: return self._sspl_modules[ RabbitMQegressProcessor.name()].is_running()
def _process_msg(self, body): """Parses the incoming message and hands off to the appropriate module""" ingressMsg = {} uuid = None try: if isinstance(body, dict) is False: ingressMsg = json.loads(body) else: ingressMsg = body # Authenticate message using username and signature fields username = ingressMsg.get("username") signature = ingressMsg.get("signature") message = ingressMsg.get("message") uuid = ingressMsg.get("uuid") msg_len = len(message) + 1 if uuid is None: uuid = "N/A" if use_security_lib and \ SSPL_SEC.sspl_verify_message(msg_len, str(message), username, signature) != 0: logger.warn( "RabbitMQingressProcessor, Authentication failed on message: %s" % ingressMsg) return # Get the incoming message type if message.get("actuator_request_type") is not None: msgType = message.get("actuator_request_type") # Validate against the actuator schema validate(ingressMsg, self._actuator_schema) elif message.get("sensor_request_type") is not None: msgType = message.get("sensor_request_type") # Validate against the sensor schema validate(ingressMsg, self._sensor_schema) else: # We only handle incoming actuator and sensor requests, ignore # everything else. return # Check for debugging being activated in the message header self._check_debug(message) self._log_debug("_process_msg, ingressMsg: %s" % ingressMsg) self._send_to_msg_handler(msgType, message, uuid) except Exception as ex: logger.error( "RabbitMQingressProcessor, _process_msg unrecognized message: %r" % ingressMsg) ack_msg = AckResponseMsg("Error Processing Msg", "Msg Handler Not Found", uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), ack_msg)
def _process_msg(self, jsonMsg): """Parses the incoming message and hands off to the appropriate logger""" self._log_debug(f"_process_msg, jsonMsg: {jsonMsg}") if isinstance(jsonMsg, dict) is False: jsonMsg = json.loads(jsonMsg) uuid = None if jsonMsg.get("sspl_ll_msg_header") is not None and \ jsonMsg.get("sspl_ll_msg_header").get("uuid") is not None: uuid = jsonMsg.get("sspl_ll_msg_header").get("uuid") self._log_debug(f"_process_msg, uuid: {uuid}") log_type = jsonMsg.get("actuator_request_type").get("logging").get("log_type") result = "N/A" # Disabled for LDR_R1 # if log_type == "IEM": # self._log_debug("_process_msg, msg_type: IEM") # if self._iem_log_locally == "true": # result = self._iem_logger.log_msg(jsonMsg) # self._log_debug(f"Log IEM results: {result}") if log_type == "HDS": # Retrieve the serial number of the drive self._log_debug("_process_msg, msg_type: HDS") log_msg = jsonMsg.get("actuator_request_type").get("logging").get("log_msg") # Parse out the json data section in the IEM and replace single quotes with double json_data = json.loads(str('{' + log_msg.split('{')[1]).replace("'", '"')) serial_number = json_data.get("serial_number") status = json_data.get("status") reason = json_data.get("reason") self._log_debug(f"_processMsg, serial_number: {serial_number}, status:{status}, reason: {reason}") # Send a message to the disk manager handler to create and transmit json msg internal_json_msg = json.dumps( {"sensor_response_type" : "disk_status_HDS", "object_path" : "HDS", "status" : status, "reason" : reason, "serial_number" : serial_number }) # Send the event to disk message handler to generate json message self._write_internal_msgQ("DiskMsgHandler", internal_json_msg) # Disabled for LDR_R1 # Hand off to the IEM logger # result = self._iem_logger.log_msg(jsonMsg) # Send ack about logging msg ack_msg = AckResponseMsg(log_type, result, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), ack_msg)
def when_i_send_in_the_enclosure_actuator_message_to_request_the_current_sensor_type_data(step, resource_type, resource_id): egressMsg = { "title": "SSPL Actuator Request", "description": "Seagate Storage Platform Library - Actuator Request", "username" : "JohnDoe", "signature" : "None", "time" : "2015-05-29 14:28:30.974749", "expires" : 500, "message" : { "sspl_ll_msg_header": { "schema_version": "1.0.0", "sspl_version": "1.0.0", "msg_version": "1.0.0" }, "sspl_ll_debug": { "debug_component" : "sensor", "debug_enabled" : True }, "response_dest": {}, "sspl_ll_msg_header": { "msg_version": "1.0.0", "uuid": "16476007-a739-4785-b5c7-f3de189cdf9d", "schema_version": "1.0.0", "sspl_version": "1.0.0" }, "request_path": { "site_id": 0, "node_id": 1, "rack_id": 0, "cluster_id": 1 }, "actuator_request_type": { "storage_enclosure": { "enclosure_request": resource_type, "resource": resource_id } } } } world.sspl_modules[RabbitMQegressProcessor.name()]._write_internal_msgQ(RabbitMQegressProcessor.name(), egressMsg)
def _generate_enclosure_alert(self, json_msg, host_name, alert_type, alert_id, severity, info, specific_info, sensor_type): """Parses the json message, also validates it and then send it to the RabbitMQ egress processor""" self._log_debug(f"RealStorEnclMsgHandler, _generate_enclosure_alert,\ json_msg {json_msg}") real_stor_encl_msg = RealStorEnclDataMsg(host_name, alert_type, alert_id, severity, info, specific_info) json_msg = real_stor_encl_msg.getJson() self._enclosure_message = json_msg self._fru_type[sensor_type] = self._enclosure_message self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg, self._event)
def _generate_psu_alert(self, json_msg, host_name, alert_type, alert_id, severity, info, specific_info, sensor_type): """Parses the json message, also validates it and then send it to the RabbitMQ egress processor""" self._log_debug(f"RealStorEnclMsgHandler, _generate_psu_alert,\ json_msg {json_msg}") real_stor_psu_data_msg = \ RealStorPSUDataMsg(host_name, alert_type, alert_id, severity, info, specific_info) json_msg = real_stor_psu_data_msg.getJson() # Saves the json message in memory to serve sspl CLI sensor request self._psu_sensor_message = json_msg self._fru_type[sensor_type] = self._psu_sensor_message self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg, self._event)
def _check_reset_all_modules(self, jsonMsg): """Restarts all modules with debug mode off. Activated by internal_msgQ""" if jsonMsg.get("sspl_ll_debug") is not None and \ jsonMsg.get("sspl_ll_debug").get("debug_component") is not None and \ jsonMsg.get("sspl_ll_debug").get("debug_component") == "all": for module in self._sspl_modules: self._log_debug("_check_reset_all_modules, module: %s" % module) # Don't restart this thread or it won't complete the loop if module != self.name(): self._restart_module(module) # Populate an actuator response message and transmit msgString = ThreadControllerMsg("All Modules", "Restarted with debug mode off").getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), msgString) return True return False
def _route_IEM(self, jsonMsg): # Send the IEM to the logging msg handler to be processed # Get the optional log_level if it exists in msg if jsonMsg.get("actuator_request_type").get("logging").get("log_level") is not None: log_level = jsonMsg.get("actuator_request_type").get("logging").get("log_level") else: log_level = "LOG_INFO" # Get the message to log in format "IEC: EVENT_CODE: EVENT_STRING: JSON DATA" log_msg = f"{log_level} {jsonMsg.get('actuator_request_type').get('logging').get('log_msg')}" internal_json_msg = json.dumps( {"message": { "IEM_routing": { "log_msg": log_msg } } }) # Send the IEM to RabbitMQegressProcessor to be routed to another IEM listener self._write_internal_msgQ(RabbitMQegressProcessor.name(), internal_json_msg)
def _generate_node_fru_data(self, jsonMsg): """Create & transmit a FRU IPMI data message as defined by the sensor response json schema""" if self._node_sensor.host_id is None: successful = self._node_sensor.read_data("None", self._get_debug(), self._units) if not successful: logger.error( "NodeDataMsgHandler, updating host information was NOT successful." ) if jsonMsg.get("sensor_request_type").get("node_data") is not None: self._fru_info = jsonMsg.get("sensor_request_type").get( "node_data") node_ipmi_data_msg = NodeIPMIDataMsg(self._fru_info) if self._uuid is not None: node_ipmi_data_msg.set_uuid(self._uuid) jsonMsg = node_ipmi_data_msg.getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), jsonMsg)
def _process_msg(self, jsonMsg): """Parses the incoming message and handles appropriately""" self._log_debug(f"RealStorActuatorMsgHandler, _process_msg, jsonMsg: {jsonMsg}") if isinstance(jsonMsg, dict) is False: jsonMsg = json.loads(jsonMsg) # Parse out the uuid so that it can be sent back in Ack message uuid = None if jsonMsg.get("sspl_ll_msg_header").get("uuid") is not None: uuid = jsonMsg.get("sspl_ll_msg_header").get("uuid") self._log_debug(f"_processMsg, uuid: {uuid}") logger.debug(f"RealStorActuatorMsgHandler: _process_msg: jsonMsg: {jsonMsg}") if jsonMsg.get("actuator_request_type").get("storage_enclosure").get("enclosure_request") is not None: enclosure_request = jsonMsg.get("actuator_request_type").get("storage_enclosure").get("enclosure_request") self._log_debug(f"_processMsg, enclosure_request: {enclosure_request}") logger.debug(f"RealStorActuatorMsgHandler: _process_msg: INSIDE: jsonMsg: {jsonMsg}") # Parse out the request field in the enclosure_request (request, fru) = enclosure_request.split(":", 1) request = request.strip() fru = fru.strip() if self._real_stor_actuator is None: try: from actuators.impl.generic.realstor_encl import RealStorActuator self._real_stor_actuator = RealStorActuator() except ImportError as e: logger.warn("RealStor Actuator not loaded") return # Perform the request and get the response real_stor_response = self._real_stor_actuator.perform_request(jsonMsg) self._log_debug(f"_process_msg, RealStor response: {real_stor_response}") json_msg = RealStorActuatorMsg(real_stor_response, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
def _process_msg(self, json_msg): """Parses the incoming message and generate the desired data message""" self._log_debug(f"RealStorEnclMsgHandler, _process_msg, json_msg: {json_msg}") if json_msg.get("sensor_request_type").get("enclosure_alert") is not None: internal_sensor_request = json_msg.get("sensor_request_type").\ get("enclosure_alert").get("status") if internal_sensor_request: resource_type = json_msg.get("sensor_request_type").\ get("enclosure_alert").get("info").get("resource_type") if ":" in resource_type: sensor_type = resource_type.split(":")[2] else: sensor_type = resource_type self._propagate_alert(json_msg, sensor_type) else: # serves the request coming from sspl CLI sensor_type = json_msg.get("sensor_request_type").\ get("enclosure_alert").get("info").\ get("resource_type") if ":" in sensor_type: sensor_type = sensor_type.split(":")[2] else: sensor_type = sensor_type sensor_message_type = self._fru_type.get(sensor_type, "") # get the previously saved json message for the sensor type # and send the RabbitMQ Message if sensor_message_type: self._write_internal_msgQ(RabbitMQegressProcessor.name(), sensor_message_type, self._event) else: self._log_debug(f"RealStorEnclMsgHandler, _process_msg, \ No past data found for {sensor_type} sensor type") else: logger.exception("RealStorEnclMsgHandler, _process_msg,\ Not a valid sensor request format")
def _transmit_json_msg(self, json_data): """Transmit message to halon by passing it to egress msg handler""" json_data["trapName"] = self._trap_name json_msg = SNMPtrapMsg(json_data).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
def _send_msg(self, iem_components, log_timestamp): """Creates JSON message from iem components and sends to RabbitMQ channel. """ # IEM format is IEC:DESCRIPTION # IEC format is SEVERITY|SOURCEID|COMPONENTID|MODULEID|EVENTID # Field lengths ----1---|---1----|------3----|----3---|---4--- # Example IEM -> "IEC: BO1001000001:Error in connecting to controller" # Actual IEC doesn't contain separator between fields. It is shown # here just for readability. Each field has fixed length. severity, source_id, component_id, module_id, event_id, description = \ [iem_components[i] for i in range(6)] # Check if severity level is valid if severity not in self.SEVERITY_LEVELS: logger.warn(f"Invalid Severity level: {severity}") return # Check for valid source id if source_id not in self.SOURCE_IDS: logger.warn(f"Invalid Source ID level: {source_id}") return # Check for valid event time event_time = self._get_epoch_time_from_timestamp(log_timestamp) if not event_time: logger.error("Timestamp is not in required format, discarding the message") return # Check for other components args = { "_comp_id": component_id, "_module_id": module_id, "_event_id": event_id } if not self._are_components_in_range(**args): return # Update severity and source_id alert_type = iem_severity_to_alert_mapping.get(severity) severity = iem_severity_types.get(severity, severity) source_id = iem_source_types.get(source_id, source_id) # Decode component_id, module_id and event_id component_id, module_id, event_id = self._decode_msg( f"{component_id}{module_id}{event_id}") info = { "site_id": self._site_id, "rack_id": self._rack_id, "node_id": self._node_id, "cluster_id" : self._cluster_id, "source_id": source_id, "component_id": component_id, "module_id": module_id, "event_id": event_id, "severity": severity, "description": description, "alert_type": alert_type, "event_time": event_time, "IEC": "".join(iem_components[:-1]) } iem_data_msg = IEMDataMsg(info) json_msg = iem_data_msg.getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
def _generate_host_update(self): """Create & transmit a host update message as defined by the sensor response json schema""" # Notify the node sensor to update its data required for the host_update message successful = self._node_sensor.read_data("host_update", self._get_debug(), self._units) if not successful: logger.error( "NodeDataMsgHandler, _generate_host_update was NOT successful." ) self._host_memory_usage_threshold = str( self._host_memory_usage_threshold) try: if self._host_memory_usage_threshold.isdigit(): self._host_memory_usage_threshold = int( self._host_memory_usage_threshold) else: self._host_memory_usage_threshold = float( self._host_memory_usage_threshold) except ValueError: logger.warning( "Host Memory Alert, Invalid host_memory_usage_threshold value are entered in config." ) # Assigning default value to _disk_usage_threshold self._host_memory_usage_threshold = self.DEFAULT_HOST_MEMORY_USAGE_THRESHOLD if self._node_sensor.total_memory[ "percent"] >= self._host_memory_usage_threshold: # Create the disk space data message and hand it over to the egress processor to transmit if not self.host_fault: self.host_fault = True # Create the disk space data message and hand it over to the egress processor to transmit fault_event = "Host memory usage increased to %s, beyond configured threshold of %s" \ %(self._node_sensor.total_memory["percent"], self._host_memory_usage_threshold) logger.warning(fault_event) logged_in_users = [] # Create the host update message and hand it over to the egress processor to transmit hostUpdateMsg = HostUpdateMsg( self._node_sensor.host_id, self._epoch_time, self._node_sensor.boot_time, self._node_sensor.up_time, self._node_sensor.uname, self._units, self.site_id, self.rack_id, self.node_id, self.cluster_id, self._node_sensor.total_memory, self._node_sensor.logged_in_users, self._node_sensor.process_count, self._node_sensor.running_process_count, self.FAULT, fault_event) # Add in uuid if it was present in the json request if self._uuid is not None: hostUpdateMsg.set_uuid(self._uuid) jsonMsg = hostUpdateMsg.getJson() # Transmit it out over rabbitMQ channel self.host_sensor_data = jsonMsg self.os_sensor_type["memory_usage"] = self.host_sensor_data self._write_internal_msgQ(RabbitMQegressProcessor.name(), jsonMsg) if (self._node_sensor.total_memory["percent"] < self._host_memory_usage_threshold) and (self.host_fault == True): fault_resolved_event = "Host memory usage decreased to %s, lesser than configured threshold of %s" \ %(self._node_sensor.total_memory["percent"], self._host_memory_usage_threshold) logger.warning(fault_resolved_event) logged_in_users = [] # Create the host update message and hand it over to the egress processor to transmit hostUpdateMsg = HostUpdateMsg( self._node_sensor.host_id, self._epoch_time, self._node_sensor.boot_time, self._node_sensor.up_time, self._node_sensor.uname, self._units, self.site_id, self.rack_id, self.node_id, self.cluster_id, self._node_sensor.total_memory, self._node_sensor.logged_in_users, self._node_sensor.process_count, self._node_sensor.running_process_count, self.FAULT_RESOLVED, fault_resolved_event) # Add in uuid if it was present in the json request if self._uuid is not None: hostUpdateMsg.set_uuid(self._uuid) jsonMsg = hostUpdateMsg.getJson() # Transmit it out over rabbitMQ channel self.host_sensor_data = jsonMsg self.os_sensor_type["memory_usage"] = self.host_sensor_data self._write_internal_msgQ(RabbitMQegressProcessor.name(), jsonMsg) self.host_fault = False
def _generate_cpu_data(self): """Create & transmit a cpu_data message as defined by the sensor response json schema""" # Notify the node sensor to update its data required for the cpu_data message successful = self._node_sensor.read_data("cpu_data", self._get_debug()) if not successful: logger.error( "NodeDataMsgHandler, _generate_cpu_data was NOT successful.") self._cpu_usage_threshold = str(self._cpu_usage_threshold) try: if self._cpu_usage_threshold.isdigit(): self._cpu_usage_threshold = int(self._cpu_usage_threshold) else: self._cpu_usage_threshold = float(self._cpu_usage_threshold) except ValueError: logger.warning( "CPU Usage Alert, Invalid host_memory_usage_threshold value are entered in config." ) # Assigning default value to _cpu_usage_threshold self._cpu_usage_threshold = self.DEFAULT_CPU_USAGE_THRESHOLD if self._node_sensor.cpu_usage >= self._cpu_usage_threshold: if not self.cpu_fault: self.cpu_fault = True # Create the cpu usage data message and hand it over to the egress processor to transmit fault_event = "CPU usage increased to %s, beyond configured threshold of %s" \ %(self._node_sensor.cpu_usage, self._cpu_usage_threshold) logger.warning(fault_event) # Create the local mount data message and hand it over to the egress processor to transmit cpuDataMsg = CPUdataMsg( self._node_sensor.host_id, self._epoch_time, self._node_sensor.csps, self._node_sensor.idle_time, self._node_sensor.interrupt_time, self._node_sensor.iowait_time, self._node_sensor.nice_time, self._node_sensor.softirq_time, self._node_sensor.steal_time, self._node_sensor.system_time, self._node_sensor.user_time, self._node_sensor.cpu_core_data, self._node_sensor.cpu_usage, self.site_id, self.rack_id, self.node_id, self.cluster_id, self.FAULT, fault_event) # Add in uuid if it was present in the json request if self._uuid is not None: cpuDataMsg.set_uuid(self._uuid) jsonMsg = cpuDataMsg.getJson() self.cpu_sensor_data = jsonMsg self.os_sensor_type["cpu_usage"] = self.cpu_sensor_data # Transmit it out over rabbitMQ channel self._write_internal_msgQ(RabbitMQegressProcessor.name(), jsonMsg) if (self._node_sensor.cpu_usage <= self._cpu_usage_threshold) and (self.cpu_fault == True): # Create the cpu usage data message and hand it over to the egress processor to transmit fault_resolved_event = "CPU usage decreased to %s, lesser than configured threshold of %s" \ %(self._node_sensor.cpu_usage, self._cpu_usage_threshold) logger.warning(fault_resolved_event) # Create the local mount data message and hand it over to the egress processor to transmit cpuDataMsg = CPUdataMsg( self._node_sensor.host_id, self._epoch_time, self._node_sensor.csps, self._node_sensor.idle_time, self._node_sensor.interrupt_time, self._node_sensor.iowait_time, self._node_sensor.nice_time, self._node_sensor.softirq_time, self._node_sensor.steal_time, self._node_sensor.system_time, self._node_sensor.user_time, self._node_sensor.cpu_core_data, self._node_sensor.cpu_usage, self.site_id, self.rack_id, self.node_id, self.cluster_id, self.FAULT_RESOLVED, fault_resolved_event) # Add in uuid if it was present in the json request if self._uuid is not None: cpuDataMsg.set_uuid(self._uuid) jsonMsg = cpuDataMsg.getJson() self.cpu_sensor_data = jsonMsg self.os_sensor_type["cpu_usage"] = self.cpu_sensor_data # Transmit it out over rabbitMQ channel self._write_internal_msgQ(RabbitMQegressProcessor.name(), jsonMsg) self.cpu_fault = False
def _process_msg(self, jsonMsg): """Parses the incoming message and calls the appropriate method""" self._log_debug("_process_msg, jsonMsg: %s" % jsonMsg) # Check to see if debug mode is being globally turned off on all modules if self._check_reset_all_modules(jsonMsg) is True: return # Parse out the module name and request module_name = jsonMsg.get("actuator_request_type").get( "thread_controller").get("module_name") thread_request = jsonMsg.get("actuator_request_type").get( "thread_controller").get("thread_request") # Parse out the uuid so that it can be sent back in Ack message uuid = None if jsonMsg.get("sspl_ll_msg_header") is not None and \ jsonMsg.get("sspl_ll_msg_header").get("uuid") is not None: uuid = jsonMsg.get("sspl_ll_msg_header").get("uuid") self._log_debug("_processMsg, uuid: %s" % uuid) # Pass along the debug section to the module if jsonMsg.get("sspl_ll_debug") is not None: self.debug_section = {"sspl_ll_debug": {}} self.debug_section["sspl_ll_debug"] = jsonMsg.get("sspl_ll_debug") else: self.debug_section = None self._log_debug("_process_msg, self.debug_section: %s" % self.debug_section) # Parse out thread request and call the appropriate method if thread_request == "restart": self._restart_module(module_name) elif thread_request == "start": self._start_module(module_name) elif thread_request == "stop": # Don't let the outside world stop us from using RabbitMQ connection or shut down this thread if module_name == "RabbitMQegressProcessor" or \ module_name == "RabbitMQingressProcessor" or \ module_name == "ThreadController": logger.warn( "Attempt to stop RabbitMQ or ThreadController Processors, \ ignoring. Please try 'restart' instead.") return self._stop_module(module_name) elif thread_request == "status": self._status_module(module_name) elif thread_request == "degrade": if module_name.lower() != "all": logger.warn( "Invalid module_name {0}. Need 'all' in module_name". format(module_name)) return self._switch_to_degraded_state(self._sspl_modules) elif thread_request == "active": if module_name.lower() != "all": logger.warn( "Invalid module_name {0}. Need 'all' in module_name". format(module_name)) return self._switch_to_active_state(self._sspl_modules) else: self._thread_response = "Error, unrecognized thread request" node_id = [] if jsonMsg.get("actuator_request_type").get("thread_controller").get("parameters") is not None and \ jsonMsg.get("actuator_request_type").get("thread_controller").get("parameters").get("node_id"): node_id = jsonMsg.get("actuator_request_type").get( "thread_controller").get("parameters").get("node_id") ack_type = {} ack_type["hostname"] = self._hostname ack_type["node_id"] = node_id # Populate an actuator response message and transmit threadControllerMsg = ThreadControllerMsg(module_name, self._thread_response, \ json.dumps(ack_type)) if uuid is not None: threadControllerMsg.set_uuid(uuid) msgString = threadControllerMsg.getJson() logger.info("ThreadController, response: %s" % str(msgString)) if self._product.lower() in [x.lower() for x in enabled_products]: self._write_internal_msgQ(RabbitMQegressProcessor.name(), msgString) elif self._product.lower() in [x.lower() for x in cs_legacy_products]: self._write_internal_msgQ(PlaneCntrlRMQegressProcessor.name(), msgString)
def _generate_disk_space_alert(self): """Create & transmit a disk_space_alert message as defined by the sensor response json schema""" # Notify the node sensor to update its data required for the disk_space_data message successful = self._node_sensor.read_data("disk_space_alert", self._get_debug(), self._units) if not successful: logger.error( "NodeDataMsgHandler, _generate_disk_space_alert was NOT successful." ) return # Changing disk_usage_threshold type according to what value type entered in config file self._disk_usage_threshold = str(self._disk_usage_threshold) try: if self._disk_usage_threshold.isdigit(): self._disk_usage_threshold = int(self._disk_usage_threshold) else: self._disk_usage_threshold = float(self._disk_usage_threshold) except ValueError: logger.warning( "Disk Space Alert, Invalid disk_usage_threshold value are entered in config." ) # Assigning default value to _disk_usage_threshold self._disk_usage_threshold = self.DEFAULT_DISK_USAGE_THRESHOLD if self._node_sensor.disk_used_percentage >= self._disk_usage_threshold: if not self.disk_fault: self.disk_fault = True # Create the disk space data message and hand it over to the egress processor to transmit fault_event = "Disk usage increased to %s, beyond configured threshold of %s" \ %(self._node_sensor.disk_used_percentage, self._disk_usage_threshold) logger.warning(fault_event) diskSpaceAlertMsg = DiskSpaceAlertMsg( self._node_sensor.host_id, self._epoch_time, self._node_sensor.total_space, self._node_sensor.free_space, self._node_sensor.disk_used_percentage, self._units, self.site_id, self.rack_id, self.node_id, self.cluster_id, self.FAULT, fault_event) # Add in uuid if it was present in the json request if self._uuid is not None: diskSpaceAlertMsg.set_uuid(self._uuid) jsonMsg = diskSpaceAlertMsg.getJson() self.disk_sensor_data = jsonMsg self.os_sensor_type["disk_space"] = self.disk_sensor_data # Transmit it out over rabbitMQ channel self._write_internal_msgQ(RabbitMQegressProcessor.name(), jsonMsg) if (self._node_sensor.disk_used_percentage <= self._disk_usage_threshold) and (self.disk_fault == True): # Create the disk space data message and hand it over to the egress processor to transmit fault_resolved_event = "Disk usage decreased to %s, lesser than configured threshold of %s" \ %(self._node_sensor.disk_used_percentage, self._disk_usage_threshold) logger.warning(fault_resolved_event) diskSpaceAlertMsg = DiskSpaceAlertMsg( self._node_sensor.host_id, self._epoch_time, self._node_sensor.total_space, self._node_sensor.free_space, self._node_sensor.disk_used_percentage, self._units, self.site_id, self.rack_id, self.node_id, self.cluster_id, self.FAULT_RESOLVED, fault_resolved_event) # Add in uuid if it was present in the json request if self._uuid is not None: diskSpaceAlertMsg.set_uuid(self._uuid) jsonMsg = diskSpaceAlertMsg.getJson() self.disk_sensor_data = jsonMsg self.os_sensor_type["disk_space"] = self.disk_sensor_data # Transmit it out over rabbitMQ channel self._write_internal_msgQ(RabbitMQegressProcessor.name(), jsonMsg) self.disk_fault = False
def _process_msg(self, ch, method, properties, body): """Parses the incoming message and hands off to the appropriate module""" ingressMsg = {} uuid = None try: if isinstance(body, dict) is False: ingressMsg = json.loads(body) else: ingressMsg = body # Authenticate message using username and signature fields username = ingressMsg.get("username") signature = ingressMsg.get("signature") message = ingressMsg.get("message") uuid = ingressMsg.get("uuid") msg_len = len(message) + 1 if uuid is None: uuid = "N/A" if use_security_lib and \ SSPL_SEC.sspl_verify_message(msg_len, str(message), username, signature) != 0: logger.warn( "RabbitMQingressProcessor, Authentication failed on message: %s" % ingressMsg) return # Get the incoming message type if message.get("actuator_request_type") is not None: msgType = message.get("actuator_request_type") # Validate against the actuator schema validate(ingressMsg, self._actuator_schema) elif message.get("sensor_request_type") is not None: msgType = message.get("sensor_request_type") # Validate against the sensor schema validate(ingressMsg, self._sensor_schema) else: # We only handle incoming actuator and sensor requests, ignore # everything else. return # Check for debugging being activated in the message header self._check_debug(message) self._log_debug("_process_msg, ingressMsg: %s" % ingressMsg) # Hand off to appropriate actuator message handler if msgType.get("logging") is not None: self._write_internal_msgQ("LoggingMsgHandler", message) elif msgType.get("thread_controller") is not None: self._write_internal_msgQ("ThreadController", message) elif msgType.get("service_controller") is not None: self._write_internal_msgQ("ServiceMsgHandler", message) elif msgType.get("node_controller") is not None: self._write_internal_msgQ("NodeControllerMsgHandler", message) elif msgType.get("storage_enclosure") is not None: self._write_internal_msgQ("RealStorActuatorMsgHandler", message) # Hand off to appropriate sensor message handler elif msgType.get("node_data") is not None: self._write_internal_msgQ("NodeDataMsgHandler", message) elif msgType.get("enclosure_alert") is not None: self._write_internal_msgQ("RealStorEnclMsgHandler", message) elif msgType.get("storage_enclosure") is not None: self._write_internal_msgQ("RealStorActuatorMsgHandler", message) # ... handle other incoming messages that have been validated else: # Send ack about not finding a msg handler ack_msg = AckResponseMsg("Error Processing Message", "Message Handler Not Found", uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), ack_msg) # Acknowledge message was received self._connection.ack(ch, delivery_tag=method.delivery_tag) except Exception as ex: logger.error( "RabbitMQingressProcessor, _process_msg unrecognized message: %r" % ingressMsg) ack_msg = AckResponseMsg("Error Processing Msg", "Msg Handler Not Found", uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), ack_msg)