def encl_sensor_message_request(resource_type): egressMsg = { "title": "SSPL Actuator Request", "description": "Seagate Storage Platform Library - Actuator Request", "username": "******", "signature": "None", "time": "1576148751", "expires": 500, "message": { "sspl_ll_msg_header": { "schema_version": "1.0.0", "sspl_version": "1.0.0", "msg_version": "1.0.0" }, "sspl_ll_debug": { "debug_component": "sensor", "debug_enabled": True }, "sensor_request_type": { "enclosure_alert": { "info": { "resource_type": resource_type } } } } } world.sspl_modules[RabbitMQegressProcessor.name()]._write_internal_msgQ( RabbitMQegressProcessor.name(), egressMsg)
def node_data_sensor_message_request(sensor_type): egressMsg = { "title": "SSPL Actuator Request", "description": "Seagate Storage Platform Library - Actuator Request", "username": "******", "signature": "None", "time": "2015-05-29 14:28:30.974749", "expires": 500, "message": { "sspl_ll_msg_header": { "schema_version": "1.0.0", "sspl_version": "1.0.0", "msg_version": "1.0.0" }, "sspl_ll_debug": { "debug_component": "sensor", "debug_enabled": True }, "sensor_request_type": { "node_data": { "sensor_type": sensor_type } } } } world.sspl_modules[RabbitMQegressProcessor.name()]._write_internal_msgQ( RabbitMQegressProcessor.name(), egressMsg)
def service_actuator_request(service_name, action): egressMsg = { "title": "SSPL-LL Actuator Request", "description": "Seagate Storage Platform Library - Actuator Request", "username": "******", "expires": 3600, "signature": "None", "time": "2020-03-06 04:08:04.071170", "message": { "sspl_ll_debug": { "debug_component": "sensor", "debug_enabled": True }, "sspl_ll_msg_header": { "msg_version": "1.0.0", "uuid": "9e6b8e53-10f7-4de0-a9aa-b7895bab7774", "schema_version": "1.0.0", "sspl_version": "2.0.0" }, "request_path": { "site_id": "1", "rack_id": "1", "node_id": "1" }, "response_dest": {}, "actuator_request_type": { "service_controller": { "service_request": action, "service_name": service_name } } } } world.sspl_modules[RabbitMQegressProcessor.name()]._write_internal_msgQ\ (RabbitMQegressProcessor.name(), egressMsg)
def fan_actuator_message_request(resource_type, resource_id): egressMsg = { "title": "SSPL Actuator Request", "description": "Seagate Storage Platform Library - Actuator Request", "username": "******", "signature": "None", "time": "2015-05-29 14:28:30.974749", "expires": 500, "message": { "sspl_ll_msg_header": { "schema_version": "1.0.0", "sspl_version": "1.0.0", "msg_version": "1.0.0" }, "sspl_ll_debug": { "debug_component": "sensor", "debug_enabled": True }, "request_path": { "site_id": "1", "rack_id": "1", "cluster_id": "1", "node_id": "1" }, "response_dest": {}, "actuator_request_type": { "storage_enclosure": { "enclosure_request": resource_type, "resource": resource_id } } } } world.sspl_modules[RabbitMQegressProcessor.name()]._write_internal_msgQ( RabbitMQegressProcessor.name(), egressMsg)
def disk_actuator_message_request(resource_type): egressMsg = { "username": "******", "expires": 3600, "description": "Seagate Storage Platform Library - Low Level - Actuator Request", "title": "SSPL-LL Actuator Request", "signature": "None", "time": "2018-07-31 04:08:04.071170", "message": { "sspl_ll_debug": { "debug_component": "sensor", "debug_enabled": True }, "sspl_ll_msg_header": { "msg_version": "1.0.0", "uuid": "9e6b8e53-10f7-4de0-a9aa-b7895bab7774", "schema_version": "1.0.0", "sspl_version": "1.0.0" }, "actuator_request_type": { "node_controller": { "node_request": resource_type, "resource": "*" } } } } world.sspl_modules[RabbitMQegressProcessor.name()]._write_internal_msgQ( RabbitMQegressProcessor.name(), egressMsg)
def _generate_local_mount_data(self): """Create & transmit a local_mount_data message as defined by the sensor response json schema""" # Notify the node sensor to update its data required for the local_mount_data message successful = self._node_sensor.read_data("local_mount_data", self._get_debug(), self._units) if not successful: logger.error( "NodeDataMsgHandler, _generate_local_mount_data was NOT successful." ) # Create the local mount data message and hand it over to the egress processor to transmit localMountDataMsg = LocalMountDataMsg( self._node_sensor.host_id, self._epoch_time, self._node_sensor.free_space, self._node_sensor.free_inodes, self._node_sensor.free_swap, self._node_sensor.total_space, self._node_sensor.total_swap, self._units) # Add in uuid if it was present in the json request if self._uuid is not None: localMountDataMsg.set_uuid(self._uuid) jsonMsg = localMountDataMsg.getJson() # Transmit it out over rabbitMQ channel self._write_internal_msgQ(RabbitMQegressProcessor.name(), jsonMsg)
def _run_thread_capture_errors(curr_module, msgQlist, conf_reader, product): """Run the given thread and log any errors that happen on it. Will stop all sspl_modules if one of them fails.""" try: # Each module is passed a reference list to message queues so it can transmit # internal messages to other modules as desired curr_module.initialize(conf_reader, msgQlist, product) curr_module.start() except BaseException as ex: logger.critical( "SSPL-Tests encountered a fatal error, terminating service Error: %s" % ex) logger.exception(ex) # Populate an actuator response message and transmit back to HAlon error_msg = "SSPL-Tests encountered an error, terminating service Error: " + \ ", Exception: " + logger.exception(ex) jsonMsg = ThreadControllerMsg(curr_module, error_msg).getJson() curr_module._write_internal_msgQ(RabbitMQegressProcessor.name(), jsonMsg) # Shut it down, error is non-recoverable for name, other_module in list(world.sspl_modules.items()): if other_module is not curr_module: other_module.shutdown()
def _send_ifdata_json_msg(self, sensor_type, resource_id, resource_type, state, severity, event=""): """A resuable method for transmitting IFDataMsg to RMQ and IEM logging""" ifDataMsg = IFdataMsg(self._node_sensor.host_id, self._node_sensor.local_time, self._node_sensor.if_data, resource_id, resource_type, self.site_id, self.node_id, self.cluster_id, self.rack_id, state, severity, event) # Add in uuid if it was present in the json request if self._uuid is not None: ifDataMsg.set_uuid(self._uuid) jsonMsg = ifDataMsg.getJson() self.if_sensor_data = jsonMsg self.os_sensor_type[sensor_type] = self.if_sensor_data # Send the event to logging msg handler to send IEM message to journald #internal_json_msg=json.dumps({ # 'actuator_request_type': { # 'logging': { # 'log_level': 'LOG_WARNING', # 'log_type': 'IEM', # 'log_msg': '{}'.format(jsonMsg)}}}) #self._write_internal_msgQ(LoggingMsgHandler.name(), internal_json_msg) # Transmit it out over rabbitMQ channel self._write_internal_msgQ(RabbitMQegressProcessor.name(), jsonMsg)
def _generate_node_fru_data(self, jsonMsg): """Create & transmit a FRU IPMI data message as defined by the sensor response json schema""" if self._node_sensor.host_id is None: successful = self._node_sensor.read_data("None", self._get_debug(), self._units) if not successful: logger.error("NodeDataMsgHandler, updating host information was NOT successful.") if jsonMsg.get("sensor_request_type").get("node_data") is not None: self._fru_info = jsonMsg.get("sensor_request_type").get("node_data") node_ipmi_data_msg = NodeIPMIDataMsg(self._fru_info) if self._uuid is not None: node_ipmi_data_msg.set_uuid(self._uuid) jsonMsg = node_ipmi_data_msg.getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), jsonMsg)
def _generate_psu_alert(self, json_msg, host_name, alert_type, alert_id, severity, info, specific_info, sensor_type): """Parses the json message, also validates it and then send it to the RabbitMQ egress processor""" self._log_debug(f"RealStorEnclMsgHandler, _generate_psu_alert,\ json_msg {json_msg}") real_stor_psu_data_msg = \ RealStorPSUDataMsg(host_name, alert_type, alert_id, severity, info, specific_info) json_msg = real_stor_psu_data_msg.getJson() # Saves the json message in memory to serve sspl CLI sensor request self._psu_sensor_message = json_msg self._fru_type[sensor_type] = self._psu_sensor_message self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg, self._event)
def _generate_enclosure_alert(self, json_msg, host_name, alert_type, alert_id, severity, info, specific_info, sensor_type): """Parses the json message, also validates it and then send it to the RabbitMQ egress processor""" self._log_debug(f"RealStorEnclMsgHandler, _generate_enclosure_alert,\ json_msg {json_msg}") real_stor_encl_msg = RealStorEnclDataMsg(host_name, alert_type, alert_id, severity, info, specific_info) json_msg = real_stor_encl_msg.getJson() self._enclosure_message = json_msg self._fru_type[sensor_type] = self._enclosure_message self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg, self._event)
def _execute_raid_request(self, node_request, actuator_instance, json_msg, uuid): """Performs a RAID request by calling perform_request method of a RAID actuator. """ # Perform the RAID request on the node and get the response raid_response = actuator_instance.perform_request(json_msg).strip() self._log_debug(f"_process_msg, raid_response: {raid_response}") json_msg = AckResponseMsg(node_request, raid_response, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) # Restart openhpid to update HPI data only if it is a H/W environment if self.setup in [ "hw", "ssu" ]: self._log_debug("restarting openhpid service to update HPI data") if "assemble" in json_msg.get("actuator_request_type").get("node_controller").get("node_request").lower(): internal_json_msg = json.dumps( {"actuator_request_type": { "service_controller": { "service_name" : "openhpid.service", "service_request": "restart" }}}) self._write_internal_msgQ(ServiceMsgHandler.name(), internal_json_msg)
def _process_msg(self, json_msg): """Parses the incoming message and generate the desired data message""" self._log_debug( f"RealStorEnclMsgHandler, _process_msg, json_msg: {json_msg}") if json_msg.get("sensor_request_type").get( "enclosure_alert") is not None: internal_sensor_request = json_msg.get("sensor_request_type").\ get("enclosure_alert").get("status") if internal_sensor_request: resource_type = json_msg.get("sensor_request_type").\ get("enclosure_alert").get("info").get("resource_type") if ":" in resource_type: sensor_type = resource_type.split(":")[2] else: sensor_type = resource_type self._propagate_alert(json_msg, sensor_type) else: # serves the request coming from sspl CLI sensor_type = json_msg.get("sensor_request_type").\ get("enclosure_alert").get("info").\ get("resource_type") if ":" in sensor_type: sensor_type = sensor_type.split(":")[2] else: sensor_type = sensor_type sensor_message_type = self._fru_type.get(sensor_type, "") # get the previously saved json message for the sensor type # and send the RabbitMQ Message if sensor_message_type: self._write_internal_msgQ(RabbitMQegressProcessor.name(), sensor_message_type, self._event) else: self._log_debug(f"RealStorEnclMsgHandler, _process_msg, \ No past data found for {sensor_type} sensor type") else: logger.exception("RealStorEnclMsgHandler, _process_msg,\ Not a valid sensor request format")
def _process_msg(self, jsonMsg): """Parses the incoming message and handles appropriately""" self._log_debug(f"RealStorActuatorMsgHandler, _process_msg, jsonMsg: {jsonMsg}") if isinstance(jsonMsg, dict) is False: jsonMsg = json.loads(jsonMsg) # Parse out the uuid so that it can be sent back in Ack message uuid = None if jsonMsg.get("sspl_ll_msg_header").get("uuid") is not None: uuid = jsonMsg.get("sspl_ll_msg_header").get("uuid") self._log_debug(f"_processMsg, uuid: {uuid}") logger.debug(f"RealStorActuatorMsgHandler: _process_msg: jsonMsg: {jsonMsg}") if jsonMsg.get("actuator_request_type").get("storage_enclosure").get("enclosure_request") is not None: enclosure_request = jsonMsg.get("actuator_request_type").get("storage_enclosure").get("enclosure_request") self._log_debug(f"_processMsg, enclosure_request: {enclosure_request}") logger.debug(f"RealStorActuatorMsgHandler: _process_msg: INSIDE: jsonMsg: {jsonMsg}") # Parse out the request field in the enclosure_request (request, fru) = enclosure_request.split(":", 1) request = request.strip() fru = fru.strip() if self._real_stor_actuator is None: try: from actuators.impl.generic.realstor_encl import RealStorActuator self._real_stor_actuator = RealStorActuator() except ImportError as e: logger.warn("RealStor Actuator not loaded") return # Perform the request and get the response real_stor_response = self._real_stor_actuator.perform_request(jsonMsg) self._log_debug(f"_process_msg, RealStor response: {real_stor_response}") json_msg = RealStorActuatorMsg(real_stor_response, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
def _process_msg(self, jsonMsg): """Parses the incoming message and generate the desired data message""" self._log_debug("_process_msg, jsonMsg: %s" % jsonMsg) if isinstance(jsonMsg, dict) is False: jsonMsg = json.loads(jsonMsg) # Parse out the uuid so that it can be sent back in response message self._uuid = None if jsonMsg.get("sspl_ll_msg_header") is not None and \ jsonMsg.get("sspl_ll_msg_header").get("uuid") is not None: self._uuid = jsonMsg.get("sspl_ll_msg_header").get("uuid") self._log_debug("_processMsg, uuid: %s" % self._uuid) if jsonMsg.get("sensor_request_type") is not None and \ jsonMsg.get("sensor_request_type").get("node_data") is not None and \ jsonMsg.get("sensor_request_type").get("node_data").get("sensor_type") is not None: self.sensor_type = jsonMsg.get("sensor_request_type").get( "node_data").get("sensor_type").split(":")[2] self._log_debug("_processMsg, sensor_type: %s" % self.sensor_type) if self.sensor_type == "system": self._generate_host_update() sensor_message_type = self.os_sensor_type.get( self.sensor_type, "") if sensor_message_type: self._write_internal_msgQ(RabbitMQegressProcessor.name(), sensor_message_type) else: self._log_debug(f"NodeDataMsgHandler, _process_msg, \ No past data found for {self.sensor_type} sensor type") elif self.sensor_type == "cpu": self._generate_cpu_data() sensor_message_type = self.os_sensor_type.get( self.sensor_type, "") if sensor_message_type: self._write_internal_msgQ(RabbitMQegressProcessor.name(), sensor_message_type) else: self._log_debug(f"NodeDataMsgHandler, _process_msg, \ No past data found for {self.sensor_type} sensor type") elif self.sensor_type == "nw": self._generate_if_data() sensor_message_type = self.os_sensor_type.get( self.sensor_type, "") if sensor_message_type: self._write_internal_msgQ(RabbitMQegressProcessor.name(), sensor_message_type) else: self._log_debug(f"NodeDataMsgHandler, _process_msg, \ No past data found for {self.sensor_type} sensor type") elif self.sensor_type == "disk_space": self._generate_disk_space_alert() sensor_message_type = self.os_sensor_type.get( self.sensor_type, "") if sensor_message_type: self._write_internal_msgQ(RabbitMQegressProcessor.name(), sensor_message_type) else: self._log_debug(f"NodeDataMsgHandler, _process_msg, \ No past data found for {self.sensor_type} sensor type") elif self.sensor_type == "raid_data": self._generate_raid_data(jsonMsg) sensor_message_type = self.os_sensor_type.get( self.sensor_type, "") if sensor_message_type: self._write_internal_msgQ(RabbitMQegressProcessor.name(), sensor_message_type) else: self._log_debug( "NodeDataMsgHandler, _process_msg " + f"No past data found for {self.sensor_type} sensor type" ) elif self.sensor_type == "raid_integrity": self._generate_raid_integrity_data(jsonMsg) sensor_message_type = self.os_sensor_type.get( self.sensor_type, "") if sensor_message_type: self._write_internal_msgQ(RabbitMQegressProcessor.name(), sensor_message_type) else: self._log_debug( "NodeDataMsgHandler, _process_msg " + f"No past data found for {self.sensor_type} sensor type" ) # Update mapping of device names to serial numbers for global use elif jsonMsg.get("sensor_response_type") is not None: if jsonMsg.get( "sensor_response_type") == "devicename_serialnumber": self._update_devicename_sn_dict(jsonMsg) elif jsonMsg.get("sensor_request_type") is not None and \ jsonMsg.get("sensor_request_type").get("node_data") is not None and \ jsonMsg.get("sensor_request_type").get("node_data").get("info") is not None and \ jsonMsg.get("sensor_request_type").get("node_data").get("info").get("resource_type") is not None: self._generate_node_fru_data(jsonMsg)
def _process_msg(self, jsonMsg): """Parses the incoming message and handles appropriately""" self._log_debug(f"_process_msg, jsonMsg: {jsonMsg}") if isinstance(jsonMsg, dict) is False: jsonMsg = json.loads(jsonMsg) # Parse out the uuid so that it can be sent back in Ack message uuid = None if jsonMsg.get("sspl_ll_msg_header").get("uuid") is not None: uuid = jsonMsg.get("sspl_ll_msg_header").get("uuid") self._log_debug(f"_processMsg, uuid: {uuid}") if jsonMsg.get("actuator_request_type").get("node_controller").get("node_request") is not None: node_request = jsonMsg.get("actuator_request_type").get("node_controller").get("node_request") self._log_debug(f"_processMsg, node_request: {node_request}") # Parse out the component field in the node_request component = node_request[0:4] # Handle generic command line requests if component == 'SSPL': # Query the Zope GlobalSiteManager for an object implementing the MOTR actuator if self._command_line_actuator is None: from actuators.Icommand_line import ICommandLine command_line_actuator_class = self._queryUtility(ICommandLine) # Instantiate CommandLine Actuator only if class is loaded if command_line_actuator_class: self._command_line_actuator = command_line_actuator_class(self._conf_reader) else: logger.warn("CommandLine Actuator not loaded") json_msg = AckResponseMsg(node_request, NodeControllerMsgHandler.UNSUPPORTED_REQUEST, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) return # Perform the request and get the response command_line_response = self._command_line_actuator.perform_request(jsonMsg).strip() self._log_debug(f"_process_msg, command line response: {command_line_response}") json_msg = AckResponseMsg(node_request, command_line_response, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) # Handle LED effects using the HPI actuator elif component == "LED:": # HPI related operations are not supported in VM environment. if self._is_env_vm(): logger.warn("HPI operations are not supported in current environment") return # Query the Zope GlobalSiteManager for an object implementing the IHPI actuator if self._HPI_actuator is None: from actuators.Ihpi import IHPI # Load HPIActuator class HPI_actuator_class = self._queryUtility(IHPI) # Instantiate HPIActuator only if class is loaded if HPI_actuator_class: self._HPI_actuator = HPI_actuator_class(self._conf_reader) else: logger.warn("HPIActuator not loaded") if self._product.lower() in [x.lower() for x in enabled_products]: json_msg = AckResponseMsg(node_request, NodeControllerMsgHandler.UNSUPPORTED_REQUEST, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) return self._log_debug(f"_process_msg, _HPI_actuator name: {self._HPI_actuator.name()}") # Perform the request using HPI and get the response hpi_response = self._HPI_actuator.perform_request(jsonMsg).strip() self._log_debug(f"_process_msg, hpi_response: {hpi_response}") json_msg = AckResponseMsg(node_request, hpi_response, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) # Set the Bezel LED color using the GEM interface elif component == "BEZE": # Query the Zope GlobalSiteManager for an object implementing the IGEM actuator if self._GEM_actuator is None: self._GEM_actuator = self._queryUtility(IGEM)(self._conf_reader) self._log_debug(f"_process_msg, _GEM_actuator name: {self._GEM_actuator.name()}") # Perform the request using GEM and get the response gem_response = self._GEM_actuator.perform_request(jsonMsg).strip() self._log_debug(f"_process_msg, gem_response: {gem_response}") json_msg = AckResponseMsg(node_request, gem_response, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) elif component == "PDU:": # Query the Zope GlobalSiteManager for an object implementing the IPDU actuator if self._PDU_actuator is None: from actuators.Ipdu import IPDU PDU_actuator_class = self._queryUtility(IPDU) # Instantiate RaritanPDU Actuator only if class is loaded if PDU_actuator_class: self._PDU_actuator = PDU_actuator_class(self._conf_reader) else: logger.warn("RaritanPDU Actuator not loaded") json_msg = AckResponseMsg(node_request, NodeControllerMsgHandler.UNSUPPORTED_REQUEST, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) return # Perform the request on the PDU and get the response pdu_response = self._PDU_actuator.perform_request(jsonMsg).strip() self._log_debug(f"_process_msg, pdu_response: {pdu_response}") json_msg = AckResponseMsg(node_request, pdu_response, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) elif component == "RAID": # If the state is INITIALIZED, We can assume that actuator is # ready to perform operation. if actuator_state_manager.is_initialized("RAIDactuator"): self._log_debug(f"_process_msg, _RAID_actuator name: {self._RAID_actuator.name()}") self._execute_raid_request( node_request, self._RAID_actuator, jsonMsg, uuid) # If the state is INITIALIZING, need to send message elif actuator_state_manager.is_initializing("RAIDactuator"): # This state will not be reached. Kept here for consistency. logger.info("RAID actuator is initializing") busy_json_msg = AckResponseMsg( node_request, "BUSY", uuid, error_no=errno.EBUSY).getJson() self._write_internal_msgQ( "RabbitMQegressProcessor", busy_json_msg) elif actuator_state_manager.is_imported("RAIDactuator"): # This case will be for first request only. Subsequent # requests will go to INITIALIZED state case. logger.info("RAID actuator is imported and initializing") from actuators.Iraid import IRAIDactuator actuator_state_manager.set_state( "RAIDactuator", actuator_state_manager.INITIALIZING) # Query the Zope GlobalSiteManager for an object implementing the IRAIDactuator raid_actuator_class = self._queryUtility(IRAIDactuator) if raid_actuator_class: # NOTE: Instantiation part should not time consuming # otherwise NodeControllerMsgHandler will get block # and will not be able serve any subsequent requests. # This applies to instantiation of evey actuator. self._RAID_actuator = raid_actuator_class() logger.info(f"_process_msg, _RAID_actuator name: {self._RAID_actuator.name()}") self._execute_raid_request( node_request, self._RAID_actuator, jsonMsg, uuid) actuator_state_manager.set_state( "RAIDactuator", actuator_state_manager.INITIALIZED) else: logger.warn("RAID actuator is not instantiated") # If there is no entry for actuator in table, We can assume # that it is not loaded for some reason. else: logger.warn("RAID actuator is not loaded or not supported") elif component == "IPMI": # Query the Zope GlobalSiteManager for an object implementing the IPMI actuator if self._IPMI_actuator is None: from actuators.Iipmi import Iipmi IPMI_actuator_class = self._queryUtility(Iipmi) # Instantiate IPMI Actuator only if class is loaded if IPMI_actuator_class: self._IPMI_actuator = IPMI_actuator_class(self._conf_reader) else: logger.warn("IPMI Actuator not loaded") json_msg = AckResponseMsg(node_request, NodeControllerMsgHandler.UNSUPPORTED_REQUEST, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) return # Perform the IPMI request on the node and get the response ipmi_response = self._IPMI_actuator.perform_request(jsonMsg).strip() self._log_debug(f"_process_msg, ipmi_response: {ipmi_response}") json_msg = AckResponseMsg(node_request, ipmi_response, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) elif component == "STOP": # HPI related operations are not supported in VM environment. if self._is_env_vm(): logger.warn("HPI operations are not supported in current environment") return # Query the Zope GlobalSiteManager for an object implementing the IHPI actuator if self._HPI_actuator is None: from actuators.Ihpi import IHPI # Load HPIActuator class HPI_actuator_class = self._queryUtility(IHPI) # Instantiate HPIActuator only if class is loaded if HPI_actuator_class: self._HPI_actuator = HPI_actuator_class(self._conf_reader) else: logger.warn("HPIActuator not loaded") if self._product.lower() in [x.lower() for x in enabled_products]: json_msg = AckResponseMsg(node_request, NodeControllerMsgHandler.UNSUPPORTED_REQUEST, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) return self._log_debug(f"_process_msg, _HPI_actuator name: {self._HPI_actuator.name()}") # Parse out the drive to stop drive_request = node_request[12:].strip() self._log_debug(f"perform_request, drive to stop: {drive_request}") # Append POWER_OFF to notify HPI actuator of desired state jsonMsg["actuator_request_type"]["node_controller"]["node_request"] = \ f"DISK: set {drive_request} POWER_OFF" self._log_debug(f"_process_msg, jsonMsg: {jsonMsg}") # Perform the request using HPI and get the response hpi_response = self._HPI_actuator.perform_request(jsonMsg).strip() self._log_debug(f"_process_msg, hpi_response: {hpi_response}") # Simplify success message as external apps don't care about details if "Success" in hpi_response: hpi_response = "Successful" json_msg = AckResponseMsg(node_request, hpi_response, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) elif component == "STAR": # HPI related operations are not supported in VM environment. if self._is_env_vm(): logger.warn("HPI operations are not supported in current environment") return # Query the Zope GlobalSiteManager for an object implementing the IHPI actuator if self._HPI_actuator is None: from actuators.Ihpi import IHPI # Load HPIActuator class HPI_actuator_class = self._queryUtility(IHPI) # Instantiate HPIActuator only if class is loaded if HPI_actuator_class: self._HPI_actuator = HPI_actuator_class(self._conf_reader) else: logger.warn("HPIActuator not loaded") if self._product.lower() in [x.lower() for x in enabled_products]: json_msg = AckResponseMsg(node_request, NodeControllerMsgHandler.UNSUPPORTED_REQUEST, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) return self._log_debug(f"_process_msg, _HPI_actuator name: {self._HPI_actuator.name()}") # Parse out the drive to start drive_request = node_request[13:].strip() self._log_debug(f"perform_request, drive to start: {drive_request}") # Append POWER_ON to notify HPI actuator of desired state jsonMsg["actuator_request_type"]["node_controller"]["node_request"] = \ f"DISK: set {drive_request} POWER_ON" self._log_debug(f"_process_msg, jsonMsg: {jsonMsg}") # Perform the request using HPI and get the response hpi_response = self._HPI_actuator.perform_request(jsonMsg).strip() self._log_debug(f"_process_msg, hpi_response: {hpi_response}") # Simplify success message as external apps don't care about details if "Success" in hpi_response: hpi_response = "Successful" json_msg = AckResponseMsg(node_request, hpi_response, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) elif component == "RESE": # HPI related operations are not supported in VM environment. if self._is_env_vm(): logger.warn("HPI operations are not supported in current environment") return # Query the Zope GlobalSiteManager for an object implementing the IHPI actuator if self._HPI_actuator is None: from actuators.Ihpi import IHPI # Load HPIActuator class HPI_actuator_class = self._queryUtility(IHPI) # Instantiate HPIActuator only if class is loaded if HPI_actuator_class: self._HPI_actuator = HPI_actuator_class(self._conf_reader) else: logger.warn("HPIActuator not loaded") if self._product.lower() in [x.lower() for x in enabled_products]: json_msg = AckResponseMsg(node_request, NodeControllerMsgHandler.UNSUPPORTED_REQUEST, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) return self._log_debug(f"_process_msg, _HPI_actuator name: {self._HPI_actuator.name()}") # Parse out the drive to power cycle drive_request = node_request[13:].strip() self._log_debug(f"perform_request, drive to power cycle: {drive_request}") # Append POWER_OFF and then POWER_ON to notify HPI actuator of desired state jsonMsg["actuator_request_type"]["node_controller"]["node_request"] = \ f"DISK: set {drive_request} POWER_OFF" self._log_debug(f"_process_msg, jsonMsg: {jsonMsg}") # Perform the request using HPI and get the response hpi_response = self._HPI_actuator.perform_request(jsonMsg).strip() self._log_debug(f"_process_msg, hpi_response: {hpi_response}") # Check for success and power the disk back on if "Success" in hpi_response: # Append POWER_ON to notify HPI actuator of desired state jsonMsg["actuator_request_type"]["node_controller"]["node_request"] = \ f"DISK: set {drive_request} POWER_ON" self._log_debug(f"_process_msg, jsonMsg: {jsonMsg}") # Perform the request using HPI and get the response hpi_response = self._HPI_actuator.perform_request(jsonMsg).strip() self._log_debug(f"_process_msg, hpi_response: {hpi_response}") # Simplify success message as external apps don't care about details if "Success" in hpi_response: hpi_response = "Successful" json_msg = AckResponseMsg(node_request, hpi_response, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) elif component == "HDPA": # If the state is INITIALIZED, We can assume that actuator is # ready to perform operation. if actuator_state_manager.is_initialized("Hdparm"): logger.info(f"_process_msg, Hdparm_actuator name: {self._hdparm_actuator.name()}") # Perform the hdparm request on the node and get the response hdparm_response = self._hdparm_actuator.perform_request(jsonMsg).strip() self._log_debug(f"_process_msg, hdparm_response: {hdparm_response}") json_msg = AckResponseMsg(node_request, hdparm_response, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) # If the state is INITIALIZING, need to send message elif actuator_state_manager.is_initializing("Hdparm"): # This state will not be reached. Kept here for consistency. logger.info("Hdparm actuator is initializing") busy_json_msg = AckResponseMsg( node_request, "BUSY", uuid, error_no=errno.EBUSY).getJson() self._write_internal_msgQ( "RabbitMQegressProcessor", busy_json_msg) elif actuator_state_manager.is_imported("Hdparm"): # This case will be for first request only. Subsequent # requests will go to INITIALIZED state case. logger.info("Hdparm actuator is imported and initializing") # Query the Zope GlobalSiteManager for an object # implementing the hdparm actuator. from actuators.Ihdparm import IHdparm actuator_state_manager.set_state( "Hdparm", actuator_state_manager.INITIALIZING) hdparm_actuator_class = self._queryUtility(IHdparm) if hdparm_actuator_class: # NOTE: Instantiation part should not time consuming # otherwise NodeControllerMsgHandler will get block and will # not be able serve any subsequent requests. This applies # to instantiation of evey actuator. self._hdparm_actuator = hdparm_actuator_class() self._log_debug(f"_process_msg, _hdparm_actuator name: {self._hdparm_actuator.name()}") # Perform the hdparm request on the node and get the response hdparm_response = self._hdparm_actuator.perform_request(jsonMsg).strip() self._log_debug(f"_process_msg, hdparm_response: {hdparm_response}") json_msg = AckResponseMsg(node_request, hdparm_response, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) actuator_state_manager.set_state( "Hdparm", actuator_state_manager.INITIALIZED) else: logger.info("Hdparm actuator is not instantiated") # If there is no entry for actuator in table, We can assume # that it is not loaded for some reason. else: logger.info("Hdparm actuator is not loaded or not supported") elif component == "SMAR": # Parse out the drive request field in json msg node_request = jsonMsg.get("actuator_request_type").get("node_controller").get("node_request") drive_request = node_request[12:].strip() self._log_debug(f"perform_request, drive: {drive_request}") # If the drive field is an asterisk then send all the smart results for all drives available if drive_request == "*": # Send the event to SystemdWatchdog to schedule SMART test internal_json_msg = json.dumps( {"sensor_request_type" : "disk_smart_test", "serial_number" : "*", "node_request" : self.host_id, "uuid" : uuid }) self._write_internal_msgQ("SystemdWatchdog", internal_json_msg) return # Put together a message to get the serial number of the drive using hdparm tool if drive_request.startswith("/"): serial_number, error = self._retrieve_serial_number(drive_request) # Send error response back on ack channel if error != "": json_msg = AckResponseMsg(node_request, error, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) return else: if self._smartctl_actuator is None: from actuators.Ismartctl import ISmartctl smartctl_actuator_class = self._queryUtility(ISmartctl) if smartctl_actuator_class: self._smartctl_actuator = self._queryUtility(ISmartctl)() self._log_debug("_process_msg, _smart_actuator name: %s" % self._smartctl_actuator.name()) else: logger.error(" No module Smartctl is present to load") serial_compare = self._smartctl_actuator._check_serial_number(drive_request) if not serial_compare: json_msg = AckResponseMsg(node_request, "Drive Not Found", uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) return else: serial_number = drive_request # Send the event to SystemdWatchdog to schedule SMART test internal_json_msg = json.dumps( {"sensor_request_type" : "disk_smart_test", "serial_number" : serial_number, "node_request" : node_request, "uuid" : uuid }) self._write_internal_msgQ("SystemdWatchdog", internal_json_msg) elif component == "DRVM": # Requesting the current status from drivemanager # Parse out the drive request field in json msg node_request = jsonMsg.get("actuator_request_type").get("node_controller").get("node_request") drive_request = node_request[15:].strip() self._log_debug(f"perform_request, drive: {drive_request}") # If the drive field is an asterisk then send all the drivemanager results for all drives available if drive_request == "*": # Send a message to the disk message handler to lookup the drivemanager status and send it out internal_json_msg = json.dumps( {"sensor_request_type" : "drvmngr_status", "serial_number" : "*", "node_request" : self.host_id, "uuid" : uuid }) # Send the event to disk message handler to generate json message self._write_internal_msgQ(DiskMsgHandler.name(), internal_json_msg) return # Put together a message to get the serial number of the drive using hdparm tool if drive_request.startswith("/"): serial_number, error = self._retrieve_serial_number(drive_request) # Send error response back on ack channel if error != "": json_msg = AckResponseMsg(node_request, error, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) return else: serial_number = drive_request # Send a message to the disk message handler to lookup the smart status and send it out internal_json_msg = json.dumps( {"sensor_request_type" : "drvmngr_status", "serial_number" : serial_number, "node_request" : node_request, "uuid" : uuid }) # Send the event to disk message handler to generate json message self._write_internal_msgQ(DiskMsgHandler.name(), internal_json_msg) elif component == "HPI_": # Requesting the current status from HPI data # Parse out the drive request field in json msg if self._is_env_vm(): logger.warn("HPI operations are not supported in current environment") return if self.setup == 'cortx': logger.warn("HPIMonitor not loaded") json_msg = AckResponseMsg(node_request, NodeControllerMsgHandler.UNSUPPORTED_REQUEST, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) return node_request = jsonMsg.get("actuator_request_type").get("node_controller").get("node_request") drive_request = node_request[11:].strip() self._log_debug(f"perform_request, drive: {drive_request}") # If the drive field is an asterisk then send all the hpi results for all drives available if drive_request == "*": # Send a message to the disk message handler to lookup the hpi status and send it out internal_json_msg = json.dumps( {"sensor_request_type" : "hpi_status", "serial_number" : "*", "node_request" : self.host_id, "uuid" : uuid }) # Send the event to disk message handler to generate json message self._write_internal_msgQ(DiskMsgHandler.name(), internal_json_msg) return # Put together a message to get the serial number of the drive using hdparm tool if drive_request.startswith("/"): serial_number, error = self._retrieve_serial_number(drive_request) # Send error response back on ack channel if error != "": json_msg = AckResponseMsg(node_request, error, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) return else: serial_number = drive_request # Send a message to the disk message handler to lookup the smart status and send it out internal_json_msg = json.dumps( {"sensor_request_type" : "hpi_status", "serial_number" : serial_number, "node_request" : node_request, "uuid" : uuid }) # Send the event to disk message handler to generate json message self._write_internal_msgQ(DiskMsgHandler.name(), internal_json_msg) elif component == "SIMU": # Requesting to simulate an event # Parse out the simulated request field node_request = jsonMsg.get("actuator_request_type").get("node_controller").get("node_request") sim_request = node_request[9:].strip().split(" ") self._log_debug(f"perform_request, sim_request: {str(sim_request)}") # Put together a message to get the serial number of the drive using hdparm tool if sim_request[1].startswith("/"): serial_number, error = self._retrieve_serial_number(sim_request[1]) # Send error response back on ack channel if error != "": json_msg = AckResponseMsg(node_request, error, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) return else: serial_number = sim_request[1] # SMART simulation requests are sent to SystemdWatchdog if sim_request[0] == "SMART_FAILURE": logger.info(f"NodeControllerMsgHandler, simulating SMART_FAILURE on drive: {serial_number}") internal_json_msg = json.dumps( {"sensor_request_type" : "simulate_failure", "serial_number" : serial_number, "node_request" : sim_request[0], "uuid" : uuid }) # Send the event to SystemdWatchdog to handle it from here self._write_internal_msgQ("SystemdWatchdog", internal_json_msg) else: # Send a message to the disk message handler to handle simulation request internal_json_msg = json.dumps( {"sensor_request_type" : "sim_event", "serial_number" : serial_number, "node_request" : sim_request[0], "uuid" : uuid }) # Send the event to disk message handler to generate json message self._write_internal_msgQ(DiskMsgHandler.name(), internal_json_msg) elif component == "NDHW": # NDHW Stands for Node HW. try: # Load and Instantiate the Actuator for the first request if self._NodeHW_actuator is None: from actuators.impl.generic.node_hw import NodeHWactuator from framework.utils.ipmi_client import IpmiFactory self.ipmi_client_name = self._conf_reader._get_value_with_default( self.NODE_HW_ACTUATOR, self.IPMI_IMPLEMENTOR, "ipmitool") ipmi_factory = IpmiFactory() ipmi_client = \ ipmi_factory.get_implementor(self.ipmi_client_name) # Instantiate NodeHWactuator only if class is loaded if ipmi_client is not None: self._NodeHW_actuator = NodeHWactuator(ipmi_client, self._conf_reader) self._NodeHW_actuator.initialize() else: logger.error(f"IPMI client: '{self.ipmi_client_name}' doesn't exist") return node_request = jsonMsg.get("actuator_request_type") # Perform the NodeHW request on the node and get the response #TODO: Send message to Ack as well as Sensor in their respective channel. node_hw_response = self._NodeHW_actuator.perform_request(node_request) self._log_debug(f"_process_msg, node_hw_response: {node_hw_response}") json_msg = NodeHwAckResponseMsg(node_request, node_hw_response, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) except ImportError as e: logger.error(f"Modules could not be loaded: {e}") return except Exception as e: logger.error(f"NodeControllerMsgHandler, _process_msg, Exception in request handling: {e}") return else: response = f"NodeControllerMsgHandler, _process_msg, unknown node controller msg: {node_request}" self._log_debug(response) json_msg = AckResponseMsg(node_request, response, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
def _send_msg(self, iem_components, log_timestamp): """Creates JSON message from iem components and sends to RabbitMQ channel. """ # IEM format is IEC:DESCRIPTION # IEC format is SEVERITY|SOURCEID|COMPONENTID|MODULEID|EVENTID # Field lengths ----1---|---1----|------3----|----3---|---4--- # Example IEM -> "IEC: BO1001000001:Error in connecting to controller" # Actual IEC doesn't contain separator between fields. It is shown # here just for readability. Each field has fixed length. severity, source_id, component_id, module_id, event_id, description = \ [iem_components[i] for i in range(6)] # Check if severity level is valid if severity not in self.SEVERITY_LEVELS: logger.warn(f"Invalid Severity level: {severity}") return # Check for valid source id if source_id not in self.SOURCE_IDS: logger.warn(f"Invalid Source ID level: {source_id}") return # Check for valid event time event_time = self._get_epoch_time_from_timestamp(log_timestamp) if not event_time: logger.error( "Timestamp is not in required format, discarding the message") return # Check for other components args = { "_comp_id": component_id, "_module_id": module_id, "_event_id": event_id } if not self._are_components_in_range(**args): return # Update severity and source_id alert_type = iem_severity_to_alert_mapping.get(severity) severity = iem_severity_types.get(severity, severity) source_id = iem_source_types.get(source_id, source_id) # Decode component_id, module_id and event_id component_id, module_id, event_id = self._decode_msg( f"{component_id}{module_id}{event_id}") info = { "site_id": self._site_id, "rack_id": self._rack_id, "node_id": self._node_id, "cluster_id": self._cluster_id, "source_id": source_id, "component_id": component_id, "module_id": module_id, "event_id": event_id, "severity": severity, "description": description, "alert_type": alert_type, "event_time": event_time, "IEC": "".join(iem_components[:-1]) } iem_data_msg = IEMDataMsg(info) json_msg = iem_data_msg.getJson() # RAAL stands for - RAise ALert logger.info(f"RAAL: {json_msg}") self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
def _generate_host_update(self): """Create & transmit a host update message as defined by the sensor response json schema""" # Notify the node sensor to update its data required for the host_update message successful = self._node_sensor.read_data("host_update", self._get_debug(), self._units) if not successful: logger.error( "NodeDataMsgHandler, _generate_host_update was NOT successful." ) self._host_memory_usage_threshold = str( self._host_memory_usage_threshold) try: if self._host_memory_usage_threshold.isdigit(): self._host_memory_usage_threshold = int( self._host_memory_usage_threshold) else: self._host_memory_usage_threshold = float( self._host_memory_usage_threshold) except ValueError: logger.warning( "Host Memory Alert, Invalid host_memory_usage_threshold value are entered in config." ) # Assigning default value to _disk_usage_threshold self._host_memory_usage_threshold = self.DEFAULT_HOST_MEMORY_USAGE_THRESHOLD if self._node_sensor.total_memory[ "percent"] >= self._host_memory_usage_threshold: # Create the disk space data message and hand it over to the egress processor to transmit if not self.host_fault: self.host_fault = True # Create the disk space data message and hand it over to the egress processor to transmit logger.warning("Host Memory usage increased to {}%, beyond configured threshold of {}%".\ format(self._node_sensor.total_memory["percent"], self._host_memory_usage_threshold)) logged_in_users = [] # Create the host update message and hand it over to the egress processor to transmit hostUpdateMsg = HostUpdateMsg( self._node_sensor.host_id, self._epoch_time, self._node_sensor.boot_time, self._node_sensor.up_time, self._node_sensor.uname, self._units, self.site_id, self.rack_id, self.node_id, self.cluster_id, self._node_sensor.total_memory, self._node_sensor.logged_in_users, self._node_sensor.process_count, self._node_sensor.running_process_count, self.FAULT) # Add in uuid if it was present in the json request if self._uuid is not None: hostUpdateMsg.set_uuid(self._uuid) jsonMsg = hostUpdateMsg.getJson() # Transmit it out over rabbitMQ channel self.host_sensor_data = jsonMsg self.os_sensor_type["system"] = self.host_sensor_data self._write_internal_msgQ(RabbitMQegressProcessor.name(), jsonMsg) if (self._node_sensor.total_memory["percent"] < self._host_memory_usage_threshold) and (self.host_fault == True): logger.warning("Host Memory usage decrease to {}%, lesser than configured threshold of {}%".\ format(self._host_memory_usage_threshold, self._node_sensor.total_memory["percent"])) logged_in_users = [] # Create the host update message and hand it over to the egress processor to transmit hostUpdateMsg = HostUpdateMsg( self._node_sensor.host_id, self._epoch_time, self._node_sensor.boot_time, self._node_sensor.up_time, self._node_sensor.uname, self._units, self.site_id, self.rack_id, self.node_id, self.cluster_id, self._node_sensor.total_memory, self._node_sensor.logged_in_users, self._node_sensor.process_count, self._node_sensor.running_process_count, self.FAULT_RESOLVED) # Add in uuid if it was present in the json request if self._uuid is not None: hostUpdateMsg.set_uuid(self._uuid) jsonMsg = hostUpdateMsg.getJson() # Transmit it out over rabbitMQ channel self.host_sensor_data = jsonMsg self.os_sensor_type["system"] = self.host_sensor_data self._write_internal_msgQ(RabbitMQegressProcessor.name(), jsonMsg) self.host_fault = False
def _generate_cpu_data(self): """Create & transmit a cpu_data message as defined by the sensor response json schema""" # Notify the node sensor to update its data required for the cpu_data message successful = self._node_sensor.read_data("cpu_data", self._get_debug()) if not successful: logger.error( "NodeDataMsgHandler, _generate_cpu_data was NOT successful.") self._cpu_usage_threshold = str(self._cpu_usage_threshold) try: if self._cpu_usage_threshold.isdigit(): self._cpu_usage_threshold = int(self._cpu_usage_threshold) else: self._cpu_usage_threshold = float(self._cpu_usage_threshold) except ValueError: logger.warning( "CPU Usage Alert, Invalid host_memory_usage_threshold value are entered in config." ) # Assigning default value to _cpu_usage_threshold self._cpu_usage_threshold = self.DEFAULT_CPU_USAGE_THRESHOLD if self._node_sensor.cpu_usage >= self._cpu_usage_threshold: if not self.cpu_fault: self.cpu_fault = True # Create the cpu usage data message and hand it over to the egress processor to transmit logger.warning("CPU usage increased to {}%, beyond configured threshold of {}%".\ format(self._node_sensor.cpu_usage, self._cpu_usage_threshold)) # Create the local mount data message and hand it over to the egress processor to transmit cpuDataMsg = CPUdataMsg( self._node_sensor.host_id, self._epoch_time, self._node_sensor.csps, self._node_sensor.idle_time, self._node_sensor.interrupt_time, self._node_sensor.iowait_time, self._node_sensor.nice_time, self._node_sensor.softirq_time, self._node_sensor.steal_time, self._node_sensor.system_time, self._node_sensor.user_time, self._node_sensor.cpu_core_data, self._node_sensor.cpu_usage, self.site_id, self.rack_id, self.node_id, self.cluster_id, self.FAULT) # Add in uuid if it was present in the json request if self._uuid is not None: cpuDataMsg.set_uuid(self._uuid) jsonMsg = cpuDataMsg.getJson() self.cpu_sensor_data = jsonMsg self.os_sensor_type["cpu"] = self.cpu_sensor_data # Transmit it out over rabbitMQ channel self._write_internal_msgQ(RabbitMQegressProcessor.name(), jsonMsg) if (self._node_sensor.cpu_usage <= self._cpu_usage_threshold) and (self.cpu_fault == True): # Create the cpu usage data message and hand it over to the egress processor to transmit logger.warning("CPU usage decrised to {}%, lesser than configured threshold of {}%".\ format(self._cpu_usage_threshold, self._node_sensor.cpu_usage)) # Create the local mount data message and hand it over to the egress processor to transmit cpuDataMsg = CPUdataMsg( self._node_sensor.host_id, self._epoch_time, self._node_sensor.csps, self._node_sensor.idle_time, self._node_sensor.interrupt_time, self._node_sensor.iowait_time, self._node_sensor.nice_time, self._node_sensor.softirq_time, self._node_sensor.steal_time, self._node_sensor.system_time, self._node_sensor.user_time, self._node_sensor.cpu_core_data, self._node_sensor.cpu_usage, self.site_id, self.rack_id, self.node_id, self.cluster_id, self.FAULT_RESOLVED) # Add in uuid if it was present in the json request if self._uuid is not None: cpuDataMsg.set_uuid(self._uuid) jsonMsg = cpuDataMsg.getJson() self.cpu_sensor_data = jsonMsg self.os_sensor_type["cpu"] = self.cpu_sensor_data # Transmit it out over rabbitMQ channel self._write_internal_msgQ(RabbitMQegressProcessor.name(), jsonMsg) self.cpu_fault = False
def _generate_disk_space_alert(self): """Create & transmit a disk_space_alert message as defined by the sensor response json schema""" # Notify the node sensor to update its data required for the disk_space_data message successful = self._node_sensor.read_data("disk_space_alert", self._get_debug(), self._units) if not successful: logger.error( "NodeDataMsgHandler, _generate_disk_space_alert was NOT successful." ) return # Changing disk_usage_threshold type according to what value type entered in config file self._disk_usage_threshold = str(self._disk_usage_threshold) try: if self._disk_usage_threshold.isdigit(): self._disk_usage_threshold = int(self._disk_usage_threshold) else: self._disk_usage_threshold = float(self._disk_usage_threshold) except ValueError: logger.warning( "Disk Space Alert, Invalid disk_usage_threshold value are entered in config." ) # Assigning default value to _disk_usage_threshold self._disk_usage_threshold = self.DEFAULT_DISK_USAGE_THRESHOLD if self._node_sensor.disk_used_percentage >= self._disk_usage_threshold: if not self.disk_fault: self.disk_fault = True # Create the disk space data message and hand it over to the egress processor to transmit logger.warning("Disk usage increased to {}%, beyond configured threshold of {}%".\ format(self._node_sensor.disk_used_percentage, self._disk_usage_threshold)) diskSpaceAlertMsg = DiskSpaceAlertMsg( self._node_sensor.host_id, self._epoch_time, self._node_sensor.total_space, self._node_sensor.free_space, self._node_sensor.disk_used_percentage, self._units, self.site_id, self.rack_id, self.node_id, self.cluster_id, self.FAULT) # Add in uuid if it was present in the json request if self._uuid is not None: diskSpaceAlertMsg.set_uuid(self._uuid) jsonMsg = diskSpaceAlertMsg.getJson() self.disk_sensor_data = jsonMsg self.os_sensor_type["disk_space"] = self.disk_sensor_data # Transmit it out over rabbitMQ channel self._write_internal_msgQ(RabbitMQegressProcessor.name(), jsonMsg) if (self._node_sensor.disk_used_percentage <= self._disk_usage_threshold) and (self.disk_fault == True): # Create the disk space data message and hand it over to the egress processor to transmit logger.warning("Disk usage decrised to {}%, lesser than threshold of {}%".\ format(self._disk_usage_threshold, self._node_sensor.disk_used_percentage, )) diskSpaceAlertMsg = DiskSpaceAlertMsg( self._node_sensor.host_id, self._epoch_time, self._node_sensor.total_space, self._node_sensor.free_space, self._node_sensor.disk_used_percentage, self._units, self.site_id, self.rack_id, self.node_id, self.cluster_id, self.FAULT_RESOLVED) # Add in uuid if it was present in the json request if self._uuid is not None: diskSpaceAlertMsg.set_uuid(self._uuid) jsonMsg = diskSpaceAlertMsg.getJson() self.disk_sensor_data = jsonMsg self.os_sensor_type["disk_space"] = self.disk_sensor_data # Transmit it out over rabbitMQ channel self._write_internal_msgQ(RabbitMQegressProcessor.name(), jsonMsg) self.disk_fault = False