def _execute_raid_request(self, node_request, actuator_instance, json_msg, uuid): """Performs a RAID request by calling perform_request method of a RAID actuator. """ # Perform the RAID request on the node and get the response raid_response = actuator_instance.perform_request(json_msg).strip() self._log_debug(f"_process_msg, raid_response: {raid_response}") json_msg = AckResponseMsg(node_request, raid_response, uuid).getJson() self._write_internal_msgQ(EgressProcessor.name(), json_msg) # Restart openhpid to update HPI data only if it is a H/W environment if self.setup in ["hw", "ssu"]: self._log_debug("restarting openhpid service to update HPI data") if "assemble" in json_msg.get("actuator_request_type").get( "node_controller").get("node_request").lower(): internal_json_msg = json.dumps({ "actuator_request_type": { "service_controller": { "service_name": "openhpid.service", "service_request": "restart" } } }) self._write_internal_msgQ(ServiceMsgHandler.name(), internal_json_msg)
def _send_to_msg_handler(self, msgType, message, uuid): # Hand off to appropriate actuator message handler if msgType.get("thread_controller") is not None: self._write_internal_msgQ("ThreadController", message) elif msgType.get("service_controller") is not None: self._write_internal_msgQ("ServiceMsgHandler", message) elif msgType.get("node_controller") is not None: self._write_internal_msgQ("NodeControllerMsgHandler", message) elif msgType.get("storage_enclosure") is not None: self._write_internal_msgQ("RealStorActuatorMsgHandler", message) # Hand off to appropriate sensor message handler elif msgType.get("node_data") is not None: self._write_internal_msgQ("NodeDataMsgHandler", message) elif msgType.get("enclosure_alert") is not None: self._write_internal_msgQ("RealStorEnclMsgHandler", message) elif msgType.get("storage_enclosure") is not None: self._write_internal_msgQ("RealStorActuatorMsgHandler", message) # ... handle other incoming messages that have been validated else: # Send ack about not finding a msg handler ack_msg = AckResponseMsg("Error Processing Message", "Message Handler Not Found", uuid).getJson() self._write_internal_msgQ(EgressProcessor.name(), ack_msg)
def _process_msg(self, body): """Parses the incoming message and hands off to the appropriate module""" ingressMsg = {} uuid = None try: if isinstance(body, dict) is False: ingressMsg = json.loads(body) else: ingressMsg = body # Authenticate message using username and signature fields username = ingressMsg.get("username") signature = ingressMsg.get("signature") message = ingressMsg.get("message") uuid = ingressMsg.get("uuid") msg_len = len(message) + 1 if uuid is None: uuid = "N/A" if use_security_lib and \ SSPL_SEC.sspl_verify_message(msg_len, str(message), username, signature) != 0: logger.warn( "IngressProcessor, Authentication failed on message: %s" % ingressMsg) return # Get the incoming message type if message.get("actuator_request_type") is not None: msgType = message.get("actuator_request_type") # Validate against the actuator schema validate(ingressMsg, self._actuator_schema) elif message.get("sensor_request_type") is not None: msgType = message.get("sensor_request_type") # Validate against the sensor schema validate(ingressMsg, self._sensor_schema) else: # We only handle incoming actuator and sensor requests, ignore # everything else. return # Check for debugging being activated in the message header self._check_debug(message) self._log_debug("_process_msg, ingressMsg: %s" % ingressMsg) self._send_to_msg_handler(msgType, message, uuid) except Exception as ex: logger.error( "IngressProcessor, _process_msg unrecognized message: %r" % ingressMsg) ack_msg = AckResponseMsg("Error Processing Msg", "Msg Handler Not Found", uuid).getJson() self._write_internal_msgQ(EgressProcessor.name(), ack_msg)
def _process_job_status(self, uuid, job_uuid): """Send the current job status requested""" self._log_debug( "_process_job_status, job_status requested on uuid: %s" % job_uuid) try: # Get a copy of the job tasks in the PlaneCntrlMsgHandler msg queue pending being worked plane_cntrl_jobs = self._get_msgQ_copy("PlaneCntrlMsgHandler") response = "Not Found" # See if the requested uuid is in the list of pending jobs for plane_cntrl_job in plane_cntrl_jobs: self._log_debug("_process_job_status, plane_cntrl_job: %s" % str(plane_cntrl_job)) if job_uuid in str(plane_cntrl_job): response = "In Queue" break ack_type = {} ack_type["hostname"] = gethostname() ack_type["command"] = "job_status" ack_type["arguments"] = str(job_uuid) # The uuid is either not found or it's in the queue to be worked ack_msg = AckResponseMsg(json.dumps(ack_type), response, uuid).getJson() self._write_internal_msgQ(PlaneCntrlRMQegressProcessor.name(), ack_msg) except Exception as ex: logger.exception( "PlaneCntrlRMQingressProcessor, _process_job_status exception: %s" % str(ex))
def _send_response(self, status, hostname, response, errors): """Transmit the response back as an Ack json msg""" ack_type = {} ack_type["hostname"], ack_type["command"], ack_type["parameters"], ack_type["status"], \ ack_type["errors"] = str(hostname, 'utf-8'), self._command, self._parameters, \ status, str(errors, 'utf-8') ack_msg = AckResponseMsg(json.dumps(ack_type), \ str(response), self._uuid).getJson() self._write_internal_msgQ(PlaneCntrlRMQegressProcessor.name(), ack_msg)
def _process_msg(self, jsonMsg): """Parses the incoming message and hands off to the appropriate logger""" self._log_debug(f"_process_msg, jsonMsg: {jsonMsg}") if isinstance(jsonMsg, dict) is False: jsonMsg = json.loads(jsonMsg) uuid = None if jsonMsg.get("sspl_ll_msg_header") is not None and \ jsonMsg.get("sspl_ll_msg_header").get("uuid") is not None: uuid = jsonMsg.get("sspl_ll_msg_header").get("uuid") self._log_debug(f"_process_msg, uuid: {uuid}") log_type = jsonMsg.get("actuator_request_type").get("logging").get("log_type") result = "N/A" # Disabled for LDR_R1 # if log_type == "IEM": # self._log_debug("_process_msg, msg_type: IEM") # if self._iem_log_locally == "true": # result = self._iem_logger.log_msg(jsonMsg) # self._log_debug(f"Log IEM results: {result}") if log_type == "HDS": # Retrieve the serial number of the drive self._log_debug("_process_msg, msg_type: HDS") log_msg = jsonMsg.get("actuator_request_type").get("logging").get("log_msg") # Parse out the json data section in the IEM and replace single quotes with double json_data = json.loads(str('{' + log_msg.split('{')[1]).replace("'", '"')) serial_number = json_data.get("serial_number") status = json_data.get("status") reason = json_data.get("reason") self._log_debug(f"_processMsg, serial_number: {serial_number}, status:{status}, reason: {reason}") # Send a message to the disk manager handler to create and transmit json msg internal_json_msg = json.dumps( {"sensor_response_type" : "disk_status_HDS", "object_path" : "HDS", "status" : status, "reason" : reason, "serial_number" : serial_number }) # Send the event to disk message handler to generate json message self._write_internal_msgQ("DiskMsgHandler", internal_json_msg) # Disabled for LDR_R1 # Hand off to the IEM logger # result = self._iem_logger.log_msg(jsonMsg) # Send ack about logging msg ack_msg = AckResponseMsg(log_type, result, uuid).getJson() self._write_internal_msgQ(EgressProcessor.name(), ack_msg)
def _process_msg(self, jsonMsg): """Parses the incoming message and handles appropriately""" self._log_debug(f"_process_msg, jsonMsg: {jsonMsg}") if isinstance(jsonMsg, dict) is False: jsonMsg = json.loads(jsonMsg) # Parse out the uuid so that it can be sent back in Ack message uuid = None if jsonMsg.get("sspl_ll_msg_header").get("uuid") is not None: uuid = jsonMsg.get("sspl_ll_msg_header").get("uuid") self._log_debug(f"_processMsg, uuid: {uuid}") if jsonMsg.get("actuator_request_type").get("node_controller").get("node_request") is not None: node_request = jsonMsg.get("actuator_request_type").get("node_controller").get("node_request") self._log_debug(f"_processMsg, node_request: {node_request}") # Parse out the component field in the node_request component = node_request[0:4] # Handle generic command line requests if component == 'SSPL': # Query the Zope GlobalSiteManager for an object implementing the MOTR actuator if self._command_line_actuator is None: from actuators.Icommand_line import ICommandLine command_line_actuator_class = self._queryUtility(ICommandLine) # Instantiate CommandLine Actuator only if class is loaded if command_line_actuator_class: self._command_line_actuator = command_line_actuator_class(self._conf_reader) else: logger.warn("CommandLine Actuator not loaded") json_msg = AckResponseMsg(node_request, NodeControllerMsgHandler.UNSUPPORTED_REQUEST, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) return # Perform the request and get the response command_line_response = self._command_line_actuator.perform_request(jsonMsg).strip() self._log_debug(f"_process_msg, command line response: {command_line_response}") json_msg = AckResponseMsg(node_request, command_line_response, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) # Handle LED effects using the HPI actuator elif component == "LED:": # HPI related operations are not supported in VM environment. if self._is_env_vm(): logger.warn("HPI operations are not supported in current environment") return # Query the Zope GlobalSiteManager for an object implementing the IHPI actuator if self._HPI_actuator is None: from actuators.Ihpi import IHPI # Load HPIActuator class HPI_actuator_class = self._queryUtility(IHPI) # Instantiate HPIActuator only if class is loaded if HPI_actuator_class: self._HPI_actuator = HPI_actuator_class(self._conf_reader) else: logger.warn("HPIActuator not loaded") if self._product.lower() in [x.lower() for x in enabled_products]: json_msg = AckResponseMsg(node_request, NodeControllerMsgHandler.UNSUPPORTED_REQUEST, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) return self._log_debug(f"_process_msg, _HPI_actuator name: {self._HPI_actuator.name()}") # Perform the request using HPI and get the response hpi_response = self._HPI_actuator.perform_request(jsonMsg).strip() self._log_debug(f"_process_msg, hpi_response: {hpi_response}") json_msg = AckResponseMsg(node_request, hpi_response, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) # Set the Bezel LED color using the GEM interface elif component == "BEZE": # Query the Zope GlobalSiteManager for an object implementing the IGEM actuator if self._GEM_actuator is None: self._GEM_actuator = self._queryUtility(IGEM)(self._conf_reader) self._log_debug(f"_process_msg, _GEM_actuator name: {self._GEM_actuator.name()}") # Perform the request using GEM and get the response gem_response = self._GEM_actuator.perform_request(jsonMsg).strip() self._log_debug(f"_process_msg, gem_response: {gem_response}") json_msg = AckResponseMsg(node_request, gem_response, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) elif component == "PDU:": # Query the Zope GlobalSiteManager for an object implementing the IPDU actuator if self._PDU_actuator is None: from actuators.Ipdu import IPDU PDU_actuator_class = self._queryUtility(IPDU) # Instantiate RaritanPDU Actuator only if class is loaded if PDU_actuator_class: self._PDU_actuator = PDU_actuator_class(self._conf_reader) else: logger.warn("RaritanPDU Actuator not loaded") json_msg = AckResponseMsg(node_request, NodeControllerMsgHandler.UNSUPPORTED_REQUEST, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) return # Perform the request on the PDU and get the response pdu_response = self._PDU_actuator.perform_request(jsonMsg).strip() self._log_debug(f"_process_msg, pdu_response: {pdu_response}") json_msg = AckResponseMsg(node_request, pdu_response, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) elif component == "RAID": # If the state is INITIALIZED, We can assume that actuator is # ready to perform operation. if actuator_state_manager.is_initialized("RAIDactuator"): self._log_debug(f"_process_msg, _RAID_actuator name: {self._RAID_actuator.name()}") self._execute_raid_request( node_request, self._RAID_actuator, jsonMsg, uuid) # If the state is INITIALIZING, need to send message elif actuator_state_manager.is_initializing("RAIDactuator"): # This state will not be reached. Kept here for consistency. logger.info("RAID actuator is initializing") busy_json_msg = AckResponseMsg( node_request, "BUSY", uuid, error_no=errno.EBUSY).getJson() self._write_internal_msgQ( "RabbitMQegressProcessor", busy_json_msg) elif actuator_state_manager.is_imported("RAIDactuator"): # This case will be for first request only. Subsequent # requests will go to INITIALIZED state case. logger.info("RAID actuator is imported and initializing") from actuators.Iraid import IRAIDactuator actuator_state_manager.set_state( "RAIDactuator", actuator_state_manager.INITIALIZING) # Query the Zope GlobalSiteManager for an object implementing the IRAIDactuator raid_actuator_class = self._queryUtility(IRAIDactuator) if raid_actuator_class: # NOTE: Instantiation part should not time consuming # otherwise NodeControllerMsgHandler will get block # and will not be able serve any subsequent requests. # This applies to instantiation of evey actuator. self._RAID_actuator = raid_actuator_class() logger.info(f"_process_msg, _RAID_actuator name: {self._RAID_actuator.name()}") self._execute_raid_request( node_request, self._RAID_actuator, jsonMsg, uuid) actuator_state_manager.set_state( "RAIDactuator", actuator_state_manager.INITIALIZED) else: logger.warn("RAID actuator is not instantiated") # If there is no entry for actuator in table, We can assume # that it is not loaded for some reason. else: logger.warn("RAID actuator is not loaded or not supported") elif component == "IPMI": # Query the Zope GlobalSiteManager for an object implementing the IPMI actuator if self._IPMI_actuator is None: from actuators.Iipmi import Iipmi IPMI_actuator_class = self._queryUtility(Iipmi) # Instantiate IPMI Actuator only if class is loaded if IPMI_actuator_class: self._IPMI_actuator = IPMI_actuator_class(self._conf_reader) else: logger.warn("IPMI Actuator not loaded") json_msg = AckResponseMsg(node_request, NodeControllerMsgHandler.UNSUPPORTED_REQUEST, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) return # Perform the IPMI request on the node and get the response ipmi_response = self._IPMI_actuator.perform_request(jsonMsg).strip() self._log_debug(f"_process_msg, ipmi_response: {ipmi_response}") json_msg = AckResponseMsg(node_request, ipmi_response, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) elif component == "STOP": # HPI related operations are not supported in VM environment. if self._is_env_vm(): logger.warn("HPI operations are not supported in current environment") return # Query the Zope GlobalSiteManager for an object implementing the IHPI actuator if self._HPI_actuator is None: from actuators.Ihpi import IHPI # Load HPIActuator class HPI_actuator_class = self._queryUtility(IHPI) # Instantiate HPIActuator only if class is loaded if HPI_actuator_class: self._HPI_actuator = HPI_actuator_class(self._conf_reader) else: logger.warn("HPIActuator not loaded") if self._product.lower() in [x.lower() for x in enabled_products]: json_msg = AckResponseMsg(node_request, NodeControllerMsgHandler.UNSUPPORTED_REQUEST, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) return self._log_debug(f"_process_msg, _HPI_actuator name: {self._HPI_actuator.name()}") # Parse out the drive to stop drive_request = node_request[12:].strip() self._log_debug(f"perform_request, drive to stop: {drive_request}") # Append POWER_OFF to notify HPI actuator of desired state jsonMsg["actuator_request_type"]["node_controller"]["node_request"] = \ f"DISK: set {drive_request} POWER_OFF" self._log_debug(f"_process_msg, jsonMsg: {jsonMsg}") # Perform the request using HPI and get the response hpi_response = self._HPI_actuator.perform_request(jsonMsg).strip() self._log_debug(f"_process_msg, hpi_response: {hpi_response}") # Simplify success message as external apps don't care about details if "Success" in hpi_response: hpi_response = "Successful" json_msg = AckResponseMsg(node_request, hpi_response, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) elif component == "STAR": # HPI related operations are not supported in VM environment. if self._is_env_vm(): logger.warn("HPI operations are not supported in current environment") return # Query the Zope GlobalSiteManager for an object implementing the IHPI actuator if self._HPI_actuator is None: from actuators.Ihpi import IHPI # Load HPIActuator class HPI_actuator_class = self._queryUtility(IHPI) # Instantiate HPIActuator only if class is loaded if HPI_actuator_class: self._HPI_actuator = HPI_actuator_class(self._conf_reader) else: logger.warn("HPIActuator not loaded") if self._product.lower() in [x.lower() for x in enabled_products]: json_msg = AckResponseMsg(node_request, NodeControllerMsgHandler.UNSUPPORTED_REQUEST, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) return self._log_debug(f"_process_msg, _HPI_actuator name: {self._HPI_actuator.name()}") # Parse out the drive to start drive_request = node_request[13:].strip() self._log_debug(f"perform_request, drive to start: {drive_request}") # Append POWER_ON to notify HPI actuator of desired state jsonMsg["actuator_request_type"]["node_controller"]["node_request"] = \ f"DISK: set {drive_request} POWER_ON" self._log_debug(f"_process_msg, jsonMsg: {jsonMsg}") # Perform the request using HPI and get the response hpi_response = self._HPI_actuator.perform_request(jsonMsg).strip() self._log_debug(f"_process_msg, hpi_response: {hpi_response}") # Simplify success message as external apps don't care about details if "Success" in hpi_response: hpi_response = "Successful" json_msg = AckResponseMsg(node_request, hpi_response, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) elif component == "RESE": # HPI related operations are not supported in VM environment. if self._is_env_vm(): logger.warn("HPI operations are not supported in current environment") return # Query the Zope GlobalSiteManager for an object implementing the IHPI actuator if self._HPI_actuator is None: from actuators.Ihpi import IHPI # Load HPIActuator class HPI_actuator_class = self._queryUtility(IHPI) # Instantiate HPIActuator only if class is loaded if HPI_actuator_class: self._HPI_actuator = HPI_actuator_class(self._conf_reader) else: logger.warn("HPIActuator not loaded") if self._product.lower() in [x.lower() for x in enabled_products]: json_msg = AckResponseMsg(node_request, NodeControllerMsgHandler.UNSUPPORTED_REQUEST, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) return self._log_debug(f"_process_msg, _HPI_actuator name: {self._HPI_actuator.name()}") # Parse out the drive to power cycle drive_request = node_request[13:].strip() self._log_debug(f"perform_request, drive to power cycle: {drive_request}") # Append POWER_OFF and then POWER_ON to notify HPI actuator of desired state jsonMsg["actuator_request_type"]["node_controller"]["node_request"] = \ f"DISK: set {drive_request} POWER_OFF" self._log_debug(f"_process_msg, jsonMsg: {jsonMsg}") # Perform the request using HPI and get the response hpi_response = self._HPI_actuator.perform_request(jsonMsg).strip() self._log_debug(f"_process_msg, hpi_response: {hpi_response}") # Check for success and power the disk back on if "Success" in hpi_response: # Append POWER_ON to notify HPI actuator of desired state jsonMsg["actuator_request_type"]["node_controller"]["node_request"] = \ f"DISK: set {drive_request} POWER_ON" self._log_debug(f"_process_msg, jsonMsg: {jsonMsg}") # Perform the request using HPI and get the response hpi_response = self._HPI_actuator.perform_request(jsonMsg).strip() self._log_debug(f"_process_msg, hpi_response: {hpi_response}") # Simplify success message as external apps don't care about details if "Success" in hpi_response: hpi_response = "Successful" json_msg = AckResponseMsg(node_request, hpi_response, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) elif component == "HDPA": # If the state is INITIALIZED, We can assume that actuator is # ready to perform operation. if actuator_state_manager.is_initialized("Hdparm"): logger.info(f"_process_msg, Hdparm_actuator name: {self._hdparm_actuator.name()}") # Perform the hdparm request on the node and get the response hdparm_response = self._hdparm_actuator.perform_request(jsonMsg).strip() self._log_debug(f"_process_msg, hdparm_response: {hdparm_response}") json_msg = AckResponseMsg(node_request, hdparm_response, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) # If the state is INITIALIZING, need to send message elif actuator_state_manager.is_initializing("Hdparm"): # This state will not be reached. Kept here for consistency. logger.info("Hdparm actuator is initializing") busy_json_msg = AckResponseMsg( node_request, "BUSY", uuid, error_no=errno.EBUSY).getJson() self._write_internal_msgQ( "RabbitMQegressProcessor", busy_json_msg) elif actuator_state_manager.is_imported("Hdparm"): # This case will be for first request only. Subsequent # requests will go to INITIALIZED state case. logger.info("Hdparm actuator is imported and initializing") # Query the Zope GlobalSiteManager for an object # implementing the hdparm actuator. from actuators.Ihdparm import IHdparm actuator_state_manager.set_state( "Hdparm", actuator_state_manager.INITIALIZING) hdparm_actuator_class = self._queryUtility(IHdparm) if hdparm_actuator_class: # NOTE: Instantiation part should not time consuming # otherwise NodeControllerMsgHandler will get block and will # not be able serve any subsequent requests. This applies # to instantiation of evey actuator. self._hdparm_actuator = hdparm_actuator_class() self._log_debug(f"_process_msg, _hdparm_actuator name: {self._hdparm_actuator.name()}") # Perform the hdparm request on the node and get the response hdparm_response = self._hdparm_actuator.perform_request(jsonMsg).strip() self._log_debug(f"_process_msg, hdparm_response: {hdparm_response}") json_msg = AckResponseMsg(node_request, hdparm_response, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) actuator_state_manager.set_state( "Hdparm", actuator_state_manager.INITIALIZED) else: logger.info("Hdparm actuator is not instantiated") # If there is no entry for actuator in table, We can assume # that it is not loaded for some reason. else: logger.info("Hdparm actuator is not loaded or not supported") elif component == "SMAR": # Parse out the drive request field in json msg node_request = jsonMsg.get("actuator_request_type").get("node_controller").get("node_request") drive_request = node_request[12:].strip() self._log_debug(f"perform_request, drive: {drive_request}") # If the drive field is an asterisk then send all the smart results for all drives available if drive_request == "*": # Send the event to SystemdWatchdog to schedule SMART test internal_json_msg = json.dumps( {"sensor_request_type" : "disk_smart_test", "serial_number" : "*", "node_request" : self.host_id, "uuid" : uuid }) self._write_internal_msgQ("SystemdWatchdog", internal_json_msg) return # Put together a message to get the serial number of the drive using hdparm tool if drive_request.startswith("/"): serial_number, error = self._retrieve_serial_number(drive_request) # Send error response back on ack channel if error != "": json_msg = AckResponseMsg(node_request, error, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) return else: if self._smartctl_actuator is None: from actuators.Ismartctl import ISmartctl smartctl_actuator_class = self._queryUtility(ISmartctl) if smartctl_actuator_class: self._smartctl_actuator = self._queryUtility(ISmartctl)() self._log_debug("_process_msg, _smart_actuator name: %s" % self._smartctl_actuator.name()) else: logger.error(" No module Smartctl is present to load") serial_compare = self._smartctl_actuator._check_serial_number(drive_request) if not serial_compare: json_msg = AckResponseMsg(node_request, "Drive Not Found", uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) return else: serial_number = drive_request # Send the event to SystemdWatchdog to schedule SMART test internal_json_msg = json.dumps( {"sensor_request_type" : "disk_smart_test", "serial_number" : serial_number, "node_request" : node_request, "uuid" : uuid }) self._write_internal_msgQ("SystemdWatchdog", internal_json_msg) elif component == "DRVM": # Requesting the current status from drivemanager # Parse out the drive request field in json msg node_request = jsonMsg.get("actuator_request_type").get("node_controller").get("node_request") drive_request = node_request[15:].strip() self._log_debug(f"perform_request, drive: {drive_request}") # If the drive field is an asterisk then send all the drivemanager results for all drives available if drive_request == "*": # Send a message to the disk message handler to lookup the drivemanager status and send it out internal_json_msg = json.dumps( {"sensor_request_type" : "drvmngr_status", "serial_number" : "*", "node_request" : self.host_id, "uuid" : uuid }) # Send the event to disk message handler to generate json message self._write_internal_msgQ(DiskMsgHandler.name(), internal_json_msg) return # Put together a message to get the serial number of the drive using hdparm tool if drive_request.startswith("/"): serial_number, error = self._retrieve_serial_number(drive_request) # Send error response back on ack channel if error != "": json_msg = AckResponseMsg(node_request, error, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) return else: serial_number = drive_request # Send a message to the disk message handler to lookup the smart status and send it out internal_json_msg = json.dumps( {"sensor_request_type" : "drvmngr_status", "serial_number" : serial_number, "node_request" : node_request, "uuid" : uuid }) # Send the event to disk message handler to generate json message self._write_internal_msgQ(DiskMsgHandler.name(), internal_json_msg) elif component == "HPI_": # Requesting the current status from HPI data # Parse out the drive request field in json msg if self._is_env_vm(): logger.warn("HPI operations are not supported in current environment") return if self.setup == 'cortx': logger.warn("HPIMonitor not loaded") json_msg = AckResponseMsg(node_request, NodeControllerMsgHandler.UNSUPPORTED_REQUEST, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) return node_request = jsonMsg.get("actuator_request_type").get("node_controller").get("node_request") drive_request = node_request[11:].strip() self._log_debug(f"perform_request, drive: {drive_request}") # If the drive field is an asterisk then send all the hpi results for all drives available if drive_request == "*": # Send a message to the disk message handler to lookup the hpi status and send it out internal_json_msg = json.dumps( {"sensor_request_type" : "hpi_status", "serial_number" : "*", "node_request" : self.host_id, "uuid" : uuid }) # Send the event to disk message handler to generate json message self._write_internal_msgQ(DiskMsgHandler.name(), internal_json_msg) return # Put together a message to get the serial number of the drive using hdparm tool if drive_request.startswith("/"): serial_number, error = self._retrieve_serial_number(drive_request) # Send error response back on ack channel if error != "": json_msg = AckResponseMsg(node_request, error, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) return else: serial_number = drive_request # Send a message to the disk message handler to lookup the smart status and send it out internal_json_msg = json.dumps( {"sensor_request_type" : "hpi_status", "serial_number" : serial_number, "node_request" : node_request, "uuid" : uuid }) # Send the event to disk message handler to generate json message self._write_internal_msgQ(DiskMsgHandler.name(), internal_json_msg) elif component == "SIMU": # Requesting to simulate an event # Parse out the simulated request field node_request = jsonMsg.get("actuator_request_type").get("node_controller").get("node_request") sim_request = node_request[9:].strip().split(" ") self._log_debug(f"perform_request, sim_request: {str(sim_request)}") # Put together a message to get the serial number of the drive using hdparm tool if sim_request[1].startswith("/"): serial_number, error = self._retrieve_serial_number(sim_request[1]) # Send error response back on ack channel if error != "": json_msg = AckResponseMsg(node_request, error, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) return else: serial_number = sim_request[1] # SMART simulation requests are sent to SystemdWatchdog if sim_request[0] == "SMART_FAILURE": logger.info(f"NodeControllerMsgHandler, simulating SMART_FAILURE on drive: {serial_number}") internal_json_msg = json.dumps( {"sensor_request_type" : "simulate_failure", "serial_number" : serial_number, "node_request" : sim_request[0], "uuid" : uuid }) # Send the event to SystemdWatchdog to handle it from here self._write_internal_msgQ("SystemdWatchdog", internal_json_msg) else: # Send a message to the disk message handler to handle simulation request internal_json_msg = json.dumps( {"sensor_request_type" : "sim_event", "serial_number" : serial_number, "node_request" : sim_request[0], "uuid" : uuid }) # Send the event to disk message handler to generate json message self._write_internal_msgQ(DiskMsgHandler.name(), internal_json_msg) elif component == "NDHW": # NDHW Stands for Node HW. try: # Load and Instantiate the Actuator for the first request if self._NodeHW_actuator is None: from actuators.impl.generic.node_hw import NodeHWactuator from framework.utils.ipmi_client import IpmiFactory self.ipmi_client_name = self._conf_reader._get_value_with_default( self.NODE_HW_ACTUATOR, self.IPMI_IMPLEMENTOR, "ipmitool") ipmi_factory = IpmiFactory() ipmi_client = \ ipmi_factory.get_implementor(self.ipmi_client_name) # Instantiate NodeHWactuator only if class is loaded if ipmi_client is not None: self._NodeHW_actuator = NodeHWactuator(ipmi_client, self._conf_reader) self._NodeHW_actuator.initialize() else: logger.error(f"IPMI client: '{self.ipmi_client_name}' doesn't exist") return node_request = jsonMsg.get("actuator_request_type") # Perform the NodeHW request on the node and get the response #TODO: Send message to Ack as well as Sensor in their respective channel. node_hw_response = self._NodeHW_actuator.perform_request(node_request) self._log_debug(f"_process_msg, node_hw_response: {node_hw_response}") json_msg = NodeHwAckResponseMsg(node_request, node_hw_response, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) except ImportError as e: logger.error(f"Modules could not be loaded: {e}") return except Exception as e: logger.error(f"NodeControllerMsgHandler, _process_msg, Exception in request handling: {e}") return else: response = f"NodeControllerMsgHandler, _process_msg, unknown node controller msg: {node_request}" self._log_debug(response) json_msg = AckResponseMsg(node_request, response, uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
def _process_msg(self, ch, method, properties, body): """Parses the incoming message and hands off to the appropriate module""" ingressMsg = {} uuid = None try: if isinstance(body, dict) is False: ingressMsg = json.loads(body) else: ingressMsg = body # Authenticate message using username and signature fields username = ingressMsg.get("username") signature = ingressMsg.get("signature") message = ingressMsg.get("message") uuid = ingressMsg.get("uuid") msg_len = len(message) + 1 if uuid is None: uuid = "N/A" if use_security_lib and \ SSPL_SEC.sspl_verify_message(msg_len, str(message), username, signature) != 0: logger.warn( "RabbitMQingressProcessor, Authentication failed on message: %s" % ingressMsg) return # Get the incoming message type if message.get("actuator_request_type") is not None: msgType = message.get("actuator_request_type") # Validate against the actuator schema validate(ingressMsg, self._actuator_schema) elif message.get("sensor_request_type") is not None: msgType = message.get("sensor_request_type") # Validate against the sensor schema validate(ingressMsg, self._sensor_schema) else: # We only handle incoming actuator and sensor requests, ignore # everything else. return # Check for debugging being activated in the message header self._check_debug(message) self._log_debug("_process_msg, ingressMsg: %s" % ingressMsg) # Hand off to appropriate actuator message handler if msgType.get("logging") is not None: self._write_internal_msgQ("LoggingMsgHandler", message) elif msgType.get("thread_controller") is not None: self._write_internal_msgQ("ThreadController", message) elif msgType.get("service_controller") is not None: self._write_internal_msgQ("ServiceMsgHandler", message) elif msgType.get("node_controller") is not None: self._write_internal_msgQ("NodeControllerMsgHandler", message) elif msgType.get("storage_enclosure") is not None: self._write_internal_msgQ("RealStorActuatorMsgHandler", message) # Hand off to appropriate sensor message handler elif msgType.get("node_data") is not None: self._write_internal_msgQ("NodeDataMsgHandler", message) elif msgType.get("enclosure_alert") is not None: self._write_internal_msgQ("RealStorEnclMsgHandler", message) elif msgType.get("storage_enclosure") is not None: self._write_internal_msgQ("RealStorActuatorMsgHandler", message) # ... handle other incoming messages that have been validated else: # Send ack about not finding a msg handler ack_msg = AckResponseMsg("Error Processing Message", "Message Handler Not Found", uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), ack_msg) # Acknowledge message was received self._connection.ack(ch, delivery_tag=method.delivery_tag) except Exception as ex: logger.error( "RabbitMQingressProcessor, _process_msg unrecognized message: %r" % ingressMsg) ack_msg = AckResponseMsg("Error Processing Msg", "Msg Handler Not Found", uuid).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), ack_msg)
def _process_msg(self, ch, method, properties, body): """Parses the incoming message and hands off to the appropriate module""" ingressMsg = {} try: if isinstance(body, dict) is False: ingressMsg = json.loads(body) else: ingressMsg = body # Authenticate message using username and signature fields username = ingressMsg.get("username") signature = ingressMsg.get("signature") message = ingressMsg.get("message") uuid = message.get("sspl_ll_msg_header").get("uuid") msg_len = len(message) + 1 if uuid is None: uuid = "N/A" if use_security_lib and \ SSPL_SEC.sspl_verify_message(msg_len, str(message), username, signature) != 0: logger.warn( "PlaneCntrlRMQingressProcessor, Authentication failed on message: %s" % ingressMsg) return # Get the incoming message type if message.get("actuator_request_type") is not None: msgType = message.get("actuator_request_type") # Validate against the actuator schema validate(ingressMsg, self._actuator_schema) elif message.get("sensor_request_type") is not None: msgType = message.get("sensor_request_type") # Validate against the sensor schema validate(ingressMsg, self._sensor_schema) else: # We only handle incoming actuator and sensor requests, ignore everything else return # Check for debugging being activated in the message header self._check_debug(message) self._log_debug("_process_msg, ingressMsg: %s" % ingressMsg) # Hand off to appropriate actuator message handler if msgType.get("plane_controller") is not None: command = message.get("actuator_request_type").get( "plane_controller").get("command") # For a job status request forward over to PlaneCntrlRMQegressProcessor if command is not None and \ command == "job_status": job_uuid = message.get("actuator_request_type").get( "plane_controller").get("arguments").get("uuid", None) node_id = None if message.get("actuator_request_type").get("plane_controller").get("parameters") is not None and \ message.get("actuator_request_type").get("plane_controller").get("parameters").get("node_id") is not None: node_id = message.get("actuator_request_type").get( "plane_controller").get("parameters").get( "node_id") # node_id set to None is a broadcast otherwise see if it applies to this node if node_id is None or \ self._hostname in str(node_id): self._process_job_status(uuid, job_uuid) else: self._write_internal_msgQ("PlaneCntrlMsgHandler", message) # Handle restarting of internal threads elif msgType.get("thread_controller") is not None: node_id = None if message.get("actuator_request_type").get("thread_controller").get("parameters") is not None and \ message.get("actuator_request_type").get("thread_controller").get("parameters").get("node_id") is not None: node_id = message.get("actuator_request_type").get( "thread_controller").get("parameters").get( "node_id", None) # node_id set to None is a broadcast otherwise see if it applies to this node if node_id is None or \ self._hostname in str(node_id): self._write_internal_msgQ("ThreadController", message) # ... handle other incoming messages that have been validated else: # Log a msg about not being able to process the message logger.info( "PlaneCntrlRMQingressProcessor, _process_msg, unrecognized message: %s" % str(ingressMsg)) except Exception as ex: logger.exception( "PlaneCntrlRMQingressProcessor, _process_msg unrecognized message: %r" % ingressMsg) ack_msg = AckResponseMsg("Error Processing Msg", "Msg Handler Not Found", uuid).getJson() self._write_internal_msgQ(PlaneCntrlRMQegressProcessor.name(), ack_msg) # Acknowledge message was received self._connection.ack(ch, delivery_tag=method.delivery_tag)
def _process_msg(self, body): """Parses the incoming message and hands off to the appropriate module""" ingressMsg = {} uuid = None try: if isinstance(body, dict) is False: ingressMsg = json.loads(body) else: ingressMsg = body # Authenticate message using username and signature fields username = ingressMsg.get("username") signature = ingressMsg.get("signature") message = ingressMsg.get("message") uuid = ingressMsg.get("uuid") msg_len = len(message) + 1 if uuid is None: uuid = "N/A" if use_security_lib and \ SSPL_SEC.sspl_verify_message(msg_len, str(message), username, signature) != 0: logger.warn( "IngressProcessor, Authentication failed on message: %s" % ingressMsg) return logger.debug("_process_msg, ingressMsg: %s" % ingressMsg) # Get the incoming message type if message.get("actuator_request_type") is not None: msgType = message.get("actuator_request_type") # Validate against the actuator schema validate(ingressMsg, self._actuator_schema) # Compare target_node_id from the request to determine # if request is meant for the current node target_node_id = message.get("target_node_id") if target_node_id is None: logger.warn( "Required attribute target_node_id is missing " "from actuator request %s, IGNORING!!" % (msgType)) return elif target_node_id == self._node_id: self._send_to_msg_handler(msgType, message, uuid) else: logger.debug( "Node identifier mismatch, actuator request ignored.") return elif message.get("sensor_request_type") is not None: msgType = message.get("sensor_request_type") # Validate against the sensor schema validate(ingressMsg, self._sensor_schema) self._send_to_msg_handler(msgType, message, uuid) else: # We only handle incoming actuator and sensor requests, ignore # everything else. return except Exception as ex: logger.error( "IngressProcessor, _process_msg failed to recognize " "message: %r with error %r" % (ingressMsg, ex)) ack_msg = AckResponseMsg("Error Processing Msg", "Msg Handler Not Found", uuid).getJson() self._write_internal_msgQ(EgressProcessor.name(), ack_msg)
def _process_msg(self, jsonMsg): """Parses the incoming message and hands off to the appropriate logger """ self._log_debug(f"_process_msg, jsonMsg: {jsonMsg}") if isinstance(jsonMsg, dict) is False: jsonMsg = json.loads(jsonMsg) # Parse out the uuid so that it can be sent back in Ack message uuid = None if jsonMsg.get("sspl_ll_msg_header") is not None and \ jsonMsg.get("sspl_ll_msg_header").get("uuid") is not None: uuid = jsonMsg.get("sspl_ll_msg_header").get("uuid") self._log_debug(f"_processMsg, uuid: {uuid}") # Handle service start, stop, restart, status requests if jsonMsg.get("actuator_request_type").get( "service_controller") is not None: self._log_debug("_processMsg, msg_type: service_controller") service_name = jsonMsg.get("actuator_request_type") \ .get("service_controller").get("service_name") service_request = jsonMsg.get("actuator_request_type") \ .get("service_controller").get("service_request") request = f"{service_request}:{service_name}" # If the state is INITIALIZED, We can assume that actuator is # ready to perform operation. if actuator_state_manager.is_initialized("Service"): self._log_debug( f"_process_msg, service_actuator name: {self._service_actuator.name()}" ) self._execute_request(self._service_actuator, jsonMsg, uuid) # If the state is INITIALIZING, need to send message elif actuator_state_manager.is_initializing("Service"): # This state will not be reached. Kept here for consistency. logger.info("Service actuator is initializing") busy_json_msg = AckResponseMsg(request, "BUSY", uuid, error_no=errno.EBUSY).getJson() self._write_internal_msgQ("RabbitMQegressProcessor", busy_json_msg) elif actuator_state_manager.is_imported("Service"): # This case will be for first request only. Subsequent # requests will go to INITIALIZED state case. logger.info("Service actuator is imported and initializing") from actuators.IService import IService actuator_state_manager.set_state( "Service", actuator_state_manager.INITIALIZING) service_actuator_class = self._query_utility(IService) if service_actuator_class: # NOTE: Instantiation part should not time consuming # otherwise ServiceMsgHandler will get block and will # not be able serve any subsequent requests. This applies # to instantiation of evey actuator. self._service_actuator = service_actuator_class() logger.info( f"_process_msg, service_actuator name: {self._service_actuator.name()}" ) self._execute_request(self._service_actuator, jsonMsg, uuid) actuator_state_manager.set_state( "Service", actuator_state_manager.INITIALIZED) else: logger.info("Service actuator is not instantiated") # If there is no entry for actuator in table, We can assume # that it is not loaded for some reason. else: logger.warn("Service actuator is not loaded or not supported") # Handle events generated by the service watchdogs elif jsonMsg.get("actuator_request_type").get( "service_watchdog_controller") is not None: self._log_debug( "_processMsg, msg_type: service_watchdog_controller") # Parse out values to be sent service_name = jsonMsg.get("actuator_request_type").get( "service_watchdog_controller").get("service_name") state = jsonMsg.get("actuator_request_type").get( "service_watchdog_controller").get("state") prev_state = jsonMsg.get("actuator_request_type").get( "service_watchdog_controller").get("previous_state") substate = jsonMsg.get("actuator_request_type").get( "service_watchdog_controller").get("substate") prev_substate = jsonMsg.get("actuator_request_type").get( "service_watchdog_controller").get("previous_substate") pid = jsonMsg.get("actuator_request_type").get( "service_watchdog_controller").get("pid") prev_pid = jsonMsg.get("actuator_request_type").get( "service_watchdog_controller").get("previous_pid") # Pull out the service_request and if it's equal to "status" then get current status (state, substate) service_request = jsonMsg.get("actuator_request_type").get( "service_watchdog_controller").get("service_request") if service_request != "None": # Query the Zope GlobalSiteManager for an object implementing IService if self._service_actuator is None: from actuators.IService import IService self._service_actuator = self._query_utility(IService)() self._log_debug( f"_process_msg, service_actuator name: {self._service_actuator.name()}" ) service_name, state, substate = self._service_actuator.perform_request( jsonMsg) self._log_debug( f"_processMsg, service_name: {service_name}, state: {state}, substate: {substate}" ) self._log_debug( f"_processMsg, prev state: {prev_state}, prev substate: {prev_substate}" ) # Create a service watchdog message and send it out jsonMsg = ServiceWatchdogMsg(service_name, state, prev_state, substate, prev_substate, pid, prev_pid).getJson() self._write_internal_msgQ("RabbitMQegressProcessor", jsonMsg) # Create an IEM if the resulting service state is failed if "fail" in state.lower() or \ "fail" in substate.lower(): json_data = { "service_name": service_name, "state": state, "previous_state": prev_state, "substate": substate, "previous_substate": prev_substate, "pid": pid, "previous_pid": prev_pid } internal_json_msg = json.dumps({ "actuator_request_type": { "logging": { "log_level": "LOG_WARNING", "log_type": "IEM", "log_msg": f"IEC: 020003001: Service entered a Failed state : {json.dumps(json_data, sort_keys=True)}" } } }) # Send the event to logging msg handler to send IEM message to journald self._write_internal_msgQ(LoggingMsgHandler.name(), internal_json_msg)