def _execute_raid_request(self, node_request, actuator_instance, json_msg,
uuid):
"""Performs a RAID request by calling perform_request method of a RAID
actuator.
"""
# Perform the RAID request on the node and get the response
raid_response = actuator_instance.perform_request(json_msg).strip()
self._log_debug(f"_process_msg, raid_response: {raid_response}")
json_msg = AckResponseMsg(node_request, raid_response, uuid).getJson()
self._write_internal_msgQ(EgressProcessor.name(), json_msg)
# Restart openhpid to update HPI data only if it is a H/W environment
if self.setup in ["hw", "ssu"]:
self._log_debug("restarting openhpid service to update HPI data")
if "assemble" in json_msg.get("actuator_request_type").get(
"node_controller").get("node_request").lower():
internal_json_msg = json.dumps({
"actuator_request_type": {
"service_controller": {
"service_name": "openhpid.service",
"service_request": "restart"
}
}
})
self._write_internal_msgQ(ServiceMsgHandler.name(),
internal_json_msg)
def _send_to_msg_handler(self, msgType, message, uuid):
# Hand off to appropriate actuator message handler
if msgType.get("thread_controller") is not None:
self._write_internal_msgQ("ThreadController", message)
elif msgType.get("service_controller") is not None:
self._write_internal_msgQ("ServiceMsgHandler", message)
elif msgType.get("node_controller") is not None:
self._write_internal_msgQ("NodeControllerMsgHandler", message)
elif msgType.get("storage_enclosure") is not None:
self._write_internal_msgQ("RealStorActuatorMsgHandler", message)
# Hand off to appropriate sensor message handler
elif msgType.get("node_data") is not None:
self._write_internal_msgQ("NodeDataMsgHandler", message)
elif msgType.get("enclosure_alert") is not None:
self._write_internal_msgQ("RealStorEnclMsgHandler", message)
elif msgType.get("storage_enclosure") is not None:
self._write_internal_msgQ("RealStorActuatorMsgHandler", message)
# ... handle other incoming messages that have been validated
else:
# Send ack about not finding a msg handler
ack_msg = AckResponseMsg("Error Processing Message",
"Message Handler Not Found",
uuid).getJson()
self._write_internal_msgQ(EgressProcessor.name(), ack_msg)
def _process_msg(self, body):
"""Parses the incoming message and hands off to the appropriate module"""
ingressMsg = {}
uuid = None
try:
if isinstance(body, dict) is False:
ingressMsg = json.loads(body)
else:
ingressMsg = body
# Authenticate message using username and signature fields
username = ingressMsg.get("username")
signature = ingressMsg.get("signature")
message = ingressMsg.get("message")
uuid = ingressMsg.get("uuid")
msg_len = len(message) + 1
if uuid is None:
uuid = "N/A"
if use_security_lib and \
SSPL_SEC.sspl_verify_message(msg_len, str(message),
username, signature) != 0:
logger.warn(
"IngressProcessor, Authentication failed on message: %s" %
ingressMsg)
return
# Get the incoming message type
if message.get("actuator_request_type") is not None:
msgType = message.get("actuator_request_type")
# Validate against the actuator schema
validate(ingressMsg, self._actuator_schema)
elif message.get("sensor_request_type") is not None:
msgType = message.get("sensor_request_type")
# Validate against the sensor schema
validate(ingressMsg, self._sensor_schema)
else:
# We only handle incoming actuator and sensor requests, ignore
# everything else.
return
# Check for debugging being activated in the message header
self._check_debug(message)
self._log_debug("_process_msg, ingressMsg: %s" % ingressMsg)
self._send_to_msg_handler(msgType, message, uuid)
except Exception as ex:
logger.error(
"IngressProcessor, _process_msg unrecognized message: %r" %
ingressMsg)
ack_msg = AckResponseMsg("Error Processing Msg",
"Msg Handler Not Found", uuid).getJson()
self._write_internal_msgQ(EgressProcessor.name(), ack_msg)
def _process_job_status(self, uuid, job_uuid):
"""Send the current job status requested"""
self._log_debug(
"_process_job_status, job_status requested on uuid: %s" % job_uuid)
try:
# Get a copy of the job tasks in the PlaneCntrlMsgHandler msg queue pending being worked
plane_cntrl_jobs = self._get_msgQ_copy("PlaneCntrlMsgHandler")
response = "Not Found"
# See if the requested uuid is in the list of pending jobs
for plane_cntrl_job in plane_cntrl_jobs:
self._log_debug("_process_job_status, plane_cntrl_job: %s" %
str(plane_cntrl_job))
if job_uuid in str(plane_cntrl_job):
response = "In Queue"
break
ack_type = {}
ack_type["hostname"] = gethostname()
ack_type["command"] = "job_status"
ack_type["arguments"] = str(job_uuid)
# The uuid is either not found or it's in the queue to be worked
ack_msg = AckResponseMsg(json.dumps(ack_type), response,
uuid).getJson()
self._write_internal_msgQ(PlaneCntrlRMQegressProcessor.name(),
ack_msg)
except Exception as ex:
logger.exception(
"PlaneCntrlRMQingressProcessor, _process_job_status exception: %s"
% str(ex))
def _send_response(self, status, hostname, response, errors):
"""Transmit the response back as an Ack json msg"""
ack_type = {}
ack_type["hostname"], ack_type["command"], ack_type["parameters"], ack_type["status"], \
ack_type["errors"] = str(hostname, 'utf-8'), self._command, self._parameters, \
status, str(errors, 'utf-8')
ack_msg = AckResponseMsg(json.dumps(ack_type), \
str(response), self._uuid).getJson()
self._write_internal_msgQ(PlaneCntrlRMQegressProcessor.name(), ack_msg)
def _process_msg(self, jsonMsg):
"""Parses the incoming message and hands off to the appropriate logger"""
self._log_debug(f"_process_msg, jsonMsg: {jsonMsg}")
if isinstance(jsonMsg, dict) is False:
jsonMsg = json.loads(jsonMsg)
uuid = None
if jsonMsg.get("sspl_ll_msg_header") is not None and \
jsonMsg.get("sspl_ll_msg_header").get("uuid") is not None:
uuid = jsonMsg.get("sspl_ll_msg_header").get("uuid")
self._log_debug(f"_process_msg, uuid: {uuid}")
log_type = jsonMsg.get("actuator_request_type").get("logging").get("log_type")
result = "N/A"
# Disabled for LDR_R1
# if log_type == "IEM":
# self._log_debug("_process_msg, msg_type: IEM")
# if self._iem_log_locally == "true":
# result = self._iem_logger.log_msg(jsonMsg)
# self._log_debug(f"Log IEM results: {result}")
if log_type == "HDS":
# Retrieve the serial number of the drive
self._log_debug("_process_msg, msg_type: HDS")
log_msg = jsonMsg.get("actuator_request_type").get("logging").get("log_msg")
# Parse out the json data section in the IEM and replace single quotes with double
json_data = json.loads(str('{' + log_msg.split('{')[1]).replace("'", '"'))
serial_number = json_data.get("serial_number")
status = json_data.get("status")
reason = json_data.get("reason")
self._log_debug(f"_processMsg, serial_number: {serial_number}, status:{status}, reason: {reason}")
# Send a message to the disk manager handler to create and transmit json msg
internal_json_msg = json.dumps(
{"sensor_response_type" : "disk_status_HDS",
"object_path" : "HDS",
"status" : status,
"reason" : reason,
"serial_number" : serial_number
})
# Send the event to disk message handler to generate json message
self._write_internal_msgQ("DiskMsgHandler", internal_json_msg)
# Disabled for LDR_R1
# Hand off to the IEM logger
# result = self._iem_logger.log_msg(jsonMsg)
# Send ack about logging msg
ack_msg = AckResponseMsg(log_type, result, uuid).getJson()
self._write_internal_msgQ(EgressProcessor.name(), ack_msg)
def _process_msg(self, jsonMsg):
"""Parses the incoming message and handles appropriately"""
self._log_debug(f"_process_msg, jsonMsg: {jsonMsg}")
if isinstance(jsonMsg, dict) is False:
jsonMsg = json.loads(jsonMsg)
# Parse out the uuid so that it can be sent back in Ack message
uuid = None
if jsonMsg.get("sspl_ll_msg_header").get("uuid") is not None:
uuid = jsonMsg.get("sspl_ll_msg_header").get("uuid")
self._log_debug(f"_processMsg, uuid: {uuid}")
if jsonMsg.get("actuator_request_type").get("node_controller").get("node_request") is not None:
node_request = jsonMsg.get("actuator_request_type").get("node_controller").get("node_request")
self._log_debug(f"_processMsg, node_request: {node_request}")
# Parse out the component field in the node_request
component = node_request[0:4]
# Handle generic command line requests
if component == 'SSPL':
# Query the Zope GlobalSiteManager for an object implementing the MOTR actuator
if self._command_line_actuator is None:
from actuators.Icommand_line import ICommandLine
command_line_actuator_class = self._queryUtility(ICommandLine)
# Instantiate CommandLine Actuator only if class is loaded
if command_line_actuator_class:
self._command_line_actuator = command_line_actuator_class(self._conf_reader)
else:
logger.warn("CommandLine Actuator not loaded")
json_msg = AckResponseMsg(node_request, NodeControllerMsgHandler.UNSUPPORTED_REQUEST, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
return
# Perform the request and get the response
command_line_response = self._command_line_actuator.perform_request(jsonMsg).strip()
self._log_debug(f"_process_msg, command line response: {command_line_response}")
json_msg = AckResponseMsg(node_request, command_line_response, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
# Handle LED effects using the HPI actuator
elif component == "LED:":
# HPI related operations are not supported in VM environment.
if self._is_env_vm():
logger.warn("HPI operations are not supported in current environment")
return
# Query the Zope GlobalSiteManager for an object implementing the IHPI actuator
if self._HPI_actuator is None:
from actuators.Ihpi import IHPI
# Load HPIActuator class
HPI_actuator_class = self._queryUtility(IHPI)
# Instantiate HPIActuator only if class is loaded
if HPI_actuator_class:
self._HPI_actuator = HPI_actuator_class(self._conf_reader)
else:
logger.warn("HPIActuator not loaded")
if self._product.lower() in [x.lower() for x in enabled_products]:
json_msg = AckResponseMsg(node_request, NodeControllerMsgHandler.UNSUPPORTED_REQUEST, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
return
self._log_debug(f"_process_msg, _HPI_actuator name: {self._HPI_actuator.name()}")
# Perform the request using HPI and get the response
hpi_response = self._HPI_actuator.perform_request(jsonMsg).strip()
self._log_debug(f"_process_msg, hpi_response: {hpi_response}")
json_msg = AckResponseMsg(node_request, hpi_response, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
# Set the Bezel LED color using the GEM interface
elif component == "BEZE":
# Query the Zope GlobalSiteManager for an object implementing the IGEM actuator
if self._GEM_actuator is None:
self._GEM_actuator = self._queryUtility(IGEM)(self._conf_reader)
self._log_debug(f"_process_msg, _GEM_actuator name: {self._GEM_actuator.name()}")
# Perform the request using GEM and get the response
gem_response = self._GEM_actuator.perform_request(jsonMsg).strip()
self._log_debug(f"_process_msg, gem_response: {gem_response}")
json_msg = AckResponseMsg(node_request, gem_response, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
elif component == "PDU:":
# Query the Zope GlobalSiteManager for an object implementing the IPDU actuator
if self._PDU_actuator is None:
from actuators.Ipdu import IPDU
PDU_actuator_class = self._queryUtility(IPDU)
# Instantiate RaritanPDU Actuator only if class is loaded
if PDU_actuator_class:
self._PDU_actuator = PDU_actuator_class(self._conf_reader)
else:
logger.warn("RaritanPDU Actuator not loaded")
json_msg = AckResponseMsg(node_request, NodeControllerMsgHandler.UNSUPPORTED_REQUEST, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
return
# Perform the request on the PDU and get the response
pdu_response = self._PDU_actuator.perform_request(jsonMsg).strip()
self._log_debug(f"_process_msg, pdu_response: {pdu_response}")
json_msg = AckResponseMsg(node_request, pdu_response, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
elif component == "RAID":
# If the state is INITIALIZED, We can assume that actuator is
# ready to perform operation.
if actuator_state_manager.is_initialized("RAIDactuator"):
self._log_debug(f"_process_msg, _RAID_actuator name: {self._RAID_actuator.name()}")
self._execute_raid_request(
node_request, self._RAID_actuator, jsonMsg, uuid)
# If the state is INITIALIZING, need to send message
elif actuator_state_manager.is_initializing("RAIDactuator"):
# This state will not be reached. Kept here for consistency.
logger.info("RAID actuator is initializing")
busy_json_msg = AckResponseMsg(
node_request, "BUSY", uuid, error_no=errno.EBUSY).getJson()
self._write_internal_msgQ(
"RabbitMQegressProcessor", busy_json_msg)
elif actuator_state_manager.is_imported("RAIDactuator"):
# This case will be for first request only. Subsequent
# requests will go to INITIALIZED state case.
logger.info("RAID actuator is imported and initializing")
from actuators.Iraid import IRAIDactuator
actuator_state_manager.set_state(
"RAIDactuator", actuator_state_manager.INITIALIZING)
# Query the Zope GlobalSiteManager for an object implementing the IRAIDactuator
raid_actuator_class = self._queryUtility(IRAIDactuator)
if raid_actuator_class:
# NOTE: Instantiation part should not time consuming
# otherwise NodeControllerMsgHandler will get block
# and will not be able serve any subsequent requests.
# This applies to instantiation of evey actuator.
self._RAID_actuator = raid_actuator_class()
logger.info(f"_process_msg, _RAID_actuator name: {self._RAID_actuator.name()}")
self._execute_raid_request(
node_request, self._RAID_actuator, jsonMsg, uuid)
actuator_state_manager.set_state(
"RAIDactuator", actuator_state_manager.INITIALIZED)
else:
logger.warn("RAID actuator is not instantiated")
# If there is no entry for actuator in table, We can assume
# that it is not loaded for some reason.
else:
logger.warn("RAID actuator is not loaded or not supported")
elif component == "IPMI":
# Query the Zope GlobalSiteManager for an object implementing the IPMI actuator
if self._IPMI_actuator is None:
from actuators.Iipmi import Iipmi
IPMI_actuator_class = self._queryUtility(Iipmi)
# Instantiate IPMI Actuator only if class is loaded
if IPMI_actuator_class:
self._IPMI_actuator = IPMI_actuator_class(self._conf_reader)
else:
logger.warn("IPMI Actuator not loaded")
json_msg = AckResponseMsg(node_request, NodeControllerMsgHandler.UNSUPPORTED_REQUEST, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
return
# Perform the IPMI request on the node and get the response
ipmi_response = self._IPMI_actuator.perform_request(jsonMsg).strip()
self._log_debug(f"_process_msg, ipmi_response: {ipmi_response}")
json_msg = AckResponseMsg(node_request, ipmi_response, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
elif component == "STOP":
# HPI related operations are not supported in VM environment.
if self._is_env_vm():
logger.warn("HPI operations are not supported in current environment")
return
# Query the Zope GlobalSiteManager for an object implementing the IHPI actuator
if self._HPI_actuator is None:
from actuators.Ihpi import IHPI
# Load HPIActuator class
HPI_actuator_class = self._queryUtility(IHPI)
# Instantiate HPIActuator only if class is loaded
if HPI_actuator_class:
self._HPI_actuator = HPI_actuator_class(self._conf_reader)
else:
logger.warn("HPIActuator not loaded")
if self._product.lower() in [x.lower() for x in enabled_products]:
json_msg = AckResponseMsg(node_request, NodeControllerMsgHandler.UNSUPPORTED_REQUEST, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
return
self._log_debug(f"_process_msg, _HPI_actuator name: {self._HPI_actuator.name()}")
# Parse out the drive to stop
drive_request = node_request[12:].strip()
self._log_debug(f"perform_request, drive to stop: {drive_request}")
# Append POWER_OFF to notify HPI actuator of desired state
jsonMsg["actuator_request_type"]["node_controller"]["node_request"] = \
f"DISK: set {drive_request} POWER_OFF"
self._log_debug(f"_process_msg, jsonMsg: {jsonMsg}")
# Perform the request using HPI and get the response
hpi_response = self._HPI_actuator.perform_request(jsonMsg).strip()
self._log_debug(f"_process_msg, hpi_response: {hpi_response}")
# Simplify success message as external apps don't care about details
if "Success" in hpi_response:
hpi_response = "Successful"
json_msg = AckResponseMsg(node_request, hpi_response, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
elif component == "STAR":
# HPI related operations are not supported in VM environment.
if self._is_env_vm():
logger.warn("HPI operations are not supported in current environment")
return
# Query the Zope GlobalSiteManager for an object implementing the IHPI actuator
if self._HPI_actuator is None:
from actuators.Ihpi import IHPI
# Load HPIActuator class
HPI_actuator_class = self._queryUtility(IHPI)
# Instantiate HPIActuator only if class is loaded
if HPI_actuator_class:
self._HPI_actuator = HPI_actuator_class(self._conf_reader)
else:
logger.warn("HPIActuator not loaded")
if self._product.lower() in [x.lower() for x in enabled_products]:
json_msg = AckResponseMsg(node_request, NodeControllerMsgHandler.UNSUPPORTED_REQUEST, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
return
self._log_debug(f"_process_msg, _HPI_actuator name: {self._HPI_actuator.name()}")
# Parse out the drive to start
drive_request = node_request[13:].strip()
self._log_debug(f"perform_request, drive to start: {drive_request}")
# Append POWER_ON to notify HPI actuator of desired state
jsonMsg["actuator_request_type"]["node_controller"]["node_request"] = \
f"DISK: set {drive_request} POWER_ON"
self._log_debug(f"_process_msg, jsonMsg: {jsonMsg}")
# Perform the request using HPI and get the response
hpi_response = self._HPI_actuator.perform_request(jsonMsg).strip()
self._log_debug(f"_process_msg, hpi_response: {hpi_response}")
# Simplify success message as external apps don't care about details
if "Success" in hpi_response:
hpi_response = "Successful"
json_msg = AckResponseMsg(node_request, hpi_response, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
elif component == "RESE":
# HPI related operations are not supported in VM environment.
if self._is_env_vm():
logger.warn("HPI operations are not supported in current environment")
return
# Query the Zope GlobalSiteManager for an object implementing the IHPI actuator
if self._HPI_actuator is None:
from actuators.Ihpi import IHPI
# Load HPIActuator class
HPI_actuator_class = self._queryUtility(IHPI)
# Instantiate HPIActuator only if class is loaded
if HPI_actuator_class:
self._HPI_actuator = HPI_actuator_class(self._conf_reader)
else:
logger.warn("HPIActuator not loaded")
if self._product.lower() in [x.lower() for x in enabled_products]:
json_msg = AckResponseMsg(node_request, NodeControllerMsgHandler.UNSUPPORTED_REQUEST, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
return
self._log_debug(f"_process_msg, _HPI_actuator name: {self._HPI_actuator.name()}")
# Parse out the drive to power cycle
drive_request = node_request[13:].strip()
self._log_debug(f"perform_request, drive to power cycle: {drive_request}")
# Append POWER_OFF and then POWER_ON to notify HPI actuator of desired state
jsonMsg["actuator_request_type"]["node_controller"]["node_request"] = \
f"DISK: set {drive_request} POWER_OFF"
self._log_debug(f"_process_msg, jsonMsg: {jsonMsg}")
# Perform the request using HPI and get the response
hpi_response = self._HPI_actuator.perform_request(jsonMsg).strip()
self._log_debug(f"_process_msg, hpi_response: {hpi_response}")
# Check for success and power the disk back on
if "Success" in hpi_response:
# Append POWER_ON to notify HPI actuator of desired state
jsonMsg["actuator_request_type"]["node_controller"]["node_request"] = \
f"DISK: set {drive_request} POWER_ON"
self._log_debug(f"_process_msg, jsonMsg: {jsonMsg}")
# Perform the request using HPI and get the response
hpi_response = self._HPI_actuator.perform_request(jsonMsg).strip()
self._log_debug(f"_process_msg, hpi_response: {hpi_response}")
# Simplify success message as external apps don't care about details
if "Success" in hpi_response:
hpi_response = "Successful"
json_msg = AckResponseMsg(node_request, hpi_response, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
elif component == "HDPA":
# If the state is INITIALIZED, We can assume that actuator is
# ready to perform operation.
if actuator_state_manager.is_initialized("Hdparm"):
logger.info(f"_process_msg, Hdparm_actuator name: {self._hdparm_actuator.name()}")
# Perform the hdparm request on the node and get the response
hdparm_response = self._hdparm_actuator.perform_request(jsonMsg).strip()
self._log_debug(f"_process_msg, hdparm_response: {hdparm_response}")
json_msg = AckResponseMsg(node_request, hdparm_response, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
# If the state is INITIALIZING, need to send message
elif actuator_state_manager.is_initializing("Hdparm"):
# This state will not be reached. Kept here for consistency.
logger.info("Hdparm actuator is initializing")
busy_json_msg = AckResponseMsg(
node_request, "BUSY", uuid, error_no=errno.EBUSY).getJson()
self._write_internal_msgQ(
"RabbitMQegressProcessor", busy_json_msg)
elif actuator_state_manager.is_imported("Hdparm"):
# This case will be for first request only. Subsequent
# requests will go to INITIALIZED state case.
logger.info("Hdparm actuator is imported and initializing")
# Query the Zope GlobalSiteManager for an object
# implementing the hdparm actuator.
from actuators.Ihdparm import IHdparm
actuator_state_manager.set_state(
"Hdparm", actuator_state_manager.INITIALIZING)
hdparm_actuator_class = self._queryUtility(IHdparm)
if hdparm_actuator_class:
# NOTE: Instantiation part should not time consuming
# otherwise NodeControllerMsgHandler will get block and will
# not be able serve any subsequent requests. This applies
# to instantiation of evey actuator.
self._hdparm_actuator = hdparm_actuator_class()
self._log_debug(f"_process_msg, _hdparm_actuator name: {self._hdparm_actuator.name()}")
# Perform the hdparm request on the node and get the response
hdparm_response = self._hdparm_actuator.perform_request(jsonMsg).strip()
self._log_debug(f"_process_msg, hdparm_response: {hdparm_response}")
json_msg = AckResponseMsg(node_request, hdparm_response, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
actuator_state_manager.set_state(
"Hdparm", actuator_state_manager.INITIALIZED)
else:
logger.info("Hdparm actuator is not instantiated")
# If there is no entry for actuator in table, We can assume
# that it is not loaded for some reason.
else:
logger.info("Hdparm actuator is not loaded or not supported")
elif component == "SMAR":
# Parse out the drive request field in json msg
node_request = jsonMsg.get("actuator_request_type").get("node_controller").get("node_request")
drive_request = node_request[12:].strip()
self._log_debug(f"perform_request, drive: {drive_request}")
# If the drive field is an asterisk then send all the smart results for all drives available
if drive_request == "*":
# Send the event to SystemdWatchdog to schedule SMART test
internal_json_msg = json.dumps(
{"sensor_request_type" : "disk_smart_test",
"serial_number" : "*",
"node_request" : self.host_id,
"uuid" : uuid
})
self._write_internal_msgQ("SystemdWatchdog", internal_json_msg)
return
# Put together a message to get the serial number of the drive using hdparm tool
if drive_request.startswith("/"):
serial_number, error = self._retrieve_serial_number(drive_request)
# Send error response back on ack channel
if error != "":
json_msg = AckResponseMsg(node_request, error, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
return
else:
if self._smartctl_actuator is None:
from actuators.Ismartctl import ISmartctl
smartctl_actuator_class = self._queryUtility(ISmartctl)
if smartctl_actuator_class:
self._smartctl_actuator = self._queryUtility(ISmartctl)()
self._log_debug("_process_msg, _smart_actuator name: %s" % self._smartctl_actuator.name())
else:
logger.error(" No module Smartctl is present to load")
serial_compare = self._smartctl_actuator._check_serial_number(drive_request)
if not serial_compare:
json_msg = AckResponseMsg(node_request, "Drive Not Found", uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
return
else:
serial_number = drive_request
# Send the event to SystemdWatchdog to schedule SMART test
internal_json_msg = json.dumps(
{"sensor_request_type" : "disk_smart_test",
"serial_number" : serial_number,
"node_request" : node_request,
"uuid" : uuid
})
self._write_internal_msgQ("SystemdWatchdog", internal_json_msg)
elif component == "DRVM":
# Requesting the current status from drivemanager
# Parse out the drive request field in json msg
node_request = jsonMsg.get("actuator_request_type").get("node_controller").get("node_request")
drive_request = node_request[15:].strip()
self._log_debug(f"perform_request, drive: {drive_request}")
# If the drive field is an asterisk then send all the drivemanager results for all drives available
if drive_request == "*":
# Send a message to the disk message handler to lookup the drivemanager status and send it out
internal_json_msg = json.dumps(
{"sensor_request_type" : "drvmngr_status",
"serial_number" : "*",
"node_request" : self.host_id,
"uuid" : uuid
})
# Send the event to disk message handler to generate json message
self._write_internal_msgQ(DiskMsgHandler.name(), internal_json_msg)
return
# Put together a message to get the serial number of the drive using hdparm tool
if drive_request.startswith("/"):
serial_number, error = self._retrieve_serial_number(drive_request)
# Send error response back on ack channel
if error != "":
json_msg = AckResponseMsg(node_request, error, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
return
else:
serial_number = drive_request
# Send a message to the disk message handler to lookup the smart status and send it out
internal_json_msg = json.dumps(
{"sensor_request_type" : "drvmngr_status",
"serial_number" : serial_number,
"node_request" : node_request,
"uuid" : uuid
})
# Send the event to disk message handler to generate json message
self._write_internal_msgQ(DiskMsgHandler.name(), internal_json_msg)
elif component == "HPI_":
# Requesting the current status from HPI data
# Parse out the drive request field in json msg
if self._is_env_vm():
logger.warn("HPI operations are not supported in current environment")
return
if self.setup == 'cortx':
logger.warn("HPIMonitor not loaded")
json_msg = AckResponseMsg(node_request, NodeControllerMsgHandler.UNSUPPORTED_REQUEST, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
return
node_request = jsonMsg.get("actuator_request_type").get("node_controller").get("node_request")
drive_request = node_request[11:].strip()
self._log_debug(f"perform_request, drive: {drive_request}")
# If the drive field is an asterisk then send all the hpi results for all drives available
if drive_request == "*":
# Send a message to the disk message handler to lookup the hpi status and send it out
internal_json_msg = json.dumps(
{"sensor_request_type" : "hpi_status",
"serial_number" : "*",
"node_request" : self.host_id,
"uuid" : uuid
})
# Send the event to disk message handler to generate json message
self._write_internal_msgQ(DiskMsgHandler.name(), internal_json_msg)
return
# Put together a message to get the serial number of the drive using hdparm tool
if drive_request.startswith("/"):
serial_number, error = self._retrieve_serial_number(drive_request)
# Send error response back on ack channel
if error != "":
json_msg = AckResponseMsg(node_request, error, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
return
else:
serial_number = drive_request
# Send a message to the disk message handler to lookup the smart status and send it out
internal_json_msg = json.dumps(
{"sensor_request_type" : "hpi_status",
"serial_number" : serial_number,
"node_request" : node_request,
"uuid" : uuid
})
# Send the event to disk message handler to generate json message
self._write_internal_msgQ(DiskMsgHandler.name(), internal_json_msg)
elif component == "SIMU":
# Requesting to simulate an event
# Parse out the simulated request field
node_request = jsonMsg.get("actuator_request_type").get("node_controller").get("node_request")
sim_request = node_request[9:].strip().split(" ")
self._log_debug(f"perform_request, sim_request: {str(sim_request)}")
# Put together a message to get the serial number of the drive using hdparm tool
if sim_request[1].startswith("/"):
serial_number, error = self._retrieve_serial_number(sim_request[1])
# Send error response back on ack channel
if error != "":
json_msg = AckResponseMsg(node_request, error, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
return
else:
serial_number = sim_request[1]
# SMART simulation requests are sent to SystemdWatchdog
if sim_request[0] == "SMART_FAILURE":
logger.info(f"NodeControllerMsgHandler, simulating SMART_FAILURE on drive: {serial_number}")
internal_json_msg = json.dumps(
{"sensor_request_type" : "simulate_failure",
"serial_number" : serial_number,
"node_request" : sim_request[0],
"uuid" : uuid
})
# Send the event to SystemdWatchdog to handle it from here
self._write_internal_msgQ("SystemdWatchdog", internal_json_msg)
else:
# Send a message to the disk message handler to handle simulation request
internal_json_msg = json.dumps(
{"sensor_request_type" : "sim_event",
"serial_number" : serial_number,
"node_request" : sim_request[0],
"uuid" : uuid
})
# Send the event to disk message handler to generate json message
self._write_internal_msgQ(DiskMsgHandler.name(), internal_json_msg)
elif component == "NDHW":
# NDHW Stands for Node HW.
try:
# Load and Instantiate the Actuator for the first request
if self._NodeHW_actuator is None:
from actuators.impl.generic.node_hw import NodeHWactuator
from framework.utils.ipmi_client import IpmiFactory
self.ipmi_client_name = self._conf_reader._get_value_with_default(
self.NODE_HW_ACTUATOR, self.IPMI_IMPLEMENTOR,
"ipmitool")
ipmi_factory = IpmiFactory()
ipmi_client = \
ipmi_factory.get_implementor(self.ipmi_client_name)
# Instantiate NodeHWactuator only if class is loaded
if ipmi_client is not None:
self._NodeHW_actuator = NodeHWactuator(ipmi_client, self._conf_reader)
self._NodeHW_actuator.initialize()
else:
logger.error(f"IPMI client: '{self.ipmi_client_name}' doesn't exist")
return
node_request = jsonMsg.get("actuator_request_type")
# Perform the NodeHW request on the node and get the response
#TODO: Send message to Ack as well as Sensor in their respective channel.
node_hw_response = self._NodeHW_actuator.perform_request(node_request)
self._log_debug(f"_process_msg, node_hw_response: {node_hw_response}")
json_msg = NodeHwAckResponseMsg(node_request, node_hw_response, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
except ImportError as e:
logger.error(f"Modules could not be loaded: {e}")
return
except Exception as e:
logger.error(f"NodeControllerMsgHandler, _process_msg, Exception in request handling: {e}")
return
else:
response = f"NodeControllerMsgHandler, _process_msg, unknown node controller msg: {node_request}"
self._log_debug(response)
json_msg = AckResponseMsg(node_request, response, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
def _process_msg(self, ch, method, properties, body):
"""Parses the incoming message and hands off to the appropriate module"""
ingressMsg = {}
uuid = None
try:
if isinstance(body, dict) is False:
ingressMsg = json.loads(body)
else:
ingressMsg = body
# Authenticate message using username and signature fields
username = ingressMsg.get("username")
signature = ingressMsg.get("signature")
message = ingressMsg.get("message")
uuid = ingressMsg.get("uuid")
msg_len = len(message) + 1
if uuid is None:
uuid = "N/A"
if use_security_lib and \
SSPL_SEC.sspl_verify_message(msg_len, str(message), username, signature) != 0:
logger.warn(
"RabbitMQingressProcessor, Authentication failed on message: %s"
% ingressMsg)
return
# Get the incoming message type
if message.get("actuator_request_type") is not None:
msgType = message.get("actuator_request_type")
# Validate against the actuator schema
validate(ingressMsg, self._actuator_schema)
elif message.get("sensor_request_type") is not None:
msgType = message.get("sensor_request_type")
# Validate against the sensor schema
validate(ingressMsg, self._sensor_schema)
else:
# We only handle incoming actuator and sensor requests, ignore
# everything else.
return
# Check for debugging being activated in the message header
self._check_debug(message)
self._log_debug("_process_msg, ingressMsg: %s" % ingressMsg)
# Hand off to appropriate actuator message handler
if msgType.get("logging") is not None:
self._write_internal_msgQ("LoggingMsgHandler", message)
elif msgType.get("thread_controller") is not None:
self._write_internal_msgQ("ThreadController", message)
elif msgType.get("service_controller") is not None:
self._write_internal_msgQ("ServiceMsgHandler", message)
elif msgType.get("node_controller") is not None:
self._write_internal_msgQ("NodeControllerMsgHandler", message)
elif msgType.get("storage_enclosure") is not None:
self._write_internal_msgQ("RealStorActuatorMsgHandler",
message)
# Hand off to appropriate sensor message handler
elif msgType.get("node_data") is not None:
self._write_internal_msgQ("NodeDataMsgHandler", message)
elif msgType.get("enclosure_alert") is not None:
self._write_internal_msgQ("RealStorEnclMsgHandler", message)
elif msgType.get("storage_enclosure") is not None:
self._write_internal_msgQ("RealStorActuatorMsgHandler",
message)
# ... handle other incoming messages that have been validated
else:
# Send ack about not finding a msg handler
ack_msg = AckResponseMsg("Error Processing Message",
"Message Handler Not Found",
uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(),
ack_msg)
# Acknowledge message was received
self._connection.ack(ch, delivery_tag=method.delivery_tag)
except Exception as ex:
logger.error(
"RabbitMQingressProcessor, _process_msg unrecognized message: %r"
% ingressMsg)
ack_msg = AckResponseMsg("Error Processing Msg",
"Msg Handler Not Found", uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), ack_msg)
def _process_msg(self, ch, method, properties, body):
"""Parses the incoming message and hands off to the appropriate module"""
ingressMsg = {}
try:
if isinstance(body, dict) is False:
ingressMsg = json.loads(body)
else:
ingressMsg = body
# Authenticate message using username and signature fields
username = ingressMsg.get("username")
signature = ingressMsg.get("signature")
message = ingressMsg.get("message")
uuid = message.get("sspl_ll_msg_header").get("uuid")
msg_len = len(message) + 1
if uuid is None:
uuid = "N/A"
if use_security_lib and \
SSPL_SEC.sspl_verify_message(msg_len, str(message), username, signature) != 0:
logger.warn(
"PlaneCntrlRMQingressProcessor, Authentication failed on message: %s"
% ingressMsg)
return
# Get the incoming message type
if message.get("actuator_request_type") is not None:
msgType = message.get("actuator_request_type")
# Validate against the actuator schema
validate(ingressMsg, self._actuator_schema)
elif message.get("sensor_request_type") is not None:
msgType = message.get("sensor_request_type")
# Validate against the sensor schema
validate(ingressMsg, self._sensor_schema)
else:
# We only handle incoming actuator and sensor requests, ignore everything else
return
# Check for debugging being activated in the message header
self._check_debug(message)
self._log_debug("_process_msg, ingressMsg: %s" % ingressMsg)
# Hand off to appropriate actuator message handler
if msgType.get("plane_controller") is not None:
command = message.get("actuator_request_type").get(
"plane_controller").get("command")
# For a job status request forward over to PlaneCntrlRMQegressProcessor
if command is not None and \
command == "job_status":
job_uuid = message.get("actuator_request_type").get(
"plane_controller").get("arguments").get("uuid", None)
node_id = None
if message.get("actuator_request_type").get("plane_controller").get("parameters") is not None and \
message.get("actuator_request_type").get("plane_controller").get("parameters").get("node_id") is not None:
node_id = message.get("actuator_request_type").get(
"plane_controller").get("parameters").get(
"node_id")
# node_id set to None is a broadcast otherwise see if it applies to this node
if node_id is None or \
self._hostname in str(node_id):
self._process_job_status(uuid, job_uuid)
else:
self._write_internal_msgQ("PlaneCntrlMsgHandler", message)
# Handle restarting of internal threads
elif msgType.get("thread_controller") is not None:
node_id = None
if message.get("actuator_request_type").get("thread_controller").get("parameters") is not None and \
message.get("actuator_request_type").get("thread_controller").get("parameters").get("node_id") is not None:
node_id = message.get("actuator_request_type").get(
"thread_controller").get("parameters").get(
"node_id", None)
# node_id set to None is a broadcast otherwise see if it applies to this node
if node_id is None or \
self._hostname in str(node_id):
self._write_internal_msgQ("ThreadController", message)
# ... handle other incoming messages that have been validated
else:
# Log a msg about not being able to process the message
logger.info(
"PlaneCntrlRMQingressProcessor, _process_msg, unrecognized message: %s"
% str(ingressMsg))
except Exception as ex:
logger.exception(
"PlaneCntrlRMQingressProcessor, _process_msg unrecognized message: %r"
% ingressMsg)
ack_msg = AckResponseMsg("Error Processing Msg",
"Msg Handler Not Found", uuid).getJson()
self._write_internal_msgQ(PlaneCntrlRMQegressProcessor.name(),
ack_msg)
# Acknowledge message was received
self._connection.ack(ch, delivery_tag=method.delivery_tag)
def _process_msg(self, body):
"""Parses the incoming message and hands off to the appropriate module"""
ingressMsg = {}
uuid = None
try:
if isinstance(body, dict) is False:
ingressMsg = json.loads(body)
else:
ingressMsg = body
# Authenticate message using username and signature fields
username = ingressMsg.get("username")
signature = ingressMsg.get("signature")
message = ingressMsg.get("message")
uuid = ingressMsg.get("uuid")
msg_len = len(message) + 1
if uuid is None:
uuid = "N/A"
if use_security_lib and \
SSPL_SEC.sspl_verify_message(msg_len, str(message),
username, signature) != 0:
logger.warn(
"IngressProcessor, Authentication failed on message: %s" % ingressMsg)
return
logger.debug("_process_msg, ingressMsg: %s" % ingressMsg)
# Get the incoming message type
if message.get("actuator_request_type") is not None:
msgType = message.get("actuator_request_type")
# Validate against the actuator schema
validate(ingressMsg, self._actuator_schema)
# Compare target_node_id from the request to determine
# if request is meant for the current node
target_node_id = message.get("target_node_id")
if target_node_id is None:
logger.warn(
"Required attribute target_node_id is missing "
"from actuator request %s, IGNORING!!" % (msgType))
return
elif target_node_id == self._node_id:
self._send_to_msg_handler(msgType, message, uuid)
else:
logger.debug(
"Node identifier mismatch, actuator request ignored.")
return
elif message.get("sensor_request_type") is not None:
msgType = message.get("sensor_request_type")
# Validate against the sensor schema
validate(ingressMsg, self._sensor_schema)
self._send_to_msg_handler(msgType, message, uuid)
else:
# We only handle incoming actuator and sensor requests, ignore
# everything else.
return
except Exception as ex:
logger.error(
"IngressProcessor, _process_msg failed to recognize "
"message: %r with error %r" % (ingressMsg, ex))
ack_msg = AckResponseMsg("Error Processing Msg",
"Msg Handler Not Found", uuid).getJson()
self._write_internal_msgQ(EgressProcessor.name(), ack_msg)
def _process_msg(self, jsonMsg):
"""Parses the incoming message and hands off to the appropriate logger
"""
self._log_debug(f"_process_msg, jsonMsg: {jsonMsg}")
if isinstance(jsonMsg, dict) is False:
jsonMsg = json.loads(jsonMsg)
# Parse out the uuid so that it can be sent back in Ack message
uuid = None
if jsonMsg.get("sspl_ll_msg_header") is not None and \
jsonMsg.get("sspl_ll_msg_header").get("uuid") is not None:
uuid = jsonMsg.get("sspl_ll_msg_header").get("uuid")
self._log_debug(f"_processMsg, uuid: {uuid}")
# Handle service start, stop, restart, status requests
if jsonMsg.get("actuator_request_type").get(
"service_controller") is not None:
self._log_debug("_processMsg, msg_type: service_controller")
service_name = jsonMsg.get("actuator_request_type") \
.get("service_controller").get("service_name")
service_request = jsonMsg.get("actuator_request_type") \
.get("service_controller").get("service_request")
request = f"{service_request}:{service_name}"
# If the state is INITIALIZED, We can assume that actuator is
# ready to perform operation.
if actuator_state_manager.is_initialized("Service"):
self._log_debug(
f"_process_msg, service_actuator name: {self._service_actuator.name()}"
)
self._execute_request(self._service_actuator, jsonMsg, uuid)
# If the state is INITIALIZING, need to send message
elif actuator_state_manager.is_initializing("Service"):
# This state will not be reached. Kept here for consistency.
logger.info("Service actuator is initializing")
busy_json_msg = AckResponseMsg(request,
"BUSY",
uuid,
error_no=errno.EBUSY).getJson()
self._write_internal_msgQ("RabbitMQegressProcessor",
busy_json_msg)
elif actuator_state_manager.is_imported("Service"):
# This case will be for first request only. Subsequent
# requests will go to INITIALIZED state case.
logger.info("Service actuator is imported and initializing")
from actuators.IService import IService
actuator_state_manager.set_state(
"Service", actuator_state_manager.INITIALIZING)
service_actuator_class = self._query_utility(IService)
if service_actuator_class:
# NOTE: Instantiation part should not time consuming
# otherwise ServiceMsgHandler will get block and will
# not be able serve any subsequent requests. This applies
# to instantiation of evey actuator.
self._service_actuator = service_actuator_class()
logger.info(
f"_process_msg, service_actuator name: {self._service_actuator.name()}"
)
self._execute_request(self._service_actuator, jsonMsg,
uuid)
actuator_state_manager.set_state(
"Service", actuator_state_manager.INITIALIZED)
else:
logger.info("Service actuator is not instantiated")
# If there is no entry for actuator in table, We can assume
# that it is not loaded for some reason.
else:
logger.warn("Service actuator is not loaded or not supported")
# Handle events generated by the service watchdogs
elif jsonMsg.get("actuator_request_type").get(
"service_watchdog_controller") is not None:
self._log_debug(
"_processMsg, msg_type: service_watchdog_controller")
# Parse out values to be sent
service_name = jsonMsg.get("actuator_request_type").get(
"service_watchdog_controller").get("service_name")
state = jsonMsg.get("actuator_request_type").get(
"service_watchdog_controller").get("state")
prev_state = jsonMsg.get("actuator_request_type").get(
"service_watchdog_controller").get("previous_state")
substate = jsonMsg.get("actuator_request_type").get(
"service_watchdog_controller").get("substate")
prev_substate = jsonMsg.get("actuator_request_type").get(
"service_watchdog_controller").get("previous_substate")
pid = jsonMsg.get("actuator_request_type").get(
"service_watchdog_controller").get("pid")
prev_pid = jsonMsg.get("actuator_request_type").get(
"service_watchdog_controller").get("previous_pid")
# Pull out the service_request and if it's equal to "status" then get current status (state, substate)
service_request = jsonMsg.get("actuator_request_type").get(
"service_watchdog_controller").get("service_request")
if service_request != "None":
# Query the Zope GlobalSiteManager for an object implementing IService
if self._service_actuator is None:
from actuators.IService import IService
self._service_actuator = self._query_utility(IService)()
self._log_debug(
f"_process_msg, service_actuator name: {self._service_actuator.name()}"
)
service_name, state, substate = self._service_actuator.perform_request(
jsonMsg)
self._log_debug(
f"_processMsg, service_name: {service_name}, state: {state}, substate: {substate}"
)
self._log_debug(
f"_processMsg, prev state: {prev_state}, prev substate: {prev_substate}"
)
# Create a service watchdog message and send it out
jsonMsg = ServiceWatchdogMsg(service_name, state, prev_state,
substate, prev_substate, pid,
prev_pid).getJson()
self._write_internal_msgQ("RabbitMQegressProcessor", jsonMsg)
# Create an IEM if the resulting service state is failed
if "fail" in state.lower() or \
"fail" in substate.lower():
json_data = {
"service_name": service_name,
"state": state,
"previous_state": prev_state,
"substate": substate,
"previous_substate": prev_substate,
"pid": pid,
"previous_pid": prev_pid
}
internal_json_msg = json.dumps({
"actuator_request_type": {
"logging": {
"log_level":
"LOG_WARNING",
"log_type":
"IEM",
"log_msg":
f"IEC: 020003001: Service entered a Failed state : {json.dumps(json_data, sort_keys=True)}"
}
}
})
# Send the event to logging msg handler to send IEM message to journald
self._write_internal_msgQ(LoggingMsgHandler.name(),
internal_json_msg)