def read_data(self):
"""Send a dict of drive status to the DiskMsgHandler"""
for drive_data in self._drive_data:
logger.info(f"HPIMonitor, read_data: {str(drive_data)}")
# Send it to the disk message handler to be processed and transmitted
self._write_internal_msgQ(DiskMsgHandler.name(), drive_data)
def _notify_DiskMsgHandler(self, status_file: str, serial_num_file):
"""Send the event to the disk message handler for generating JSON message"""
if not os.path.isfile(status_file):
logger.warn(
f"status_file: {status_file} does not exist, ignoring.")
return
if not os.path.isfile(serial_num_file):
logger.warn(
f"serial_num_file: {serial_num_file} does not exist, ignoring."
)
return
# Read in status and see if it has changed
with open(status_file, "r") as datafile:
status = datafile.read().replace('\n', '')
# See if there's a reason file
reason_file = os.path.join(os.path.dirname(status_file), "reason")
if os.path.isfile(reason_file):
with open(reason_file, "r") as datafile:
reason = datafile.read().replace('\n', '')
status = f"{status}_{reason}"
# Do nothing if the drive status has not changed
if self._drive_status[os.path.dirname(status_file)] == status:
return
# Update the status for this drive
self._log_debug(
f"Status change, status_file: {status_file}, status: {status}")
self._drive_status[os.path.dirname(status_file)] = status
# Read in the serial number
with open(serial_num_file, "r") as datafile:
serial_number = datafile.read().replace('\n', '')
# Remove base dcs dir since it contains no relevant data
data_str = status_file[len(self._drive_mngr_base_dir) + 1:]
# Send a message to the disk manager handler to create and transmit json msg
internal_json_msg = json.dumps({
"sensor_response_type": "disk_status_drivemanager",
"event_path": data_str,
"status": status,
"serial_number": serial_number
})
# Send the event to disk message handler to generate json message
self._write_internal_msgQ(DiskMsgHandler.name(), internal_json_msg)
# Reset debug mode if persistence is not enabled
self._disable_debug_if_persist_false()
def _process_msg(self, jsonMsg):
"""Parses the incoming message and handles appropriately"""
self._log_debug(f"_process_msg, jsonMsg: {jsonMsg}")
if isinstance(jsonMsg, dict) is False:
jsonMsg = json.loads(jsonMsg)
# Parse out the uuid so that it can be sent back in Ack message
uuid = None
if jsonMsg.get("sspl_ll_msg_header").get("uuid") is not None:
uuid = jsonMsg.get("sspl_ll_msg_header").get("uuid")
self._log_debug(f"_processMsg, uuid: {uuid}")
if jsonMsg.get("actuator_request_type").get("node_controller").get("node_request") is not None:
node_request = jsonMsg.get("actuator_request_type").get("node_controller").get("node_request")
self._log_debug(f"_processMsg, node_request: {node_request}")
# Parse out the component field in the node_request
component = node_request[0:4]
# Handle generic command line requests
if component == 'SSPL':
# Query the Zope GlobalSiteManager for an object implementing the MOTR actuator
if self._command_line_actuator is None:
from actuators.Icommand_line import ICommandLine
command_line_actuator_class = self._queryUtility(ICommandLine)
# Instantiate CommandLine Actuator only if class is loaded
if command_line_actuator_class:
self._command_line_actuator = command_line_actuator_class(self._conf_reader)
else:
logger.warn("CommandLine Actuator not loaded")
json_msg = AckResponseMsg(node_request, NodeControllerMsgHandler.UNSUPPORTED_REQUEST, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
return
# Perform the request and get the response
command_line_response = self._command_line_actuator.perform_request(jsonMsg).strip()
self._log_debug(f"_process_msg, command line response: {command_line_response}")
json_msg = AckResponseMsg(node_request, command_line_response, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
# Handle LED effects using the HPI actuator
elif component == "LED:":
# HPI related operations are not supported in VM environment.
if self._is_env_vm():
logger.warn("HPI operations are not supported in current environment")
return
# Query the Zope GlobalSiteManager for an object implementing the IHPI actuator
if self._HPI_actuator is None:
from actuators.Ihpi import IHPI
# Load HPIActuator class
HPI_actuator_class = self._queryUtility(IHPI)
# Instantiate HPIActuator only if class is loaded
if HPI_actuator_class:
self._HPI_actuator = HPI_actuator_class(self._conf_reader)
else:
logger.warn("HPIActuator not loaded")
if self._product.lower() in [x.lower() for x in enabled_products]:
json_msg = AckResponseMsg(node_request, NodeControllerMsgHandler.UNSUPPORTED_REQUEST, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
return
self._log_debug(f"_process_msg, _HPI_actuator name: {self._HPI_actuator.name()}")
# Perform the request using HPI and get the response
hpi_response = self._HPI_actuator.perform_request(jsonMsg).strip()
self._log_debug(f"_process_msg, hpi_response: {hpi_response}")
json_msg = AckResponseMsg(node_request, hpi_response, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
# Set the Bezel LED color using the GEM interface
elif component == "BEZE":
# Query the Zope GlobalSiteManager for an object implementing the IGEM actuator
if self._GEM_actuator is None:
self._GEM_actuator = self._queryUtility(IGEM)(self._conf_reader)
self._log_debug(f"_process_msg, _GEM_actuator name: {self._GEM_actuator.name()}")
# Perform the request using GEM and get the response
gem_response = self._GEM_actuator.perform_request(jsonMsg).strip()
self._log_debug(f"_process_msg, gem_response: {gem_response}")
json_msg = AckResponseMsg(node_request, gem_response, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
elif component == "PDU:":
# Query the Zope GlobalSiteManager for an object implementing the IPDU actuator
if self._PDU_actuator is None:
from actuators.Ipdu import IPDU
PDU_actuator_class = self._queryUtility(IPDU)
# Instantiate RaritanPDU Actuator only if class is loaded
if PDU_actuator_class:
self._PDU_actuator = PDU_actuator_class(self._conf_reader)
else:
logger.warn("RaritanPDU Actuator not loaded")
json_msg = AckResponseMsg(node_request, NodeControllerMsgHandler.UNSUPPORTED_REQUEST, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
return
# Perform the request on the PDU and get the response
pdu_response = self._PDU_actuator.perform_request(jsonMsg).strip()
self._log_debug(f"_process_msg, pdu_response: {pdu_response}")
json_msg = AckResponseMsg(node_request, pdu_response, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
elif component == "RAID":
# If the state is INITIALIZED, We can assume that actuator is
# ready to perform operation.
if actuator_state_manager.is_initialized("RAIDactuator"):
self._log_debug(f"_process_msg, _RAID_actuator name: {self._RAID_actuator.name()}")
self._execute_raid_request(
node_request, self._RAID_actuator, jsonMsg, uuid)
# If the state is INITIALIZING, need to send message
elif actuator_state_manager.is_initializing("RAIDactuator"):
# This state will not be reached. Kept here for consistency.
logger.info("RAID actuator is initializing")
busy_json_msg = AckResponseMsg(
node_request, "BUSY", uuid, error_no=errno.EBUSY).getJson()
self._write_internal_msgQ(
"RabbitMQegressProcessor", busy_json_msg)
elif actuator_state_manager.is_imported("RAIDactuator"):
# This case will be for first request only. Subsequent
# requests will go to INITIALIZED state case.
logger.info("RAID actuator is imported and initializing")
from actuators.Iraid import IRAIDactuator
actuator_state_manager.set_state(
"RAIDactuator", actuator_state_manager.INITIALIZING)
# Query the Zope GlobalSiteManager for an object implementing the IRAIDactuator
raid_actuator_class = self._queryUtility(IRAIDactuator)
if raid_actuator_class:
# NOTE: Instantiation part should not time consuming
# otherwise NodeControllerMsgHandler will get block
# and will not be able serve any subsequent requests.
# This applies to instantiation of evey actuator.
self._RAID_actuator = raid_actuator_class()
logger.info(f"_process_msg, _RAID_actuator name: {self._RAID_actuator.name()}")
self._execute_raid_request(
node_request, self._RAID_actuator, jsonMsg, uuid)
actuator_state_manager.set_state(
"RAIDactuator", actuator_state_manager.INITIALIZED)
else:
logger.warn("RAID actuator is not instantiated")
# If there is no entry for actuator in table, We can assume
# that it is not loaded for some reason.
else:
logger.warn("RAID actuator is not loaded or not supported")
elif component == "IPMI":
# Query the Zope GlobalSiteManager for an object implementing the IPMI actuator
if self._IPMI_actuator is None:
from actuators.Iipmi import Iipmi
IPMI_actuator_class = self._queryUtility(Iipmi)
# Instantiate IPMI Actuator only if class is loaded
if IPMI_actuator_class:
self._IPMI_actuator = IPMI_actuator_class(self._conf_reader)
else:
logger.warn("IPMI Actuator not loaded")
json_msg = AckResponseMsg(node_request, NodeControllerMsgHandler.UNSUPPORTED_REQUEST, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
return
# Perform the IPMI request on the node and get the response
ipmi_response = self._IPMI_actuator.perform_request(jsonMsg).strip()
self._log_debug(f"_process_msg, ipmi_response: {ipmi_response}")
json_msg = AckResponseMsg(node_request, ipmi_response, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
elif component == "STOP":
# HPI related operations are not supported in VM environment.
if self._is_env_vm():
logger.warn("HPI operations are not supported in current environment")
return
# Query the Zope GlobalSiteManager for an object implementing the IHPI actuator
if self._HPI_actuator is None:
from actuators.Ihpi import IHPI
# Load HPIActuator class
HPI_actuator_class = self._queryUtility(IHPI)
# Instantiate HPIActuator only if class is loaded
if HPI_actuator_class:
self._HPI_actuator = HPI_actuator_class(self._conf_reader)
else:
logger.warn("HPIActuator not loaded")
if self._product.lower() in [x.lower() for x in enabled_products]:
json_msg = AckResponseMsg(node_request, NodeControllerMsgHandler.UNSUPPORTED_REQUEST, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
return
self._log_debug(f"_process_msg, _HPI_actuator name: {self._HPI_actuator.name()}")
# Parse out the drive to stop
drive_request = node_request[12:].strip()
self._log_debug(f"perform_request, drive to stop: {drive_request}")
# Append POWER_OFF to notify HPI actuator of desired state
jsonMsg["actuator_request_type"]["node_controller"]["node_request"] = \
f"DISK: set {drive_request} POWER_OFF"
self._log_debug(f"_process_msg, jsonMsg: {jsonMsg}")
# Perform the request using HPI and get the response
hpi_response = self._HPI_actuator.perform_request(jsonMsg).strip()
self._log_debug(f"_process_msg, hpi_response: {hpi_response}")
# Simplify success message as external apps don't care about details
if "Success" in hpi_response:
hpi_response = "Successful"
json_msg = AckResponseMsg(node_request, hpi_response, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
elif component == "STAR":
# HPI related operations are not supported in VM environment.
if self._is_env_vm():
logger.warn("HPI operations are not supported in current environment")
return
# Query the Zope GlobalSiteManager for an object implementing the IHPI actuator
if self._HPI_actuator is None:
from actuators.Ihpi import IHPI
# Load HPIActuator class
HPI_actuator_class = self._queryUtility(IHPI)
# Instantiate HPIActuator only if class is loaded
if HPI_actuator_class:
self._HPI_actuator = HPI_actuator_class(self._conf_reader)
else:
logger.warn("HPIActuator not loaded")
if self._product.lower() in [x.lower() for x in enabled_products]:
json_msg = AckResponseMsg(node_request, NodeControllerMsgHandler.UNSUPPORTED_REQUEST, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
return
self._log_debug(f"_process_msg, _HPI_actuator name: {self._HPI_actuator.name()}")
# Parse out the drive to start
drive_request = node_request[13:].strip()
self._log_debug(f"perform_request, drive to start: {drive_request}")
# Append POWER_ON to notify HPI actuator of desired state
jsonMsg["actuator_request_type"]["node_controller"]["node_request"] = \
f"DISK: set {drive_request} POWER_ON"
self._log_debug(f"_process_msg, jsonMsg: {jsonMsg}")
# Perform the request using HPI and get the response
hpi_response = self._HPI_actuator.perform_request(jsonMsg).strip()
self._log_debug(f"_process_msg, hpi_response: {hpi_response}")
# Simplify success message as external apps don't care about details
if "Success" in hpi_response:
hpi_response = "Successful"
json_msg = AckResponseMsg(node_request, hpi_response, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
elif component == "RESE":
# HPI related operations are not supported in VM environment.
if self._is_env_vm():
logger.warn("HPI operations are not supported in current environment")
return
# Query the Zope GlobalSiteManager for an object implementing the IHPI actuator
if self._HPI_actuator is None:
from actuators.Ihpi import IHPI
# Load HPIActuator class
HPI_actuator_class = self._queryUtility(IHPI)
# Instantiate HPIActuator only if class is loaded
if HPI_actuator_class:
self._HPI_actuator = HPI_actuator_class(self._conf_reader)
else:
logger.warn("HPIActuator not loaded")
if self._product.lower() in [x.lower() for x in enabled_products]:
json_msg = AckResponseMsg(node_request, NodeControllerMsgHandler.UNSUPPORTED_REQUEST, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
return
self._log_debug(f"_process_msg, _HPI_actuator name: {self._HPI_actuator.name()}")
# Parse out the drive to power cycle
drive_request = node_request[13:].strip()
self._log_debug(f"perform_request, drive to power cycle: {drive_request}")
# Append POWER_OFF and then POWER_ON to notify HPI actuator of desired state
jsonMsg["actuator_request_type"]["node_controller"]["node_request"] = \
f"DISK: set {drive_request} POWER_OFF"
self._log_debug(f"_process_msg, jsonMsg: {jsonMsg}")
# Perform the request using HPI and get the response
hpi_response = self._HPI_actuator.perform_request(jsonMsg).strip()
self._log_debug(f"_process_msg, hpi_response: {hpi_response}")
# Check for success and power the disk back on
if "Success" in hpi_response:
# Append POWER_ON to notify HPI actuator of desired state
jsonMsg["actuator_request_type"]["node_controller"]["node_request"] = \
f"DISK: set {drive_request} POWER_ON"
self._log_debug(f"_process_msg, jsonMsg: {jsonMsg}")
# Perform the request using HPI and get the response
hpi_response = self._HPI_actuator.perform_request(jsonMsg).strip()
self._log_debug(f"_process_msg, hpi_response: {hpi_response}")
# Simplify success message as external apps don't care about details
if "Success" in hpi_response:
hpi_response = "Successful"
json_msg = AckResponseMsg(node_request, hpi_response, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
elif component == "HDPA":
# If the state is INITIALIZED, We can assume that actuator is
# ready to perform operation.
if actuator_state_manager.is_initialized("Hdparm"):
logger.info(f"_process_msg, Hdparm_actuator name: {self._hdparm_actuator.name()}")
# Perform the hdparm request on the node and get the response
hdparm_response = self._hdparm_actuator.perform_request(jsonMsg).strip()
self._log_debug(f"_process_msg, hdparm_response: {hdparm_response}")
json_msg = AckResponseMsg(node_request, hdparm_response, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
# If the state is INITIALIZING, need to send message
elif actuator_state_manager.is_initializing("Hdparm"):
# This state will not be reached. Kept here for consistency.
logger.info("Hdparm actuator is initializing")
busy_json_msg = AckResponseMsg(
node_request, "BUSY", uuid, error_no=errno.EBUSY).getJson()
self._write_internal_msgQ(
"RabbitMQegressProcessor", busy_json_msg)
elif actuator_state_manager.is_imported("Hdparm"):
# This case will be for first request only. Subsequent
# requests will go to INITIALIZED state case.
logger.info("Hdparm actuator is imported and initializing")
# Query the Zope GlobalSiteManager for an object
# implementing the hdparm actuator.
from actuators.Ihdparm import IHdparm
actuator_state_manager.set_state(
"Hdparm", actuator_state_manager.INITIALIZING)
hdparm_actuator_class = self._queryUtility(IHdparm)
if hdparm_actuator_class:
# NOTE: Instantiation part should not time consuming
# otherwise NodeControllerMsgHandler will get block and will
# not be able serve any subsequent requests. This applies
# to instantiation of evey actuator.
self._hdparm_actuator = hdparm_actuator_class()
self._log_debug(f"_process_msg, _hdparm_actuator name: {self._hdparm_actuator.name()}")
# Perform the hdparm request on the node and get the response
hdparm_response = self._hdparm_actuator.perform_request(jsonMsg).strip()
self._log_debug(f"_process_msg, hdparm_response: {hdparm_response}")
json_msg = AckResponseMsg(node_request, hdparm_response, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
actuator_state_manager.set_state(
"Hdparm", actuator_state_manager.INITIALIZED)
else:
logger.info("Hdparm actuator is not instantiated")
# If there is no entry for actuator in table, We can assume
# that it is not loaded for some reason.
else:
logger.info("Hdparm actuator is not loaded or not supported")
elif component == "SMAR":
# Parse out the drive request field in json msg
node_request = jsonMsg.get("actuator_request_type").get("node_controller").get("node_request")
drive_request = node_request[12:].strip()
self._log_debug(f"perform_request, drive: {drive_request}")
# If the drive field is an asterisk then send all the smart results for all drives available
if drive_request == "*":
# Send the event to SystemdWatchdog to schedule SMART test
internal_json_msg = json.dumps(
{"sensor_request_type" : "disk_smart_test",
"serial_number" : "*",
"node_request" : self.host_id,
"uuid" : uuid
})
self._write_internal_msgQ("SystemdWatchdog", internal_json_msg)
return
# Put together a message to get the serial number of the drive using hdparm tool
if drive_request.startswith("/"):
serial_number, error = self._retrieve_serial_number(drive_request)
# Send error response back on ack channel
if error != "":
json_msg = AckResponseMsg(node_request, error, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
return
else:
if self._smartctl_actuator is None:
from actuators.Ismartctl import ISmartctl
smartctl_actuator_class = self._queryUtility(ISmartctl)
if smartctl_actuator_class:
self._smartctl_actuator = self._queryUtility(ISmartctl)()
self._log_debug("_process_msg, _smart_actuator name: %s" % self._smartctl_actuator.name())
else:
logger.error(" No module Smartctl is present to load")
serial_compare = self._smartctl_actuator._check_serial_number(drive_request)
if not serial_compare:
json_msg = AckResponseMsg(node_request, "Drive Not Found", uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
return
else:
serial_number = drive_request
# Send the event to SystemdWatchdog to schedule SMART test
internal_json_msg = json.dumps(
{"sensor_request_type" : "disk_smart_test",
"serial_number" : serial_number,
"node_request" : node_request,
"uuid" : uuid
})
self._write_internal_msgQ("SystemdWatchdog", internal_json_msg)
elif component == "DRVM":
# Requesting the current status from drivemanager
# Parse out the drive request field in json msg
node_request = jsonMsg.get("actuator_request_type").get("node_controller").get("node_request")
drive_request = node_request[15:].strip()
self._log_debug(f"perform_request, drive: {drive_request}")
# If the drive field is an asterisk then send all the drivemanager results for all drives available
if drive_request == "*":
# Send a message to the disk message handler to lookup the drivemanager status and send it out
internal_json_msg = json.dumps(
{"sensor_request_type" : "drvmngr_status",
"serial_number" : "*",
"node_request" : self.host_id,
"uuid" : uuid
})
# Send the event to disk message handler to generate json message
self._write_internal_msgQ(DiskMsgHandler.name(), internal_json_msg)
return
# Put together a message to get the serial number of the drive using hdparm tool
if drive_request.startswith("/"):
serial_number, error = self._retrieve_serial_number(drive_request)
# Send error response back on ack channel
if error != "":
json_msg = AckResponseMsg(node_request, error, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
return
else:
serial_number = drive_request
# Send a message to the disk message handler to lookup the smart status and send it out
internal_json_msg = json.dumps(
{"sensor_request_type" : "drvmngr_status",
"serial_number" : serial_number,
"node_request" : node_request,
"uuid" : uuid
})
# Send the event to disk message handler to generate json message
self._write_internal_msgQ(DiskMsgHandler.name(), internal_json_msg)
elif component == "HPI_":
# Requesting the current status from HPI data
# Parse out the drive request field in json msg
if self._is_env_vm():
logger.warn("HPI operations are not supported in current environment")
return
if self.setup == 'cortx':
logger.warn("HPIMonitor not loaded")
json_msg = AckResponseMsg(node_request, NodeControllerMsgHandler.UNSUPPORTED_REQUEST, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
return
node_request = jsonMsg.get("actuator_request_type").get("node_controller").get("node_request")
drive_request = node_request[11:].strip()
self._log_debug(f"perform_request, drive: {drive_request}")
# If the drive field is an asterisk then send all the hpi results for all drives available
if drive_request == "*":
# Send a message to the disk message handler to lookup the hpi status and send it out
internal_json_msg = json.dumps(
{"sensor_request_type" : "hpi_status",
"serial_number" : "*",
"node_request" : self.host_id,
"uuid" : uuid
})
# Send the event to disk message handler to generate json message
self._write_internal_msgQ(DiskMsgHandler.name(), internal_json_msg)
return
# Put together a message to get the serial number of the drive using hdparm tool
if drive_request.startswith("/"):
serial_number, error = self._retrieve_serial_number(drive_request)
# Send error response back on ack channel
if error != "":
json_msg = AckResponseMsg(node_request, error, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
return
else:
serial_number = drive_request
# Send a message to the disk message handler to lookup the smart status and send it out
internal_json_msg = json.dumps(
{"sensor_request_type" : "hpi_status",
"serial_number" : serial_number,
"node_request" : node_request,
"uuid" : uuid
})
# Send the event to disk message handler to generate json message
self._write_internal_msgQ(DiskMsgHandler.name(), internal_json_msg)
elif component == "SIMU":
# Requesting to simulate an event
# Parse out the simulated request field
node_request = jsonMsg.get("actuator_request_type").get("node_controller").get("node_request")
sim_request = node_request[9:].strip().split(" ")
self._log_debug(f"perform_request, sim_request: {str(sim_request)}")
# Put together a message to get the serial number of the drive using hdparm tool
if sim_request[1].startswith("/"):
serial_number, error = self._retrieve_serial_number(sim_request[1])
# Send error response back on ack channel
if error != "":
json_msg = AckResponseMsg(node_request, error, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
return
else:
serial_number = sim_request[1]
# SMART simulation requests are sent to SystemdWatchdog
if sim_request[0] == "SMART_FAILURE":
logger.info(f"NodeControllerMsgHandler, simulating SMART_FAILURE on drive: {serial_number}")
internal_json_msg = json.dumps(
{"sensor_request_type" : "simulate_failure",
"serial_number" : serial_number,
"node_request" : sim_request[0],
"uuid" : uuid
})
# Send the event to SystemdWatchdog to handle it from here
self._write_internal_msgQ("SystemdWatchdog", internal_json_msg)
else:
# Send a message to the disk message handler to handle simulation request
internal_json_msg = json.dumps(
{"sensor_request_type" : "sim_event",
"serial_number" : serial_number,
"node_request" : sim_request[0],
"uuid" : uuid
})
# Send the event to disk message handler to generate json message
self._write_internal_msgQ(DiskMsgHandler.name(), internal_json_msg)
elif component == "NDHW":
# NDHW Stands for Node HW.
try:
# Load and Instantiate the Actuator for the first request
if self._NodeHW_actuator is None:
from actuators.impl.generic.node_hw import NodeHWactuator
from framework.utils.ipmi_client import IpmiFactory
self.ipmi_client_name = self._conf_reader._get_value_with_default(
self.NODE_HW_ACTUATOR, self.IPMI_IMPLEMENTOR,
"ipmitool")
ipmi_factory = IpmiFactory()
ipmi_client = \
ipmi_factory.get_implementor(self.ipmi_client_name)
# Instantiate NodeHWactuator only if class is loaded
if ipmi_client is not None:
self._NodeHW_actuator = NodeHWactuator(ipmi_client, self._conf_reader)
self._NodeHW_actuator.initialize()
else:
logger.error(f"IPMI client: '{self.ipmi_client_name}' doesn't exist")
return
node_request = jsonMsg.get("actuator_request_type")
# Perform the NodeHW request on the node and get the response
#TODO: Send message to Ack as well as Sensor in their respective channel.
node_hw_response = self._NodeHW_actuator.perform_request(node_request)
self._log_debug(f"_process_msg, node_hw_response: {node_hw_response}")
json_msg = NodeHwAckResponseMsg(node_request, node_hw_response, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
except ImportError as e:
logger.error(f"Modules could not be loaded: {e}")
return
except Exception as e:
logger.error(f"NodeControllerMsgHandler, _process_msg, Exception in request handling: {e}")
return
else:
response = f"NodeControllerMsgHandler, _process_msg, unknown node controller msg: {node_request}"
self._log_debug(response)
json_msg = AckResponseMsg(node_request, response, uuid).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
def _init_drive_status(self):
# Allow time for the drivemanager to come up and populate the directory
time.sleep(5)
# Ensure there are enclosures present
self._validate_drive_manager_dir()
enclosures = os.listdir(self._drive_mngr_base_dir)
# Remove the 'discovery' file
enclosures = [enclosure for enclosure in enclosures \
if not os.path.isdir(os.path.join(self._drive_mngr_base_dir, enclosure))]
for enclosure in enclosures:
disk_dir = os.path.join(self._drive_mngr_base_dir, enclosure,
"disk")
logger.info(f"DriveManager initializing: {disk_dir}")
disks = os.listdir(disk_dir)
for disk in disks:
# Read in the status file for each disk and fill into dict
pathname = os.path.join(disk_dir, disk)
# Ignore the discovery file
if not os.path.isdir(pathname):
continue
# Read in the serial number for the disk
serial_num_file = os.path.join(pathname, "serial_number")
if not os.path.isfile(serial_num_file):
logger.error(
f"DriveManager error no serial_number file for disk: {disk}"
)
continue
try:
with open(serial_num_file, "r") as datafile:
serial_number = datafile.read().replace('\n', '')
except Exception as e:
logger.info(
f"DriveManager, _init_drive_status, exception: {e}")
# Read in the status for the disk
status_file = os.path.join(pathname, "status")
if not os.path.isfile(status_file):
logger.error(
f"DriveManager error no status file for disk: {disk}")
continue
try:
with open(status_file, "r") as datafile:
status = datafile.read().replace('\n', '')
# Read in the reason file if it's present
reason_file = os.path.join(pathname, "reason")
if os.path.isfile(reason_file):
with open(reason_file, "r") as datafile:
reason = datafile.read().replace('\n', '')
# Append the reason to the status file
self._drive_status[pathname] = f"{status}_{reason}"
else:
self._drive_status[pathname] = status
logger.info(
f"DriveManager, pathname: {pathname}, status: {self._drive_status[pathname]}"
)
# Remove base dcs dir since it contains no relevant data
data_str = status_file[len(self._drive_mngr_base_dir) + 1:]
# Send a message to the disk manager handler to create and transmit json msg
internal_json_msg = json.dumps({
"sensor_response_type":
"disk_status_drivemanager",
"event_path":
data_str,
"status":
self._drive_status[pathname],
"serial_number":
serial_number
})
# Send the event to disk message handler to generate json message
self._write_internal_msgQ(DiskMsgHandler.name(),
internal_json_msg)
except Exception as e:
logger.info(
f"DriveManager, _init_drive_status, exception: {e}")
logger.info("DriveManager, initialization completed")
def _notify_DiskMsgHandler(self, updated_file):
"""Send the event to the disk message handler for generating JSON message"""
# Parse out the drive location without the ending filename that changed
driveloc = os.path.dirname(updated_file)
# Check to see if the drive is present
serial_number = self._gather_data(f"{driveloc}/serial_number")
# Update the status to status_reason used throughout
if self._gather_data(f"{driveloc}/status") == "available":
status = "OK_None"
else:
status = "EMPTY_None"
disk_installed: bool
if self._gather_data(f"{driveloc}/disk_installed") == "1":
disk_installed = True
else:
disk_installed = False
disk_powered: bool
if self._gather_data(f"{driveloc}/disk_powered") == "1":
disk_powered = True
else:
disk_powered = False
# See if we need to use the previously saved serial number when disk is uninstalled
if "disk_installed" in updated_file:
# If field changed to disk being uninstalled then check for a valid serial number
if disk_installed is False and \
serial_number == "ZBX_NOTPRESENT":
if self._drive_data.get(driveloc) is not None:
serial_number = self._drive_data.get(driveloc).get(
"serial_number")
logger.info(
f"Disk was removed, s/n=ZBX_NOTPRESENT, replacing with s/n: {serial_number}"
)
# Send a message to the disk message handler to transmit
json_data = {
"sensor_response_type": "disk_status_hpi",
"event_path": driveloc[len(self._hpi_mntr_base_dir) + 1:],
"status": status,
"drawer": self._gather_data(f"{driveloc}/drawer"),
"location": self._gather_data(f"{driveloc}/location"),
"manufacturer": self._gather_data(f"{driveloc}/manufacturer"),
"productName": self._gather_data(f"{driveloc}/product_name"),
"productVersion": self._gather_data(f"{driveloc}/product_version"),
"serial_number": serial_number,
"wwn": self._gather_data(f"{driveloc}/wwn"),
"disk_installed": disk_installed,
"disk_powered": disk_powered
}
# Do nothing if the overall state has not changed anywhere
if self._drive_data.get(driveloc) is not None and \
self._drive_data.get(driveloc) == json_data:
return
# Store the JSON data into the dict for global access
if serial_number != "ZBX_NOTPRESENT":
self._drive_data[driveloc] = json_data
# Send the event to disk message handler to generate json message
self._write_internal_msgQ(DiskMsgHandler.name(), json_data)
# Restart openhpid to update HPI data for newly installed drives
if serial_number == "ZBX_NOTPRESENT" and \
disk_installed is True and \
disk_powered is True:
logger.info(
"HPImonitor, _notify_DiskMsgHandler, Restarting openhpid")
time.sleep(20)
command = "/usr/bin/systemctl restart openhpid"
response, error = self._run_command(command)
if len(error) > 0:
logger.info(f"Error restarting openhpid: {error}")
else:
logger.info("Restarted openhpid succesfully")
# Reset debug mode if persistence is not enabled
self._disable_debug_if_persist_false()
def _init_drive_data(self):
"""Initialize the dict containing HPI data for each disk"""
# Wait for the dcs-collector to populate the /tmp/dcs/hpi directory
while not os.path.isdir(self._hpi_mntr_base_dir):
logger.info(
f"HPIMonitor, dir not found: {self._hpi_mntr_base_dir} ")
logger.info(f"HPIMonitor, rechecking in {self._start_delay} secs")
time.sleep(int(self._start_delay))
enclosures = os.listdir(self._hpi_mntr_base_dir)
# Remove the 'discovery' file and any others, only care about enclosure dirs
enclosures = [enclosure for enclosure in enclosures \
if not os.path.isdir(os.path.join(self._hpi_mntr_base_dir, enclosure))]
for enclosure in enclosures:
disk_dir = os.path.join(self._hpi_mntr_base_dir, enclosure, "disk")
self._log_debug(f"initializing: {disk_dir}")
disks = os.listdir(disk_dir)
for disk in disks:
# Create the drive location path
driveloc = os.path.join(disk_dir, disk)
# Ignore the discovery file
if not os.path.isdir(driveloc):
continue
# Check to see if the drive is present
serial_number = self._gather_data(driveloc + "/serial_number")
# Update the status to status_reason used throughout
if self._gather_data(driveloc + "/status") == "available":
status = "OK_None"
else:
status = "EMPTY_None"
disk_installed: bool
if self._gather_data(driveloc + "/disk_installed") == "1":
disk_installed = True
else:
disk_installed = False
disk_powered: bool
if self._gather_data(driveloc + "/disk_powered") == "1":
disk_powered = True
else:
disk_powered = False
# Read in the date for each disk and fill into dict
json_data = {
"sensor_response_type":
"disk_status_hpi",
"event_path":
driveloc[len(self._hpi_mntr_base_dir) + 1:],
"status":
status,
"drawer":
self._gather_data(f"{driveloc}/drawer"),
"location":
self._gather_data(f"{driveloc}/location"),
"manufacturer":
self._gather_data(f"{driveloc}/manufacturer"),
"productName":
self._gather_data(f"{driveloc}/product_name"),
"productVersion":
self._gather_data(f"{driveloc}/product_version"),
"serial_number":
serial_number,
"wwn":
self._gather_data(f"{driveloc}/wwn"),
"disk_installed":
disk_installed,
"disk_powered":
disk_powered
}
logger.info(f"HPIMonitor: {str(json_data)}")
# Store the JSON data into the dict for global access
self._drive_data[driveloc] = json_data
# Send it out to initialize anyone listening
self._write_internal_msgQ(DiskMsgHandler.name(), json_data)
logger.info("HPIMonitor, initialization completed")