class MemFaultSensor(SensorThread, InternalMsgQ): """Memory fault Sensor which runs on its own thread once every power cycle and is responsible for identifying total RAM memory on the node and any errors in it using available tool/utility""" SENSOR_NAME = "MemFaultSensor" PRIORITY = 1 RESOURCE_TYPE = "node:os:memory" # section in the configuration store SYSTEM_INFORMATION_KEY = "SYSTEM_INFORMATION" SITE_ID_KEY = "site_id" CLUSTER_ID_KEY = "cluster_id" NODE_ID_KEY = "node_id" RACK_ID_KEY = "rack_id" POLLING_INTERVAL_KEY = "polling_interval" CACHE_DIR_NAME = "server" RESOURCE_ID = "0" DEFAULT_POLLING_INTERVAL = '0' PROBE = "probe" # Dependency list DEPENDENCIES = { "plugins": ["NodeDataMsgHandler", "LoggingMsgHandler"], "rpms": [] } @staticmethod def name(): """@return: name of the module.""" return MemFaultSensor.SENSOR_NAME def __init__(self, utility_instance=None): """init method""" super(MemFaultSensor, self).__init__(self.SENSOR_NAME, self.PRIORITY) # Initialize the utility instance self._utility_instance = utility_instance self.total_mem = None self.mem_path_file = None self.prev_mem = None self.fault_alert_state = "Neutral State" # Flag to indicate suspension of module self._suspended = False def initialize(self, conf_reader, msgQlist, product): """initialize configuration reader and internal msg queues""" # Initialize ScheduledMonitorThread and InternalMsgQ super(MemFaultSensor, self).initialize(conf_reader) super(MemFaultSensor, self).initialize_msgQ(msgQlist) self._site_id = self._conf_reader._get_value_with_default( self.SYSTEM_INFORMATION_KEY, COMMON_CONFIGS.get(self.SYSTEM_INFORMATION_KEY).get( self.SITE_ID_KEY), '001') self._cluster_id = self._conf_reader._get_value_with_default( self.SYSTEM_INFORMATION_KEY, COMMON_CONFIGS.get(self.SYSTEM_INFORMATION_KEY).get( self.CLUSTER_ID_KEY), '001') self._rack_id = self._conf_reader._get_value_with_default( self.SYSTEM_INFORMATION_KEY, COMMON_CONFIGS.get(self.SYSTEM_INFORMATION_KEY).get( self.RACK_ID_KEY), '001') self._node_id = self._conf_reader._get_value_with_default( self.SYSTEM_INFORMATION_KEY, COMMON_CONFIGS.get(self.SYSTEM_INFORMATION_KEY).get( self.NODE_ID_KEY), '001') # get the mem fault implementor from configuration mem_fault_utility = self._conf_reader._get_value_with_default( self.name().capitalize(), self.PROBE, "procfs") self.polling_interval = int( self._conf_reader._get_value_with_default( self.SENSOR_NAME.upper(), self.POLLING_INTERVAL_KEY, self.DEFAULT_POLLING_INTERVAL)) # Creating the instance of ToolFactory class self.tool_factory = ToolFactory() try: # Get the instance of the utility using ToolFactory self._utility_instance = self._utility_instance or \ self.tool_factory.get_instance(mem_fault_utility) # self._utility_instance.initialize() except KeyError as key_error: logger.error( "Unable to get the instance of {} \ Utility. Hence shutting down the sensor {}"\ .format(mem_fault_utility, MemFaultSensor.SENSOR_NAME)) self.shutdown() cache_dir_path = os.path.join(DATA_PATH, self.CACHE_DIR_NAME) self.MEM_FAULT_SENSOR_DATA = os.path.join( cache_dir_path, f'MEM_FAULT_SENSOR_DATA_{self._node_id}') return True def get_stored_mem_info(self): """ Get the memory info from consul""" if store.exists(self.MEM_FAULT_SENSOR_DATA): consul_data = (store.get(self.MEM_FAULT_SENSOR_DATA)).split(":") self.prev_mem = consul_data[0].strip() self.fault_alert_state = consul_data[1].strip() def put_mem_info(self, total_memory_size): """ Store the current memory in Consul""" store.put(f"{total_memory_size}:{self.fault_alert_state}", self.MEM_FAULT_SENSOR_DATA) def run(self): """Run the sensor on its own thread""" alert_type = "fault" mem_path = self._utility_instance.get_proc_memory('meminfo') if mem_path.is_file(): self.mem_path_file = mem_path.read_text() mem_info_fields = self.mem_path_file.split() if mem_info_fields[0] == 'MemTotal:': self.total_mem = mem_info_fields[1] # Get data from store if available and compare to the current value self.get_stored_mem_info() if self.prev_mem is not None: # Fault and Fault_resolved Both conditions are handled. if int(self.prev_mem) > int(self.total_mem): # update the store with new value, raise an alert of type "fault" if self.fault_alert_state == "Neutral State": self.fault_alert_state = "Fault Generated" self._generate_alert(alert_type) self.put_mem_info(self.prev_mem) elif (int(self.prev_mem) <= int(self.total_mem)) and ( self.fault_alert_state == "Fault Generated"): self.fault_alert_state = "Neutral State" alert_type = "fault_resolved" self._generate_alert(alert_type) self.put_mem_info(self.total_mem) else: self.put_mem_info(self.total_mem) else: logger.error( "MemFaultSensor: invalid file, shutting down the sensor") self.shutdown() return True else: logger.error( "MemFaultSensor: file does not exist, shutting down the sensor" ) self.shutdown() return True # Do not proceed if module is suspended # Memory sensor is going to trigger only during SSPL reboot; at reboot time a sensor # can not be in suspended state. # Commented code is retained if in future we want to make the sensor periodic, # this piece will be needed #if self._suspended is True: # self._scheduler.enter(self.polling_interval, self._priority, self.run, ()) # return # Check for debug mode being activated self._read_my_msgQ_noWait() # self scheduling is commented so that the process runs only once per SSPL reboot # Enable with correct polling_interval if in future memory sensor needs to run periodically #self._scheduler.enter(self.polling_interval, self._priority, self.run, ()) def _create_json_message(self, alert_type): """Creates a defined json message structure which can flow inside SSPL modules""" internal_json_msg = None severity_reader = SeverityReader() severity = severity_reader.map_severity(alert_type) epoch_time = str(int(time.time())) alert_id = self._get_alert_id(epoch_time) host_name = socket.gethostname() specific_info = {} specific_info_list = [] if alert_type == "fault": specific_info["event"] = \ "Total available main memory value decreased from {} kB to {} kB"\ .format(self.prev_mem, self.total_mem) elif alert_type == "fault_resolved": specific_info["event"] = \ "Total main memory value available {} kB"\ .format(self.total_mem) # populate all the data from /proc/meminfo split_strs = [ s.split(maxsplit=1) for s in self.mem_path_file.splitlines() ] dictionary_str = dict(split_strs) specific_info["meminfo"] = dictionary_str specific_info_list.append(specific_info) alert_specific_info = specific_info_list info = { "site_id": self._site_id, "cluster_id": self._cluster_id, "rack_id": self._rack_id, "node_id": self._node_id, "resource_type": self.RESOURCE_TYPE, "resource_id": self.RESOURCE_ID, "event_time": epoch_time } internal_json_msg = json.dumps({ "sensor_request_type": { "node_data": { "status": "update", "host_id": host_name, "alert_type": alert_type, "severity": severity, "alert_id": alert_id, "info": info, "specific_info": alert_specific_info } } }) return internal_json_msg def _get_alert_id(self, epoch_time): """Returns alert id which is a combination of epoch_time and salt value """ salt = str(uuid.uuid4().hex) alert_id = epoch_time + salt return alert_id def _generate_alert(self, alert_type): """Queues the message to NodeData Message Handler""" json_msg = self._create_json_message(alert_type) if json_msg: self._write_internal_msgQ(NodeDataMsgHandler.name(), json_msg) def suspend(self): """Suspends the module thread. It should be non-blocking""" super(MemFaultSensor, self).suspend() self._suspended = True def resume(self): """Resumes the module thread. It should be non-blocking""" super(MemFaultSensor, self).resume() self._suspended = False def shutdown(self): """Clean up scheduler queue and gracefully shutdown thread""" super(MemFaultSensor, self).shutdown()
class CPUFaultSensor(SensorThread, InternalMsgQ): """CPU Fault Sensor which runs on its own thread on each boot up and is responsible for sensing changes in online CPUs using available tool/utility""" SENSOR_NAME = "CPUFaultSensor" PRIORITY = 1 RESOURCE_TYPE = "node:os:cpu:core" # Section in the configuration store SYSTEM_INFORMATION_KEY = "SYSTEM_INFORMATION" SITE_ID_KEY = "site_id" CLUSTER_ID_KEY = "cluster_id" NODE_ID_KEY = "node_id" RACK_ID_KEY = "rack_id" CACHE_DIR_NAME = "server" RESOURCE_ID = "CPU-" PROBE = "probe" # Dependency list DEPENDENCIES = { "plugins": ["NodeDataMsgHandler", "LoggingMsgHandler"], "rpms": [] } @staticmethod def name(): """@return: name of the module.""" return CPUFaultSensor.SENSOR_NAME def __init__(self, utility_instance=None): """init method""" super(CPUFaultSensor, self).__init__(self.SENSOR_NAME, self.PRIORITY) # Initialize the utility instance self._utility_instance = utility_instance # CPU info self.stored_cpu_info = None self.prev_cpu_info = None self.current_cpu_info = None def initialize(self, conf_reader, msgQlist, product): """initialize configuration reader and internal msg queues""" # Initialize ScheduledMonitorThread and InternalMsgQ super(CPUFaultSensor, self).initialize(conf_reader) super(CPUFaultSensor, self).initialize_msgQ(msgQlist) self._site_id = self._conf_reader._get_value_with_default( self.SYSTEM_INFORMATION_KEY, COMMON_CONFIGS.get(self.SYSTEM_INFORMATION_KEY).get( self.SITE_ID_KEY), '001') self._cluster_id = self._conf_reader._get_value_with_default( self.SYSTEM_INFORMATION_KEY, COMMON_CONFIGS.get(self.SYSTEM_INFORMATION_KEY).get( self.CLUSTER_ID_KEY), '001') self._rack_id = self._conf_reader._get_value_with_default( self.SYSTEM_INFORMATION_KEY, COMMON_CONFIGS.get(self.SYSTEM_INFORMATION_KEY).get( self.RACK_ID_KEY), '001') self._node_id = self._conf_reader._get_value_with_default( self.SYSTEM_INFORMATION_KEY, COMMON_CONFIGS.get(self.SYSTEM_INFORMATION_KEY).get( self.NODE_ID_KEY), '001') # get the cpu fault implementor from configuration cpu_fault_utility = self._conf_reader._get_value_with_default( self.name().capitalize(), self.PROBE, 'sysfs') # Creating the instance of ToolFactory class self.tool_factory = ToolFactory() try: # Get the instance of the utility using ToolFactory self._utility_instance = self._utility_instance or \ self.tool_factory.get_instance(cpu_fault_utility) except Exception as e: logger.error( f"Error while initializing, shutting down CPUFaultSensor : {e}" ) self.shutdown() cache_dir_path = os.path.join(DATA_PATH, self.CACHE_DIR_NAME) self.CPU_FAULT_SENSOR_DATA = os.path.join( cache_dir_path, f'CPU_FAULT_SENSOR_DATA_{self._node_id}') return True def read_stored_cpu_info(self): """Read the most recent stored cpu info""" try: if self.stored_cpu_info is None: self.stored_cpu_info = store.get(self.CPU_FAULT_SENSOR_DATA) if self.stored_cpu_info is not None and self._node_id in self.stored_cpu_info.keys( ): self.prev_cpu_info = self.stored_cpu_info[ self._node_id]['CPU_LIST'] except Exception as e: logger.error( f"Error while reading stored cpu info, shutting down CPUFaultSensor : {e}" ) self.shutdown() def read_current_cpu_info(self): """Read current cpu info""" try: self.current_cpu_info = self._utility_instance.get_cpu_info() except Exception as e: logger.error( f"Error while reading current cpu info, shutting down CPUFaultSensor : {e}" ) self.shutdown() def run(self): """Run the sensor on its own thread""" # Check for debug mode being activated self._read_my_msgQ_noWait() # Read recent stored cpu info self.read_stored_cpu_info() # Store alerts to be sent here self.alerts_for = {} # Specific info field for alerts self.specific_info = [] # Read current cpu info self.read_current_cpu_info() to_update = False # Compare with previous cpu info # If a cpu is present in prev_cpu_info and not present in current_cpu_info : fault alert is generated # If a cpu is present in current_cpu_info and not present in prev_cpu_info : two possibilities # 1) if cpu has an outstanding fault alert : it is a repaired cpu, hence generate fault_resolved # 2) if cpu has no outstanding alert : it is a newly added cpu, do not do anything try: if self.prev_cpu_info: if self.current_cpu_info != self.prev_cpu_info: # Create a set of all relevant cpus cpu_list = set(self.prev_cpu_info + self.current_cpu_info) # Iterate through the set for cpu in cpu_list: if cpu not in self.current_cpu_info and cpu not in self.stored_cpu_info[ self._node_id]['FAULT_LIST']: # This is a failed cpu self.stored_cpu_info[ self._node_id]['FAULT_LIST'].append(cpu) self.alerts_for[cpu] = "fault" elif cpu not in self.prev_cpu_info and cpu in self.stored_cpu_info[ self._node_id]['FAULT_LIST']: # This is a repaired cpu self.alerts_for[cpu] = "fault_resolved" # Update stored cpu info for next run self.stored_cpu_info[ self._node_id]['CPU_LIST'] = self.current_cpu_info to_update = True else: # Previous cpu info not available, need to store current info if not self.stored_cpu_info: # No info is available self.stored_cpu_info = {} # Add info for the current node self.stored_cpu_info[self._node_id] = {} self.stored_cpu_info[ self._node_id]['CPU_LIST'] = self.current_cpu_info self.stored_cpu_info[self._node_id]['FAULT_LIST'] = [] # Update stored cpu info to_update = True except Exception as e: logger.error( f"Error while processing cpu info, shutting down CPUFaultSensor : {e}" ) self.shutdown() # Send alerts for cpu, alert_type in self.alerts_for.items(): if self._generate_alert( cpu, alert_type) == True and alert_type == "fault_resolved": # Delete from the FAULT_LIST self.stored_cpu_info[self._node_id]['FAULT_LIST'].remove(cpu) # Update stored cpu info if to_update: store.put(self.stored_cpu_info, self.CPU_FAULT_SENSOR_DATA) def fill_specific_info(self): """Fills the specific info to be sent via alert""" if not self.specific_info: # Create a set of all relevant cpus cpu_list = set(self.prev_cpu_info + self.current_cpu_info) # Iterate through the set for cpu in cpu_list: item = {} item['resource_id'] = self.RESOURCE_ID + str(cpu) # Keep default state online item['state'] = "online" if cpu in self.alerts_for.keys(): if self.alerts_for[cpu] == "fault": item['state'] = "offline" self.specific_info.append(item) def _create_json_message(self, cpu, alert_type): """Creates a defined json message structure which can flow inside SSPL modules""" internal_json_msg = None severity_reader = SeverityReader() severity = severity_reader.map_severity(alert_type) epoch_time = str(int(time.time())) alert_id = self._get_alert_id(epoch_time) host_name = socket.gethostname() # Populate specific info self.fill_specific_info() alert_specific_info = self.specific_info info = { "site_id": self._site_id, "cluster_id": self._cluster_id, "rack_id": self._rack_id, "node_id": self._node_id, "resource_type": self.RESOURCE_TYPE, "resource_id": self.RESOURCE_ID + str(cpu), "event_time": epoch_time } internal_json_msg = json.dumps({ "sensor_request_type": { "node_data": { "status": "update", "host_id": host_name, "alert_type": alert_type, "severity": severity, "alert_id": alert_id, "info": info, "specific_info": alert_specific_info } } }) return internal_json_msg def _get_alert_id(self, epoch_time): """Returns alert id which is a combination of epoch_time and salt value """ salt = str(uuid.uuid4().hex) alert_id = epoch_time + salt return alert_id def _generate_alert(self, cpu, alert_type): """Queues the message to NodeData Message Handler""" try: json_msg = self._create_json_message(cpu, alert_type) if json_msg: # RAAL stands for - RAise ALert logger.info(f"RAAL: {json_msg}") self._write_internal_msgQ(NodeDataMsgHandler.name(), json_msg) return True except Exception as e: logger.error(f"Exception while sending alert : {e}") return False def shutdown(self): """Clean up scheduler queue and gracefully shutdown thread""" super(CPUFaultSensor, self).shutdown()
class SASPortSensor(SensorThread, InternalMsgQ): """SAS Port Sensor which runs on its own thread periodically and is responsible for sensing changes is SAS ports/cable using available tool/utility""" SENSOR_NAME = "SASPortSensor" PRIORITY = 1 RESOURCE_TYPE = "node:interface:sas" # section in the configuration store SYSTEM_INFORMATION = "SYSTEM_INFORMATION" SITE_ID = "site_id" CLUSTER_ID = "cluster_id" NODE_ID = "node_id" RACK_ID = "rack_id" POLLING_INTERVAL = "polling_interval" CACHE_DIR_NAME = "server" RESOURCE_ID = "SASHBA-0" DEFAULT_POLLING_INTERVAL = '30' PROBE = "probe" # Dependency list DEPENDENCIES = { "plugins": ["NodeDataMsgHandler", "LoggingMsgHandler"], "rpms": [] } # Number of SAS Ports NUM_SAS_PORTS = 4 # Number of Phys in a Port NUM_PHYS_PER_PORT = 4 # Current Data Version CURRENT_DATA_VERSION = 1 @staticmethod def name(): """@return: name of the module.""" return SASPortSensor.SENSOR_NAME def __init__(self, utility_instance=None): """init method""" super(SASPortSensor, self).__init__(self.SENSOR_NAME, self.PRIORITY) # Initialize the utility instance self._utility_instance = utility_instance self.phy_dir_to_linkrate_mapping = None # Flag to indicate suspension of module self._suspended = False self._count = 0 self.phy_link_count = 0 self.sas_ports_status = {} self.port_phy_list_dict = {} self.sas_phy_stored_alert = None def initialize(self, conf_reader, msgQlist, product): """initialize configuration reader and internal msg queues""" # Initialize ScheduledMonitorThread and InternalMsgQ super(SASPortSensor, self).initialize(conf_reader) super(SASPortSensor, self).initialize_msgQ(msgQlist) self._site_id = Conf.get(GLOBAL_CONF, SITE_ID_KEY, 'DC01') self._rack_id = Conf.get(GLOBAL_CONF, RACK_ID_KEY, 'RC01') self._node_id = Conf.get(GLOBAL_CONF, NODE_ID_KEY, 'SN01') self._cluster_id = Conf.get(GLOBAL_CONF, CLUSTER_ID_KEY, 'CC01') # Get the sas port implementor from configuration sas_port_utility = Conf.get( SSPL_CONF, f"{self.name().capitalize()}>{self.PROBE}", "sysfs") self.polling_interval = int( Conf.get(SSPL_CONF, f"{self.SENSOR_NAME.upper()}>{self.POLLING_INTERVAL}", self.DEFAULT_POLLING_INTERVAL)) # Creating the instance of ToolFactory class self.tool_factory = ToolFactory() cache_dir_path = os.path.join(DATA_PATH, self.CACHE_DIR_NAME) self.SAS_PORT_SENSOR_DATA = os.path.join( cache_dir_path, f'SAS_PORT_SENSOR_DATA_{self._node_id}') alert_type = None try: # Get the instance of the utility using ToolFactory self._utility_instance = self._utility_instance or \ self.tool_factory.get_instance(sas_port_utility) self._utility_instance.initialize() phy_status = None link_value_phy_status_collection = () # Call to sas phy dirctory which will return a dictionary # which has phy_name to negotiated link rate mapping # Ex: {"phy-0:0": "<12.0, Unknown>"} self.phy_dir_to_linkrate_mapping = \ self._utility_instance.get_phy_negotiated_link_rate() # Iterate over populated dictionary and restructure it # Ex: if phy-0:0 is 12.0/6.0/3.0, considered as UP. # {"phy-0:0": ("link_rate", <Up/Down>)} for phy, value in self.phy_dir_to_linkrate_mapping.items(): if 'Gbit'.lower() in value.strip().lower(): phy_status = 'up' # Increment global phy_link count for UP status self.phy_link_count += 1 else: phy_status = 'fault' link_value_phy_status_collection = (value, phy_status) self.phy_dir_to_linkrate_mapping[ phy] = link_value_phy_status_collection # Get the stored previous alert info self.sas_phy_stored_alert = store.get(self.SAS_PORT_SENSOR_DATA) self.check_and_send_alert() except KeyError as key_error: logger.error("Unable to get the instance of {} \ Utility. Hence shutting down the sensor".format( sas_port_utility)) self.shutdown() except Exception as e: if e == errno.ENOENT: logger.error("Problem occured while reading from sas_phy \ directory. directory path doesn't directory. Hence \ shuting down the sensor") elif e == errno.EACCES: logger.error( "Problem occured while reading from sas_phy directory. \ Not enough permission to read from the directory. \ Hence shuting down the sensor") else: logger.error( "Problem occured while reading from sas_phy directory. \ {0}. Hence shuting down the sensor".format(e)) self.shutdown() return True def update_sas_ports_status(self): """ Reads current phy status and updates port connectivity status Assumption : phys will be present in multiples of 4 """ phy_list = [*self.phy_dir_to_linkrate_mapping] phy_list = sort_phy_list(phy_list) # Now we have a sorted list of phys # Phys 0-3 for the 0th sas port, and so on in groups of 4 phys # List containing status of all phys hba = [] for phy in phy_list: if self.phy_dir_to_linkrate_mapping[phy][1] == 'up': hba.append(1) else: hba.append(0) for i in range(0, self.NUM_SAS_PORTS): # Save phy names forming this port for future use self.port_phy_list_dict[i] = phy_list[ self.NUM_PHYS_PER_PORT * i : \ self.NUM_PHYS_PER_PORT * i + self.NUM_PHYS_PER_PORT ] # Check port status s = set(hba[self.NUM_PHYS_PER_PORT * i:self.NUM_PHYS_PER_PORT * i + self.NUM_PHYS_PER_PORT]) if len(s) == 1 and 0 in s: port_status = 'down' elif len(s) == 1 and 1 in s: port_status = 'up' else: port_status = 'degraded' # Store the data self.sas_ports_status[i] = port_status def check_and_send_conn_alert(self): """ Sends conn fault alert if all phys go down Sends conn fault_resolved alert if at least 1 sas port (4 phys) comes up """ # Case 1 : all fault for fault alert cur_all_fault = True # Case 2 : all fault_resolved for fault_resolved alert cur_all_fault_resolved = True # Previous conn alert that was sent prev_conn_alert = self.sas_phy_stored_alert['conn'] # Current for port, value in self.sas_phy_stored_alert.items(): if port in ['version', 'conn']: # This is key for conn alert, skip continue # Case 1 : All faults in current status if value != 'fault': cur_all_fault = False # Case 2 : All fault_resolved in current status elif value != 'fault_resolved': cur_all_fault_resolved = False if prev_conn_alert == 'fault_resolved' and cur_all_fault: # Send conn fault alert alert_type = 'fault' self._generate_alert(alert_type, -1) self.sas_phy_stored_alert['conn'] = alert_type elif prev_conn_alert == 'fault' and cur_all_fault_resolved: # Send conn fault_resolved alert alert_type = 'fault_resolved' self._generate_alert(alert_type, -1) self.sas_phy_stored_alert['conn'] = alert_type def handle_current_version_data(self): """Contains logic to check and send alert if data has version == 1.""" # Compare current status of each port with previous alert_type for port, value in self.sas_phy_stored_alert.items(): if port in ['version', 'conn']: # Skip continue if value == 'fault_resolved' and \ self.sas_ports_status[port] == 'down': alert_type = 'fault' self._generate_alert(alert_type, port) self.sas_phy_stored_alert[port] = alert_type elif value == 'fault' and \ self.sas_ports_status[port] == 'up': alert_type = 'fault_resolved' self._generate_alert(alert_type, port) self.sas_phy_stored_alert[port] = alert_type # See if conn failure/conn resolved alert needs to be sent self.check_and_send_conn_alert() # Save data to store store.put(self.sas_phy_stored_alert, self.SAS_PORT_SENSOR_DATA) def check_and_send_alert(self): """Checks whether conditions are met and sends alert if required Alerts will be sent if - 1. All 4 phys of a sas port go up -> down : fault alert 2. All 4 phys of a sas port come down -> up : fault_resolved alert Sensor data stored in persistent storage is a dict of { sas_port_number : alert_type } """ # Update sas ports status self.update_sas_ports_status() # Check the version of stored alert version = None try: # Try to get the version # Exception will be raised if stored alert is None or no Version is available version = self.sas_phy_stored_alert['version'] except Exception: logger.warning( f"Found no data or old data format for SASPortSensor, \ updating data format to version {self.CURRENT_DATA_VERSION}" ) # Versioning is not implemented or there is no data, write new data # Initialize dummy fault_resolved for all sas ports and conn self.sas_phy_stored_alert = {} self.sas_phy_stored_alert['version'] = self.CURRENT_DATA_VERSION self.sas_phy_stored_alert['conn'] = 'fault_resolved' for i in range(0, self.NUM_SAS_PORTS): self.sas_phy_stored_alert[i] = 'fault_resolved' # Save data to store store.put(self.sas_phy_stored_alert, self.SAS_PORT_SENSOR_DATA) if version == self.CURRENT_DATA_VERSION: self.handle_current_version_data() def run(self): """Run the sensor on its own thread""" alert_type = None status = None new_phy_up = 0 new_phy_down = 0 # Do not proceed if module is suspended if self._suspended == True: self._scheduler.enter(self.polling_interval, self._priority, self.run, ()) return # Check for debug mode being activated self._read_my_msgQ_noWait() try: phy_link_rate_dict = \ self._utility_instance.get_phy_negotiated_link_rate() if phy_link_rate_dict: for key, value in phy_link_rate_dict.items(): link_rate = value.strip() prev_linkrate_value = \ self.phy_dir_to_linkrate_mapping[key][0].strip() prev_alert_type = \ self.phy_dir_to_linkrate_mapping[key][1].strip() status = prev_alert_type # Compare local dict wrt global dictionary for change in the # negotiated link rate if link_rate.lower() != prev_linkrate_value.lower(): # If current link rate has no value like 12/6/3 Gbit # and previously it was up, then it's a fault condition if 'Gbit'.lower() not in link_rate.lower( ) and prev_alert_type.lower() == 'up': # Increment count for new phy down which were up previously new_phy_down += 1 # Make respective phy_status as fault status = 'fault' # Check if 12/6/3 Gbit is there in the current link rate and # the previous alert_type is fault. If so, means phy is Up again elif 'Gbit'.lower() in link_rate.lower( ) and prev_alert_type.lower() == 'fault': # Mark respective phy_status as Up status = 'up' # Increment count for new phy up new_phy_up += 1 # Finally update the global dict with current link rate # and respctive phy status self.phy_dir_to_linkrate_mapping[key] = (link_rate, status) # Get current phy status i.e number of Up phys new_phy_link_count = self.phy_link_count + new_phy_up - new_phy_down # Get the last sent alert info self.sas_phy_stored_alert = store.get( self.SAS_PORT_SENSOR_DATA) self.check_and_send_alert() # Update current active phy count for next iteration self.phy_link_count = new_phy_link_count except Exception as ae: logger.exception(ae) # Fire every 30 seconds to see if there's a change in the phy status self._scheduler.enter(self.polling_interval, self._priority, self.run, ()) def _create_json_message(self, alert_type, port): """Creates a defined json message structure which can flow inside SSPL modules""" internal_json_msg = None severity_reader = SeverityReader() severity = severity_reader.map_severity(alert_type) epoch_time = str(int(time.time())) alert_id = self._get_alert_id(epoch_time) host_name = socket.gethostname() specific_info = {} specific_info_list = [] description = "N/A" # specific_info will contain all 16 phys for conn level alert # Only 4 phys for port level alert for key, val in self.phy_dir_to_linkrate_mapping.items(): if port != -1: # This is a port level alert, skip phys that are not relevant if key not in self.port_phy_list_dict[port]: # Skip adding this phy continue # Key will be phy-0:0. So, aplit it using ':' # So, structure will be SASHBA-0:phy-0 phy_number = key.split(":")[1] specific_info[ "resource_id"] = self.RESOURCE_ID + ':' + "phy-" + phy_number specific_info[ "negotiated_link_rate"] = self.phy_dir_to_linkrate_mapping[ key][0].strip() specific_info_list.append(specific_info) specific_info = {} alert_specific_info = specific_info_list if port == -1: # This is a SAS HBA level connection alert if alert_type == 'fault': description = "SAS connection error detected in SAS HBA %s." % self.RESOURCE_ID elif alert_type == 'fault_resolved': description = "SAS connection re-established in SAS HBA %s." % self.RESOURCE_ID info = { "site_id": self._site_id, "cluster_id": self._cluster_id, "rack_id": self._rack_id, "node_id": self._node_id, "resource_type": self.RESOURCE_TYPE, # node:interface:sas "resource_id": self.RESOURCE_ID, # SASHBA-0 "event_time": epoch_time, "description": description } else: # This is a port level alert if alert_type == 'fault': description = ( "No connectivity detected on the SAS port %s, possible" "causes could be missing SAS cable, bad cable connection," "faulty cable or SAS port failure." % port) elif alert_type == 'fault_resolved': description = "Connection established on SAS port." info = { "site_id": self._site_id, "cluster_id": self._cluster_id, "rack_id": self._rack_id, "node_id": self._node_id, "resource_type": self.RESOURCE_TYPE + ':port', # node:interface:sas:port "resource_id": self.RESOURCE_ID + f'-port-{port}', # SASHBA-0-port-0 "event_time": epoch_time, "description": description } internal_json_msg = json.dumps({ "sensor_request_type": { "node_data": { "status": "update", "host_id": host_name, "alert_type": alert_type, "severity": severity, "alert_id": alert_id, "info": info, "specific_info": alert_specific_info } } }) return internal_json_msg def _get_alert_id(self, epoch_time): """Returns alert id which is a combination of epoch_time and salt value """ salt = str(uuid.uuid4().hex) alert_id = epoch_time + salt return alert_id def _generate_alert(self, alert_type, port): """Queues the message to NodeData Message Handler""" json_msg = self._create_json_message(alert_type, port) if json_msg: self._write_internal_msgQ(NodeDataMsgHandler.name(), json_msg) def suspend(self): """Suspends the module thread. It should be non-blocking""" super(SASPortSensor, self).suspend() self._suspended = True def resume(self): """Resumes the module thread. It should be non-blocking""" super(SASPortSensor, self).resume() self._suspended = False def shutdown(self): """Clean up scheduler queue and gracefully shutdown thread""" super(SASPortSensor, self).shutdown()
class NodeData(Debug): """Obtains data about the node and makes it available""" SENSOR_NAME = "NodeData" # conf attribute initialization PROBE = 'probe' @staticmethod def name(): """@return: name of the module.""" return NodeData.SENSOR_NAME @staticmethod def impact(): """Returns impact of the module.""" return ("Server CPU, network, disk space, process and local mount " "data can not be monitored.") def __init__(self): super(NodeData, self).__init__() self.os_utils = OSUtils() self._epoch_time = str(int(time.time())) # Total number of CPUs self.cpus = psutil.cpu_count() self.host_id = self.os_utils.get_fqdn() # Calculate the load averages on separate blocking threads self.load_1min_average = [] self.load_5min_average = [] self.load_15min_average = [] self.prev_bmcip = None load_1min_avg = threading.Thread(target=self._load_1min_avg).start() load_5min_avg = threading.Thread(target=self._load_5min_avg).start() load_15min_avg = threading.Thread(target=self._load_15min_avg).start() self.conf_reader = ConfigReader() nw_fault_utility = Conf.get( SSPL_CONF, f"{self.name().capitalize()}>{self.PROBE}", "sysfs") self._utility_instance = None try: # Creating the instance of ToolFactory class self.tool_factory = ToolFactory() # Get the instance of the utility using ToolFactory self._utility_instance = self._utility_instance or \ self.tool_factory.get_instance(nw_fault_utility) if self._utility_instance: # Initialize the path as /sys/class/net/ self.nw_interface_path = self._utility_instance.get_sys_dir_path( 'net') except KeyError as key_error: logger.error( f'NodeData, Unable to get the instance of {nw_fault_utility} Utility' ) except Exception as err: logger.error( f'NodeData, Problem occured while getting the instance of {nw_fault_utility}' ) def read_data(self, subset, debug, units="MB"): """Updates data based on a subset""" self._set_debug(debug) self._log_debug("read_data, subset: %s, units: %s" % (subset, units)) try: # Determine the units factor value self.units_factor = 1 if units == "GB": self.units_factor = 1000000000 elif units == "MB": self.units_factor = 1000000 elif units == "KB": self.units_factor = 1000 self.host_id = self.os_utils.get_fqdn() # get_fqdn() function checks the socket.gethostname() to get the host name if it not available # then it try to find host name from socket.gethostbyaddr(socket.gethostname())[0] and return the # meaningful host name. self.local_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S %Z') # Branch off and gather data based upon value sent into subset if subset == "host_update": self._get_host_update_data() elif subset == "local_mount_data": self._get_local_mount_data() elif subset == "cpu_data": self._get_cpu_data() elif subset == "if_data": self._get_if_data() elif subset == "disk_space_alert": self._get_disk_space_alert_data() except Exception as e: raise Exception(f"Failed to read data, {e}") return True def _get_host_update_data(self): """Retrieves node information for the host_update json message""" logged_in_users = [] uname_keys = ("sysname", "nodename", "version", "release", "machine") self.up_time = int(psutil.boot_time()) self.boot_time = self._epoch_time self.uname = dict(zip(uname_keys, os.uname())) self.total_memory = dict(psutil.virtual_memory()._asdict()) self.process_count = len(psutil.pids()) for users in psutil.users(): logged_in_users.append(dict(users._asdict())) self.logged_in_users = logged_in_users # Calculate the current number of running processes at this moment total_running_proc = 0 for proc in psutil.process_iter(): try: pinfo = proc.as_dict(attrs=['status']) if pinfo['status'] not in (psutil.STATUS_ZOMBIE, psutil.STATUS_DEAD, psutil.STATUS_STOPPED, psutil.STATUS_IDLE, psutil.STATUS_SLEEPING): total_running_proc += 1 except psutil.NoSuchProcess: logger.warn( f"(psutil) Process '{proc.name()}' exited unexpectedly.") self.running_process_count = total_running_proc def _get_local_mount_data(self): """Retrieves node information for the local_mount_data json message""" self.total_space = int(psutil.disk_usage("/")[0]) // int( self.units_factor) self.free_space = int(psutil.disk_usage("/")[2]) // int( self.units_factor) self.total_swap = int(psutil.swap_memory()[0]) // int( self.units_factor) self.free_swap = int(psutil.swap_memory()[2]) // int(self.units_factor) self.free_inodes = int(100 - math.ceil((float(os.statvfs("/").f_files - os.statvfs("/").f_ffree) \ / os.statvfs("/").f_files) * 100)) def _get_cpu_data(self): """Retrieves node information for the cpu_data json message""" cpu_core_usage_dict = dict() cpu_data = psutil.cpu_times_percent() self._log_debug( "_get_cpu_data, cpu_data: %s %s %s %s %s %s %s %s %s %s" % cpu_data) self.csps = 0 # What the hell is csps - cycles per second? self.user_time = int(cpu_data[0]) self.nice_time = int(cpu_data[1]) self.system_time = int(cpu_data[2]) self.idle_time = int(cpu_data[3]) self.iowait_time = int(cpu_data[4]) self.interrupt_time = int(cpu_data[5]) self.softirq_time = int(cpu_data[6]) self.steal_time = int(cpu_data[7]) self.cpu_usage = psutil.cpu_percent(interval=1, percpu=False) # Array to hold data about each CPU core self.cpu_core_data = [] index = 0 while index < self.cpus: self._log_debug( "_get_cpu_data, index: %s, 1 min: %s, 5 min: %s, 15 min: %s" % (index, self.load_1min_average[index], self.load_5min_average[index], self.load_15min_average[index])) cpu_core_data = { "coreId": index, "load1MinAvg": int(self.load_1min_average[index]), "load5MinAvg": int(self.load_5min_average[index]), "load15MinAvg": int(self.load_15min_average[index]), "ips": 0 } self.cpu_core_data.append(cpu_core_data) index += 1 def _get_if_data(self): """Retrieves node information for the if_data json message""" net_data = psutil.net_io_counters(pernic=True) # Array to hold data about each network interface self.if_data = [] bmc_data = self._get_bmc_info() for interface, if_data in net_data.items(): self._log_debug("_get_if_data, interface: %s %s" % (interface, net_data)) nw_status = self._fetch_nw_status() nw_cable_conn_status = self.fetch_nw_cable_conn_status(interface) if_data = { "ifId": interface, "networkErrors": (net_data[interface].errin + net_data[interface].errout), "droppedPacketsIn": net_data[interface].dropin, "packetsIn": net_data[interface].packets_recv, "trafficIn": net_data[interface].bytes_recv, "droppedPacketsOut": net_data[interface].dropout, "packetsOut": net_data[interface].packets_sent, "trafficOut": net_data[interface].bytes_sent, "nwStatus": nw_status[interface][0], "ipV4": nw_status[interface][1], "nwCableConnStatus": nw_cable_conn_status } self.if_data.append(if_data) self.if_data.append(bmc_data) def _fetch_nw_status(self): nw_dict = {} nws = os.popen("ip --br a | awk '{print $1, $2, $3}'").read().split( '\n')[:-1] for nw in nws: if nw.split(' ')[2]: ip = nw.split(' ')[2].split("/")[0] else: ip = "" nw_dict[nw.split(' ')[0]] = [nw.split(' ')[1], ip] logger.debug("network info going is : {}".format(nw_dict)) return nw_dict def fetch_nw_cable_conn_status(self, interface): carrier_status = None try: carrier_status = Network().get_link_state(interface) except NetworkError as err: # NetworkError i.e. all OSError exceptions indicate that # the carrier file is not available to access which # constitute the UNKOWN status for network cable. logger.debug(err) carrier_status = "UNKNOWN" except Exception as e: # All other exceptions are unexpected and are logged as errors. logger.excpetion( "Problem occured while reading from nw carrier file:" f" {self.nw_interface_path}/{interface}/carrier. Error: {e}") return carrier_status def _get_bmc_info(self): """ nwCableConnection will be default UNKNOWN, Until solution to find bmc eth port cable connection status is found. """ try: bmcdata = { 'ifId': 'ebmc0', 'ipV4Prev': "", 'ipV4': "", 'nwStatus': "DOWN", 'nwCableConnStatus': 'UNKNOWN' } ipdata = sp.Popen( "sudo ipmitool lan print", shell=True, stdout=sp.PIPE, stderr=sp.PIPE).communicate()[0].decode().strip() bmcip = re.findall("\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", ipdata) if bmcip: bmcip = bmcip[0] pingbmchost = "ping -c1 -W1 -q " + bmcip child = sp.Popen(pingbmchost.split(), stdout=sp.PIPE) streamdata = child.communicate( )[0] #child must be communicated before fetching return code. retcode = child.returncode if self.prev_bmcip is not None and self.prev_bmcip != bmcip: bmcdata['ipV4Prev'] = self.prev_bmcip bmcdata['ipV4'] = bmcip self.prev_bmcip = bmcip else: self.prev_bmcip = bmcdata['ipV4Prev'] = bmcdata[ 'ipV4'] = bmcip if retcode == 0: bmcdata['nwStatus'] = "UP" else: logger.warn("BMC Host:{0} is not reachable".format(bmcip)) except Exception as e: logger.error( "Exception occurs while fetching bmc_info:{}".format(e)) return bmcdata def _get_disk_space_alert_data(self): """Retrieves node information for the disk_space_alert_data json message""" self.total_space = int(psutil.disk_usage("/")[0]) // int( self.units_factor) self.free_space = int(psutil.disk_usage("/")[2]) // int( self.units_factor) self.disk_used_percentage = psutil.disk_usage("/")[3] def _load_1min_avg(self): """Loop forever calculating the one minute average load""" # Initialize list to -1 indicating the time interval has not occurred yet index = 0 while index < self.cpus: self.load_1min_average.append(-1) index += 1 while True: # API call blocks for one minute and then returns the value self.load_1min_average = psutil.cpu_percent(interval=1, percpu=True) def _load_5min_avg(self): """Loop forever calculating the five minute average load""" # Initialize list to -1 indicating the time interval has not occurred yet index = 0 while index < self.cpus: self.load_5min_average.append(-1) index += 1 while True: # API call blocks for five minutes and then returns the value self.load_5min_average = psutil.cpu_percent(interval=5, percpu=True) def _load_15min_avg(self): """Loop forever calculating the fifteen minute average load""" # Initialize list to -1 indicating the time interval has not occurred yet index = 0 while index < self.cpus: self.load_15min_average.append(-1) index += 1 while True: # API call blocks for fifteen minutes and then returns the value self.load_15min_average = psutil.cpu_percent(interval=15, percpu=True)
class SASPortSensor(SensorThread, InternalMsgQ): """SAS Port Sensor which runs on its own thread periodically and is responsible for sensing changes is SAS ports/cable using available tool/utility""" SENSOR_NAME = "SASPortSensor" PRIORITY = 1 RESOURCE_TYPE = "node:interface:sas" # section in the configuration store SYSTEM_INFORMATION = "SYSTEM_INFORMATION" SITE_ID = "site_id" CLUSTER_ID = "cluster_id" NODE_ID = "node_id" RACK_ID = "rack_id" POLLING_INTERVAL = "polling_interval" CACHE_DIR_NAME = "server" RESOURCE_ID = "SASHBA-0" DEFAULT_POLLING_INTERVAL = '30' PROBE = "probe" # Dependency list DEPENDENCIES = { "plugins": ["NodeDataMsgHandler", "LoggingMsgHandler"], "rpms": [] } MIN_PHY_COUNT = 4 @staticmethod def name(): """@return: name of the module.""" return SASPortSensor.SENSOR_NAME def __init__(self, utility_instance=None): """init method""" super(SASPortSensor, self).__init__(self.SENSOR_NAME, self.PRIORITY) # Initialize the utility instance self._utility_instance = utility_instance self.phy_dir_to_linkrate_mapping = None # Flag to indicate suspension of module self._suspended = False self._count = 0 self.phy_link_count = 0 def initialize(self, conf_reader, msgQlist, product): """initialize configuration reader and internal msg queues""" # Initialize ScheduledMonitorThread and InternalMsgQ super(SASPortSensor, self).initialize(conf_reader) super(SASPortSensor, self).initialize_msgQ(msgQlist) self._site_id = self._conf_reader._get_value_with_default( self.SYSTEM_INFORMATION, COMMON_CONFIGS.get(self.SYSTEM_INFORMATION).get(self.SITE_ID), '001') self._cluster_id = self._conf_reader._get_value_with_default( self.SYSTEM_INFORMATION, COMMON_CONFIGS.get(self.SYSTEM_INFORMATION).get(self.CLUSTER_ID), '001') self._rack_id = self._conf_reader._get_value_with_default( self.SYSTEM_INFORMATION, COMMON_CONFIGS.get(self.SYSTEM_INFORMATION).get(self.RACK_ID), '001') self._node_id = self._conf_reader._get_value_with_default( self.SYSTEM_INFORMATION, COMMON_CONFIGS.get(self.SYSTEM_INFORMATION).get(self.NODE_ID), '001') # Get the sas port implementor from configuration sas_port_utility = self._conf_reader._get_value_with_default( self.name().capitalize(), self.PROBE, "sysfs") self.polling_interval = int( self._conf_reader._get_value_with_default( self.SENSOR_NAME.upper(), self.POLLING_INTERVAL, self.DEFAULT_POLLING_INTERVAL)) # Creating the instance of ToolFactory class self.tool_factory = ToolFactory() cache_dir_path = os.path.join(DATA_PATH, self.CACHE_DIR_NAME) self.SAS_PORT_SENSOR_DATA = os.path.join( cache_dir_path, f'SAS_PORT_SENSOR_DATA_{self._node_id}') alert_type = None try: # Get the instance of the utility using ToolFactory self._utility_instance = self._utility_instance or \ self.tool_factory.get_instance(sas_port_utility) self._utility_instance.initialize() phy_status = None link_value_phy_status_collection = () # Call to sas phy dirctory which will return a dictionary # which has phy_name to negotiated link rate mapping # Ex: {"phy-0:0": "<12.0, Unknown>"} self.phy_dir_to_linkrate_mapping = \ self._utility_instance.get_phy_negotiated_link_rate() # Iterate over populated dictionary and restructure it # Ex: if phy-0:0 is 12.0/6.0/3.0, considered as UP. # {"phy-0:0": ("link_rate", <Up/Down>)} for phy, value in self.phy_dir_to_linkrate_mapping.items(): if 'Gbit'.lower() in value.strip().lower(): phy_status = 'Up' # Increment global phy_link count for UP status self.phy_link_count += 1 else: phy_status = 'fault' link_value_phy_status_collection = (value, phy_status) self.phy_dir_to_linkrate_mapping[ phy] = link_value_phy_status_collection # Get the stored previous alert info self.sas_phy_stored_alert = store.get(self.SAS_PORT_SENSOR_DATA) self.check_and_send_alert(self.phy_link_count) except KeyError as key_error: logger.error("Unable to get the instance of {} \ Utility. Hence shutting down the sensor".format( sas_port_utility)) self.shutdown() except Exception as e: if e == errno.ENOENT: logger.error("Problem occured while reading from sas_phy \ directory. directory path doesn't directory. Hence \ shuting down the sensor") elif e == errno.EACCES: logger.error( "Problem occured while reading from sas_phy directory. \ Not enough permission to read from the directory. \ Hence shuting down the sensor") else: logger.error( "Problem occured while reading from sas_phy directory. \ {0}. Hence shuting down the sensor".format(e)) self.shutdown() return True def check_and_send_alert(self, new_phy_link_count): """Checks whether conditions are met and sends alert if required Alerts will be sent if - 1. All phys are down -> fault alert 2. 4 phys are up -> fault_resolved alert 3. Next group of 4 phys comes up -> informational alert Sensor data stored in Consul is a tuple (alert_type, phy_link_count) """ if self.sas_phy_stored_alert == None: # No info is stored for this node in Consul # Initialize alert_type to dummy fault_resolved self.sas_phy_stored_alert = ('fault_resolved', new_phy_link_count) # Save data to Consul store.put(self.sas_phy_stored_alert, self.SAS_PORT_SENSOR_DATA) elif self.sas_phy_stored_alert[0] == 'fault': # Previous alert sent for this node was fault, check if fault is resolved if new_phy_link_count >= self.MIN_PHY_COUNT: alert_type = 'fault_resolved' # Send alert self._generate_alert(alert_type) # Save data to Consul self.sas_phy_stored_alert = (alert_type, new_phy_link_count) store.put(self.sas_phy_stored_alert, self.SAS_PORT_SENSOR_DATA) elif self.sas_phy_stored_alert[0] in ['fault_resolved', 'insertion']: # Check if we need to send informational alert if new_phy_link_count > self.sas_phy_stored_alert[ 1] and new_phy_link_count % self.MIN_PHY_COUNT == 0: alert_type = 'insertion' # Send alert self._generate_alert(alert_type) # Save data to Consul self.sas_phy_stored_alert = (alert_type, new_phy_link_count) store.put(self.sas_phy_stored_alert, self.SAS_PORT_SENSOR_DATA) # Check to see if we need to send fault alert if new_phy_link_count == 0: alert_type = 'fault' # Send alert self._generate_alert(alert_type) # Save data to Consul self.sas_phy_stored_alert = (alert_type, new_phy_link_count) store.put(self.sas_phy_stored_alert, self.SAS_PORT_SENSOR_DATA) def run(self): """Run the sensor on its own thread""" alert_type = None status = None new_phy_up = 0 new_phy_down = 0 # Do not proceed if module is suspended if self._suspended == True: self._scheduler.enter(self.polling_interval, self._priority, self.run, ()) return # Check for debug mode being activated self._read_my_msgQ_noWait() try: phy_link_rate_dict = \ self._utility_instance.get_phy_negotiated_link_rate() if phy_link_rate_dict: for key, value in phy_link_rate_dict.items(): link_rate = value.strip() prev_linkrate_value = \ self.phy_dir_to_linkrate_mapping[key][0].strip() prev_alert_type = \ self.phy_dir_to_linkrate_mapping[key][1].strip() status = prev_alert_type # Compare local dict wrt global dictionary for change in the # negitiated link rate if link_rate.lower() != prev_linkrate_value.lower(): # If current link rate has no value like 12/6/3 Gbit # and previously it was up, then it's a fault condition if 'Gbit'.lower() not in link_rate.lower( ) and prev_alert_type.lower() == 'up': # Increment count for new phy down which were up previously new_phy_down += 1 # Make respective phy_status as fault status = 'fault' # Check if 12/6/3 Gbit is there in the current link rate and # the previous alert_type is fault. If so, means phy is Up again elif 'Gbit'.lower() in link_rate.lower( ) and prev_alert_type.lower() == 'fault': # Mark respective phy_status as Up status = 'Up' # Increment count for new phy up new_phy_up += 1 # Finally update the global dict with current link rate # and respctive phy status self.phy_dir_to_linkrate_mapping[key] = (link_rate, status) # Get current phy status i.e number of Up phys new_phy_link_count = self.phy_link_count + new_phy_up - new_phy_down # Get the last sent alert info # It is a tuple of (alert_type, phy_link_count) self.sas_phy_stored_alert = store.get( self.SAS_PORT_SENSOR_DATA) self.check_and_send_alert(new_phy_link_count) # Update current active phy count for next iteration self.phy_link_count = new_phy_link_count except Exception as ae: logger.exception(ae) # Fire every 30 seconds to see if there's a change in the phy status self._scheduler.enter(self.polling_interval, self._priority, self.run, ()) def _create_json_message(self, alert_type): """Creates a defined json message structure which can flow inside SSPL modules""" internal_json_msg = None severity_reader = SeverityReader() severity = severity_reader.map_severity(alert_type) epoch_time = str(int(time.time())) alert_id = self._get_alert_id(epoch_time) host_name = socket.gethostname() specific_info = {} specific_info_list = [] for key, val in self.phy_dir_to_linkrate_mapping.items(): # key will be phy-0:0. So, aplit it using ':' # So, structure will be SASHBA-0:phy-0 phy_number = key.split(":")[1] specific_info[ "resource_id"] = self.RESOURCE_ID + ':' + "phy-" + phy_number specific_info[ "negotiated_link_rate"] = self.phy_dir_to_linkrate_mapping[ key][0].strip() specific_info_list.append(specific_info) specific_info = {} alert_specific_info = specific_info_list info = { "site_id": self._site_id, "cluster_id": self._cluster_id, "rack_id": self._rack_id, "node_id": self._node_id, "resource_type": self.RESOURCE_TYPE, "resource_id": self.RESOURCE_ID, "event_time": epoch_time } internal_json_msg = json.dumps({ "sensor_request_type": { "node_data": { "status": "update", "host_id": host_name, "alert_type": alert_type, "severity": severity, "alert_id": alert_id, "info": info, "specific_info": alert_specific_info } } }) return internal_json_msg def _get_alert_id(self, epoch_time): """Returns alert id which is a combination of epoch_time and salt value """ salt = str(uuid.uuid4().hex) alert_id = epoch_time + salt return alert_id def _generate_alert(self, alert_type): """Queues the message to NodeData Message Handler""" json_msg = self._create_json_message(alert_type) if json_msg: self._write_internal_msgQ(NodeDataMsgHandler.name(), json_msg) def suspend(self): """Suspends the module thread. It should be non-blocking""" super(SASPortSensor, self).suspend() self._suspended = True def resume(self): """Resumes the module thread. It should be non-blocking""" super(SASPortSensor, self).resume() self._suspended = False def shutdown(self): """Clean up scheduler queue and gracefully shutdown thread""" super(SASPortSensor, self).shutdown()