def read_data(self): """Send a dict of drive status to the DiskMsgHandler""" for drive_data in self._drive_data: logger.info(f"HPIMonitor, read_data: {str(drive_data)}") # Send it to the disk message handler to be processed and transmitted self._write_internal_msgQ(DiskMsgHandler.name(), drive_data)
def _restart_module(self, module_name): """Restart a module""" self._log_debug("_restart_module, module_name: %s" % module_name) try: # Stop the module if it's running and let existing thread die gracefully if self._status_module(module_name) is True: self._stop_module(module_name) # Allow module a few seconds to shut down gracefully max_wait = 10 curr_wait = 1 while self._status_module(module_name) is True: time.sleep(3) logger.info("Retrying: %s" % str(curr_wait)) self._stop_module(module_name) curr_wait += 1 if curr_wait > max_wait: break # Start the module self._start_module(module_name) except Exception as ae: logger.warn("Restart thread failed: %s" % str(ae)) self._thread_response = "Restart Failed" else: self._thread_response = "Restart Successful"
def run(self): """Run the module periodically on its own thread.""" if (self._product.lower() in [x.lower() for x in enabled_products]) and \ not self._threads_initialized: if self._product.lower() in [x.lower() for x in cs_products]: # Wait for the dcs-collector to populate the /tmp/dcs/hpi directory while not os.path.isdir(self._hpi_base_dir): logger.info("ThreadController, dir not found: %s " % self._hpi_base_dir) logger.info("ThreadController, rechecking in %s secs" % self._start_delay) time.sleep(int(self._start_delay)) logger.debug("ThreadController._sspl_modules is {}".format( self._sspl_modules)) # Allow other threads to initialize continue_waiting = False for (n, m) in self._sspl_modules.items(): if not isinstance(m, SensorThread): continue thread_init_status = m.get_thread_init_status() logger.debug("Thread status for {} is {}".format( m.__class__, thread_init_status)) if thread_init_status == SensorThreadState.FAILED: m.shutdown() elif thread_init_status == SensorThreadState.WAITING: continue_waiting = True if continue_waiting: self._scheduler.enter(10, self._priority, self.run, ()) return # Notify external applications that've started up successfully startup_msg = "SSPL-LL service has started successfully" json_msg = ThreadControllerMsg(ThreadController.name(), startup_msg).getJson() self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg) self._threads_initialized = True #self._set_debug(True) #self._set_debug_persist(True) self._log_debug("Start accepting requests") try: # Block on message queue until it contains an entry jsonMsg, _ = self._read_my_msgQ() if jsonMsg is not None: self._process_msg(jsonMsg) # Keep processing until the message queue is empty while not self._is_my_msgQ_empty(): jsonMsg, _ = self._read_my_msgQ() if jsonMsg is not None: self._process_msg(jsonMsg) except Exception as ex: # Log it and restart the whole process when a failure occurs logger.exception("ThreadController restarting: %r" % ex) self._scheduler.enter(1, self._priority, self.run, ()) self._log_debug("Finished processing successfully")
def init_rabbitMQ_msg_processors(): """The main bootstrap for sspl automated tests""" # Initialize logging try: init_logging("SSPL-Tests", "DEBUG") except Exception as err: # We don't have logger since it threw an exception, use generic 'print' print("[ Error ] when initializing logging :") print(err) print("Exiting ...") exit(os.EX_USAGE) # Modules to be used for testing conf_modules = Conf.get(SSPL_TEST_CONF, f"{SSPL_SETTING}>{MODULES}") # Create a map of references to all the module's message queues. Each module # is passed this mapping so that it can send messages to other modules. msgQlist = {} # Create a mapping of all the instantiated modules to their names world.sspl_modules = {} # Read in product value from configuration file product = Conf.get(GLOBAL_CONF, f"release>{PRODUCT_NAME}") logger.info("sspl-ll Bootstrap: product name supported: %s" % product) # Use reflection to instantiate the class based upon its class name in config file for conf_thread in conf_modules: klass = globals()[conf_thread] # Create mappings of modules and their message queues world.sspl_modules[klass.name()] = klass() msgQlist[klass.name()] = queue.Queue() # Convert to a dict # TODO: Check use of this world.diskmonitor_file = json.loads("{}") try: # Loop through the list of instanced modules and start them on threads threads = [] for name, curr_module in list(world.sspl_modules.items()): logger.info("SSPL-Tests Starting %s" % curr_module.name()) curr_module._set_debug(True) thread = Thread(target=_run_thread_capture_errors, args=(curr_module, msgQlist, conf_reader, product)) thread.start() threads.append(thread) # Allow threads to startup before running tests time.sleep(2) # Clear the message queue buffer out from msgs sent at startup while not world.sspl_modules[ RabbitMQingressProcessorTests.name()]._is_my_msgQ_empty(): world.sspl_modules[ RabbitMQingressProcessorTests.name()]._read_my_msgQ() except Exception as ex: logger.exception(ex)
def run(self): """Run the sensor on its own thread""" # Do not proceed if module is suspended if self._suspended == True: logger.info("Scheduling RAID vaidate again") self._scheduler.enter(self._polling_interval, self._priority, self.run, ()) return # Check for debug mode being activated self._read_my_msgQ_noWait() try: #cleanup self._cleanup() # Validate the raid data files and notify the node data msg handler self._raid_health_monitor() logger.info("Scheduling RAID vaidate again") self._scheduler.enter(self._polling_interval, self._priority, self.run, ()) except Exception as ae: logger.exception(ae)
def _send_ifdata_json_msg(self, sensor_type, resource_id, resource_type, state, severity, event=""): """A resuable method for transmitting IFDataMsg to RMQ and IEM logging""" ifDataMsg = IFdataMsg(self._node_sensor.host_id, self._node_sensor.local_time, self._node_sensor.if_data, resource_id, resource_type, self.site_id, self.node_id, self.cluster_id, self.rack_id, state, severity, event) # Add in uuid if it was present in the json request if self._uuid is not None: ifDataMsg.set_uuid(self._uuid) jsonMsg = ifDataMsg.getJson() self.if_sensor_data = jsonMsg self.os_sensor_type[sensor_type] = self.if_sensor_data # Send the event to logging msg handler to send IEM message to journald #internal_json_msg=json.dumps({ # 'actuator_request_type': { # 'logging': { # 'log_level': 'LOG_WARNING', # 'log_type': 'IEM', # 'log_msg': '{}'.format(jsonMsg)}}}) #self._write_internal_msgQ(LoggingMsgHandler.name(), internal_json_msg) logger.info(f"RAAL: {jsonMsg}") # Transmit it out over rabbitMQ channel self._write_internal_msgQ(RabbitMQegressProcessor.name(), jsonMsg)
def load_storage_fru_list(self): """Get Storage FRU list and merge it with storage_fru_list, maintained in global config, with which FRU list can be extended for a solution. Ex: Storage Enclosures not listing disk as FRU, though its most common FRU in storage, and practically it can be replaced easily. So if for a solution, FRU list needs to be extended beyond what publishes, 'storage_fru_list' from global config can be used. Some of the usual FRU examples are:- disk. """ self.fru_list = [] # Read FRU data using /show/frus cli api. fru_data = singleton_realstorencl.get_realstor_encl_data("frus") for fru_dict in fru_data: if "name" in fru_dict.keys(): self.fru_list.append(fru_dict["name"]) self.fru_list = list(set(self.fru_list)) try: self.hot_swapped_frus = Conf.get( GLOBAL_CONF, "storage_enclosure>storage_fru_list>hot_swappable", ['disk', 'controller']) self.cold_swapped_frus = Conf.get( GLOBAL_CONF, "storage_enclosure>storage_fru_list", []) except ValueError as e: logger.error("Failed to get storage_fru_list from config." f"Error:{e}") self.fru_list = list( set(self.fru_list + self.hot_swapped_frus + self.cold_swapped_frus)) logger.info(f"Fetched Enclosure FRU list:{self.fru_list}")
def _run_ipmitool_subcommand(self, subcommand, grep_args=None, out_file=subprocess.PIPE): """executes ipmitool sub-commands, and optionally greps the output""" ipmi_tool = self.IPMITOOL # A dummy file path check to select ipmi simulator if # simulator is required, otherwise default ipmitool. if os.path.exists("/tmp/activate_ipmisimtool"): res, retcode = self._run_command(command=f"{self.IPMISIMTOOL} sel info") if retcode == 0: ipmi_tool = self.IPMISIMTOOL logger.info("IPMI simulator is activated") command = ipmi_tool + subcommand if grep_args is not None: command += " | grep " if isinstance(grep_args, list): grep_args_str = "" for arg in grep_args: grep_args_str = "'{}' ".format(arg) command += grep_args_str else: command += "'{}'".format(grep_args) res, retcode = self._run_command(command, out_file) return res, retcode
def _generate_local_mount_data(self): """Create & transmit a local_mount_data message as defined by the sensor response json schema""" # Notify the node sensor to update its data required for the local_mount_data message successful = self._node_sensor.read_data("local_mount_data", self._get_debug(), self._units) if not successful: logger.error( "NodeDataMsgHandler, _generate_local_mount_data was NOT successful." ) # Create the local mount data message and hand it over to the egress processor to transmit localMountDataMsg = LocalMountDataMsg( self._node_sensor.host_id, self._epoch_time, self._node_sensor.free_space, self._node_sensor.free_inodes, self._node_sensor.free_swap, self._node_sensor.total_space, self._node_sensor.total_swap, self._units) # Add in uuid if it was present in the json request if self._uuid is not None: localMountDataMsg.set_uuid(self._uuid) jsonMsg = localMountDataMsg.getJson() logger.info(f"RAAL: {jsonMsg}") # Transmit it out over rabbitMQ channel self._write_internal_msgQ(RabbitMQegressProcessor.name(), jsonMsg)
def _get_hostname(self): try: return self.os_utils.get_fqdn() except Exception as e: logger.exception("Got exception {} when trying to get hostname" " using getfqdn().".format(e)) logger.info(" Trying with ip addr command") try: from subprocess import run, PIPE from re import findall IP_CMD = "ip -f inet addr show scope global up | grep inet" IP_REGEX = b'\\b(\\d{1,3}(?:\\.\d{1,3}){3})/\d{1,2}\\b' ip_out = run(IP_CMD, stdout=PIPE, shell=True, check=True) ip_list = re.findall(IP_REGEX, ip_out.stdout) if ip_list: return ip_list[0] except Exception as e: logger.exception("Got exception {} when trying to get hostname" " using ip addr command.".format(e)) # Ultimate fallback, when we are completely out of options logger.info("Using localhost") return "localhost"
def update_memcache_faults(self): self.memcache_faults = self.latest_faults #Update faults in persistent cache logger.info("Updating faults persistent cache!!") store.put(self.memcache_faults, self.faults_persistent_cache)
def shutdown(self): """Clean up scheduler queue and gracefully shutdown thread""" super(DriveManager, self).shutdown() try: self._blocking_notifier.stop() except Exception: logger.info("DriveManager, shutting down.")
def _raid_health_monitor(self): try: devices = self._get_devices() logger.debug("Fetched devices:{}".format(devices)) for device in devices: # Update the state as 'check' for RAID device file result = self._update_raid_device_file(device) if result == "failed": self._retry_execution(self._update_raid_device_file, device) logger.info("RAID device state is changed to 'check'") # Check RAID device array state is 'idle' or not result = self._check_raid_state(device) if result == "failed": self._retry_execution(self._check_raid_state, device) logger.debug("'idle' state is found in Raid device:{}." .format(device)) # Check Mismatch count in RAID device files. result = self._check_mismatch_count(device) if result == "failed": self._retry_execution(self._check_mismatch_count, device) logger.debug("No mismatch count is found in Raid device:{}" .format(device)) if self._alert_resolved: self.alert_type = self.FAULT_RESOLVED self._alert_msg = "null" self._send_json_msg(self.alert_type, device, self._alert_msg) except Exception as ae: logger.error("Failed in monitoring RAID health. ERROR:{}" .format(str(ae)))
def run(self): """Run the module periodically on its own thread. """ self._log_debug("Start accepting requests") try: result = self._channel.queue_declare(exclusive=True) self._channel.queue_bind(exchange=self._exchange_name, queue=result.method.queue, routing_key=self._routing_key) self._channel.basic_consume(self._process_msg, queue=result.method.queue) self._channel.start_consuming() except Exception: if self.is_running() == True: logger.info("RabbitMQingressProcessorTests ungracefully breaking out of run loop, restarting.") # Configure RabbitMQ Exchange to receive messages self._configure_exchange() self._scheduler.enter(1, self._priority, self.run, ()) else: logger.info("RabbitMQingressProcessorTests gracefully breaking out of run Loop, not restarting.") self._log_debug("Finished processing successfully")
def get_health_info(self, rpath): """ Fetch health information for given rpath rpath: Resource path to fetch its health Examples: node>storage[0] node>storage[0]>fw node>storage[0]>fw>logical_volumes """ logger.info(self.log.svc_log(f"Get Health data for rpath:{rpath}")) info = {} resource_found = False nodes = rpath.strip().split(">") leaf_node, _ = self.get_node_details(nodes[-1]) # Fetch health information for all sub nodes if leaf_node == "storage": resource_found = True info = self.get_storage_health_info() elif leaf_node in self.storage_resources: resource_found = True for resource, method in self.storage_resources[leaf_node].items(): try: info.update({resource: method()}) resource_found = True except Exception as err: logger.error( self.log.svc_log(f"{err.__class__.__name__}:{err}")) info = None else: for node in nodes: resource, _ = self.get_node_details(node) for res_type in self.storage_resources: method = self.storage_resources[res_type].get(resource) if not method: logger.error( self.log.svc_log( f"No mapping function found for {res_type}")) continue try: resource_found = True info = method() break except Exception as err: logger.error( self.log.svc_log( f"{err.__class__.__name__}: {err}")) info = None if resource_found: break if not resource_found: msg = f"Invalid rpath or health provider doesn't have support for'{rpath}'." logger.error(self.log.svc_log(f"{msg}")) raise ResourceMapError(errno.EINVAL, msg) return info
def _read_config(self): """Configure the RabbitMQ exchange with defaults available""" try: self._virtual_host = self._conf_reader._get_value_with_default( self.RABBITMQPROCESSOR, self.VIRT_HOST, 'SSPL') # Read common RabbitMQ configuration self._primary_rabbitmq_host = self._conf_reader._get_value_with_default( self.RABBITMQPROCESSOR, self.PRIMARY_RABBITMQ_HOST, 'localhost') # Read RabbitMQ configuration for sensor messages self._queue_name = self._conf_reader._get_value_with_default( self.RABBITMQPROCESSOR, self.QUEUE_NAME, 'sensor-queue') self._exchange_name = self._conf_reader._get_value_with_default( self.RABBITMQPROCESSOR, self.EXCHANGE_NAME, 'sspl-out') self._routing_key = self._conf_reader._get_value_with_default( self.RABBITMQPROCESSOR, self.ROUTING_KEY, 'sensor-key') # Read RabbitMQ configuration for Ack messages self._ack_queue_name = self._conf_reader._get_value_with_default( self.RABBITMQPROCESSOR, self.ACK_QUEUE_NAME, 'sensor-queue') self._ack_routing_key = self._conf_reader._get_value_with_default( self.RABBITMQPROCESSOR, self.ACK_ROUTING_KEY, 'sensor-key') self._username = self._conf_reader._get_value_with_default( self.RABBITMQPROCESSOR, self.USER_NAME, 'sspluser') self._password = self._conf_reader._get_value_with_default( self.RABBITMQPROCESSOR, self.PASSWORD, '') self._signature_user = self._conf_reader._get_value_with_default( self.RABBITMQPROCESSOR, self.SIGNATURE_USERNAME, 'sspl-ll') self._signature_token = self._conf_reader._get_value_with_default( self.RABBITMQPROCESSOR, self.SIGNATURE_TOKEN, 'FAKETOKEN1234') self._signature_expires = self._conf_reader._get_value_with_default( self.RABBITMQPROCESSOR, self.SIGNATURE_EXPIRES, "3600") self._iem_route_addr = self._conf_reader._get_value_with_default( self.RABBITMQPROCESSOR, self.IEM_ROUTE_ADDR, '') self._iem_route_exchange_name = self._conf_reader._get_value_with_default( self.RABBITMQPROCESSOR, self.IEM_ROUTE_EXCHANGE_NAME, 'sspl-in') cluster_id = self._conf_reader._get_value_with_default( self.SYSTEM_INFORMATION_KEY, COMMON_CONFIGS.get(self.SYSTEM_INFORMATION_KEY).get( self.CLUSTER_ID_KEY), '') # Decrypt RabbitMQ Password decryption_key = encryptor.gen_key(cluster_id, ServiceTypes.RABBITMQ.value) self._password = encryptor.decrypt(decryption_key, self._password.encode('ascii'), "RabbitMQegressProcessor") if self._iem_route_addr != "": logger.info(" Routing IEMs to host: %s" % self._iem_route_addr) logger.info(" Using IEM exchange: %s" % self._iem_route_exchange_name) except Exception as ex: logger.error("RabbitMQegressProcessor, _read_config: %r" % ex)
def _generate_alert(self, alert_type, port): """Queues the message to NodeData Message Handler""" json_msg = self._create_json_message(alert_type, port) if json_msg: # RAAL stands for - RAise ALert logger.info(f"RAAL: {json_msg}") self._write_internal_msgQ(NodeDataMsgHandler.name(), json_msg)
def shutdown(self): """Clean up scheduler queue and gracefully shutdown thread""" super(LoggingProcessor, self).shutdown() try: self._connection.cleanup() except pika.exceptions.ConnectionClosed: logger.info( "LoggingProcessor, shutdown, RabbitMQ ConnectionClosed")
def _log_debug(self, message): """Logging messages""" if self._debug: log_msg = self.name() + ", " + message if use_journal: journal.send(log_msg, PRIORITY=7, SYSLOG_IDENTIFIER="sspl-ll") else: logger.info(log_msg)
def _raid_health_monitor(self): try: devices = self._get_devices() if len(devices) == 0: return logger.debug("Fetched devices:{}".format(devices)) for device in devices: # Update the state as 'check' for RAID device file result = self._update_raid_device_file(device) if result == "failed": self._retry_execution(self._update_raid_device_file, device) logger.info("RAID device state is changed to 'check'") # Check RAID device array state is 'idle' or not result = self._check_raid_state(device) if result == "failed": logger.warning( "'Idle' state not found for RAID device:{}".format( device)) # Retry to check RAID state self._retry_execution(self._check_raid_state, device) logger.info( "'idle' state is found in Raid device:{}.".format(device)) # Check Mismatch count in RAID device files. result = self._check_mismatch_count(device) if result == "failed": # Persist RAID device fault state and send alert fault_status_file = self.DEFAULT_RAID_DATA_PATH + device + "_" + RaidDataConfig.RAID_MISMATCH_FAULT_STATUS.value if os.path.exists(fault_status_file): with open(fault_status_file, 'r') as fs: data = fs.read().rstrip() if self.FAULT_RESOLVED in data: self.alert_type = self.FAULT self._alert_msg = RaidAlertMsgs.MISMATCH_MSG.value self._send_json_msg(self.alert_type, device, self._alert_msg) self._update_fault_state_file( device, self.FAULT, fault_status_file) else: self.alert_type = self.FAULT self._alert_msg = RaidAlertMsgs.MISMATCH_MSG.value self._send_json_msg(self.alert_type, device, self._alert_msg) self._update_fault_state_file(device, self.FAULT, fault_status_file) # Retry to check mismatch_cnt self._retry_execution(self._check_mismatch_count, device) logger.debug( "No mismatch count is found in Raid device:{}".format( device)) except Exception as ae: logger.error("Failed in monitoring RAID health. ERROR:{}".format( str(ae)))
def shutdown(self): """Clean up scheduler queue and gracefully shutdown thread""" super(RabbitMQingressProcessor, self).shutdown() try: self._connection.cleanup() except pika.exceptions.ConnectionClosed: logger.info("RabbitMQingressProcessorTests, shutdown, RabbitMQ ConnectionClosed") except Exception as err: logger.info("RabbitMQingressProcessorTests, shutdown, RabbitMQ {}".format(str(err)))
def _send_json_msg(self, json_msg): """Sends JSON message to Handler""" if not json_msg: return self._event.clear() # RAAL stands for - RAise ALert logger.info(f"RAAL: {json_msg}") self._write_internal_msgQ(RealStorEnclMsgHandler.name(), json_msg, self._event)
def shutdown(self): """Clean up scheduler queue and gracefully shutdown thread""" super(RabbitMQingressProcessorTests, self).shutdown() try: if self._connection is not None: self._connection.close() self._channel.stop_consuming() except pika.exceptions.ConnectionClosed: logger.info("RabbitMQingressProcessorTests, shutdown, RabbitMQ ConnectionClosed")
def get_data(self, rpath): """Fetch Manifest information for given rpath.""" logger.info(self.log.svc_log(f"Get Manifest data for rpath:{rpath}")) info = {} resource_found = False nodes = rpath.strip().split(">") leaf_node, _ = StorageResourceMap.get_node_info(nodes[-1]) # Fetch health information for all sub nodes if leaf_node == "storage": # Example rpath: 'node>storage[0]' resource_found = True info = self.get_storage_manifest_info() elif leaf_node in self.storage_resources: # Example rpath: 'node>storage[0]>hw' resource_found = True for resource, method in self.storage_resources[leaf_node].items(): try: info.update({resource: method()}) resource_found = True except Exception as err: logger.error( self.log.svc_log(f"{err.__class__.__name__}:{err}")) info = None else: # Example rpath: 'node>storage[0]>hw>disk' for node in nodes: resource, _ = StorageResourceMap.get_node_info(node) for res_type in self.storage_resources: method = self.storage_resources[res_type].get(resource) if not method: logger.error( self.log.svc_log( f"No mapping function found for {res_type}")) continue try: resource_found = True info = method() break except Exception as err: logger.error( self.log.svc_log( f"{err.__class__.__name__}: {err}")) info = None if resource_found: break if not resource_found: msg = f"Invalid rpath or manifest provider doesn't have support for'{rpath}'." logger.error(self.log.svc_log(f"{msg}")) raise ResourceMapError(errno.EINVAL, msg) info = MonUtils.normalize_kv(info, HEALTH_UNDESIRED_VALS, "Not Available") return info
def _read_config(self): """Read in configuration values""" try: self._user = self._conf_reader._get_value_with_default( self.IPMI, self.USER, 'admin') self._pass = self._conf_reader._get_value_with_default( self.IPMI, self.PASS, 'admin') logger.info("IPMI Config: user: %s" % self._user) except Exception as e: logger.exception(e)
def _send_json_message(self, json_msg): """Transmit data to RealStorMsgHandler to be processed and sent out""" self._event.clear() # RAAL stands for - RAise ALert logger.info(f"RAAL: {json_msg}") # Send the event to real stor message handler # to generate json message and send out self._write_internal_msgQ(RealStorEnclMsgHandler.name(), json_msg, self._event)
def _process_msg(self, body): """Parses the incoming message and hands off to the appropriate module""" try: # Encode and remove blankspace,\n,\t - Leaving as it might be useful # ingressMsgTxt = json.dumps(body, ensure_ascii=True).encode('utf8') # ingressMsg = json.loads(' '.join(ingressMsgTxt.split())) # Enable debugging if it's found in the message if "debug" in body.lower(): self._set_debug(True) # Try encoding message to handle escape chars if present try: log_msg = body.encode('utf8') except Exception as de: logger.debug( "_process_msg, no encoding applied, writing to syslog") log_msg = body # See if there is log level at the beginning log_level = log_msg[0:log_msg.index(" ")] if LOGLEVELS.get(log_level) is not None: priority = LOGLEVELS[log_level] log_msg = log_msg[log_msg.index(" "):] else: priority = LOG_INFO # Default to info log level log_level = "LOG_INFO" # See if there is an id available event_code = None try: event_code_start = log_msg.index("IEC:") + 4 event_code_stop = log_msg.index(":", event_code_start) # Parse out the event code and remove any blank spaces event_code = log_msg[event_code_start:event_code_stop].strip() self._log_debug("log_msg, event_code: %s" % event_code) except Exception as e: # Log message has no IEC to use as message_id in journal, ignoring self._log_debug( 'Log message has no IEC to use as message_id in journal, ignoring: error: {}' .format(e)) # Not an IEM so just dump it to the journal and don't worry about email and routing back to CMU if event_code is None: if use_journal: journal.send(log_msg, MESSAGE_ID=event_code, PRIORITY=priority, SYSLOG_IDENTIFIER="sspl-ll") else: logger.info(log_msg) except Exception as ex: logger.error("_process_msg: %r" % ex)
def generate_iem(module, event_code, severity, description): """Generate iem and send it to a MessgaeBroker.""" try: logger.info(f"Sending IEM alert for module:{module}" f" and event_code:{event_code}") EventMessage.send(module=module, event_id=event_code, severity=severity, message_blob=description) except EventMessageError as e: logger.error("Failed to send IEM alert." f"Error:{e}")
def _read_config(self): """Read in configuration values""" try: self._iem_routing_enabled = Conf.get(SSPL_CONF, f"{self.LOGGINGMSGHANDLER}>{self.IEM_ROUTING_ENABLED}", 'false') self._iem_log_locally = Conf.get(SSPL_CONF, f"{self.LOGGINGMSGHANDLER}>{self.IEM_LOG_LOCALLY}", 'true') logger.info(f"IEM routing enabled: {str(self._iem_routing_enabled)}") logger.info(f"IEM log locally: {str(self._iem_log_locally)}") except Exception as ex: logger.exception(f"_read_config: {ex}")
def _debug_off_globally(self): """Turns debug mode off on all threads""" jsonMsg = {'sspl_ll_debug': {'debug_component':'all', 'debug_enabled' : False}} for _msgQ in self._msgQlist: if _msgQ != "ThreadController": logger.info("_debug_off_globally, notifying: %s" % _msgQ) self._write_internal_msgQ(_msgQ, jsonMsg) # Notify the ThreadController to bounce all threads so that blocking ones switch debug mode jsonMsg = {'sspl_ll_debug': {'debug_component':'all'}} self._write_internal_msgQ("ThreadController", jsonMsg)