def read_data(self):
"""Send a dict of drive status to the DiskMsgHandler"""
for drive_data in self._drive_data:
logger.info(f"HPIMonitor, read_data: {str(drive_data)}")
# Send it to the disk message handler to be processed and transmitted
self._write_internal_msgQ(DiskMsgHandler.name(), drive_data)
def _restart_module(self, module_name):
"""Restart a module"""
self._log_debug("_restart_module, module_name: %s" % module_name)
try:
# Stop the module if it's running and let existing thread die gracefully
if self._status_module(module_name) is True:
self._stop_module(module_name)
# Allow module a few seconds to shut down gracefully
max_wait = 10
curr_wait = 1
while self._status_module(module_name) is True:
time.sleep(3)
logger.info("Retrying: %s" % str(curr_wait))
self._stop_module(module_name)
curr_wait += 1
if curr_wait > max_wait:
break
# Start the module
self._start_module(module_name)
except Exception as ae:
logger.warn("Restart thread failed: %s" % str(ae))
self._thread_response = "Restart Failed"
else:
self._thread_response = "Restart Successful"
def run(self):
"""Run the module periodically on its own thread."""
if (self._product.lower() in [x.lower() for x in enabled_products]) and \
not self._threads_initialized:
if self._product.lower() in [x.lower() for x in cs_products]:
# Wait for the dcs-collector to populate the /tmp/dcs/hpi directory
while not os.path.isdir(self._hpi_base_dir):
logger.info("ThreadController, dir not found: %s " %
self._hpi_base_dir)
logger.info("ThreadController, rechecking in %s secs" %
self._start_delay)
time.sleep(int(self._start_delay))
logger.debug("ThreadController._sspl_modules is {}".format(
self._sspl_modules))
# Allow other threads to initialize
continue_waiting = False
for (n, m) in self._sspl_modules.items():
if not isinstance(m, SensorThread):
continue
thread_init_status = m.get_thread_init_status()
logger.debug("Thread status for {} is {}".format(
m.__class__, thread_init_status))
if thread_init_status == SensorThreadState.FAILED:
m.shutdown()
elif thread_init_status == SensorThreadState.WAITING:
continue_waiting = True
if continue_waiting:
self._scheduler.enter(10, self._priority, self.run, ())
return
# Notify external applications that've started up successfully
startup_msg = "SSPL-LL service has started successfully"
json_msg = ThreadControllerMsg(ThreadController.name(),
startup_msg).getJson()
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
self._threads_initialized = True
#self._set_debug(True)
#self._set_debug_persist(True)
self._log_debug("Start accepting requests")
try:
# Block on message queue until it contains an entry
jsonMsg, _ = self._read_my_msgQ()
if jsonMsg is not None:
self._process_msg(jsonMsg)
# Keep processing until the message queue is empty
while not self._is_my_msgQ_empty():
jsonMsg, _ = self._read_my_msgQ()
if jsonMsg is not None:
self._process_msg(jsonMsg)
except Exception as ex:
# Log it and restart the whole process when a failure occurs
logger.exception("ThreadController restarting: %r" % ex)
self._scheduler.enter(1, self._priority, self.run, ())
self._log_debug("Finished processing successfully")
def init_rabbitMQ_msg_processors():
"""The main bootstrap for sspl automated tests"""
# Initialize logging
try:
init_logging("SSPL-Tests", "DEBUG")
except Exception as err:
# We don't have logger since it threw an exception, use generic 'print'
print("[ Error ] when initializing logging :")
print(err)
print("Exiting ...")
exit(os.EX_USAGE)
# Modules to be used for testing
conf_modules = Conf.get(SSPL_TEST_CONF, f"{SSPL_SETTING}>{MODULES}")
# Create a map of references to all the module's message queues. Each module
# is passed this mapping so that it can send messages to other modules.
msgQlist = {}
# Create a mapping of all the instantiated modules to their names
world.sspl_modules = {}
# Read in product value from configuration file
product = Conf.get(GLOBAL_CONF, f"release>{PRODUCT_NAME}")
logger.info("sspl-ll Bootstrap: product name supported: %s" % product)
# Use reflection to instantiate the class based upon its class name in config file
for conf_thread in conf_modules:
klass = globals()[conf_thread]
# Create mappings of modules and their message queues
world.sspl_modules[klass.name()] = klass()
msgQlist[klass.name()] = queue.Queue()
# Convert to a dict
# TODO: Check use of this
world.diskmonitor_file = json.loads("{}")
try:
# Loop through the list of instanced modules and start them on threads
threads = []
for name, curr_module in list(world.sspl_modules.items()):
logger.info("SSPL-Tests Starting %s" % curr_module.name())
curr_module._set_debug(True)
thread = Thread(target=_run_thread_capture_errors,
args=(curr_module, msgQlist, conf_reader, product))
thread.start()
threads.append(thread)
# Allow threads to startup before running tests
time.sleep(2)
# Clear the message queue buffer out from msgs sent at startup
while not world.sspl_modules[
RabbitMQingressProcessorTests.name()]._is_my_msgQ_empty():
world.sspl_modules[
RabbitMQingressProcessorTests.name()]._read_my_msgQ()
except Exception as ex:
logger.exception(ex)
def run(self):
"""Run the sensor on its own thread"""
# Do not proceed if module is suspended
if self._suspended == True:
logger.info("Scheduling RAID vaidate again")
self._scheduler.enter(self._polling_interval, self._priority,
self.run, ())
return
# Check for debug mode being activated
self._read_my_msgQ_noWait()
try:
#cleanup
self._cleanup()
# Validate the raid data files and notify the node data msg handler
self._raid_health_monitor()
logger.info("Scheduling RAID vaidate again")
self._scheduler.enter(self._polling_interval, self._priority,
self.run, ())
except Exception as ae:
logger.exception(ae)
def _send_ifdata_json_msg(self,
sensor_type,
resource_id,
resource_type,
state,
severity,
event=""):
"""A resuable method for transmitting IFDataMsg to RMQ and IEM logging"""
ifDataMsg = IFdataMsg(self._node_sensor.host_id,
self._node_sensor.local_time,
self._node_sensor.if_data, resource_id,
resource_type, self.site_id, self.node_id,
self.cluster_id, self.rack_id, state, severity,
event)
# Add in uuid if it was present in the json request
if self._uuid is not None:
ifDataMsg.set_uuid(self._uuid)
jsonMsg = ifDataMsg.getJson()
self.if_sensor_data = jsonMsg
self.os_sensor_type[sensor_type] = self.if_sensor_data
# Send the event to logging msg handler to send IEM message to journald
#internal_json_msg=json.dumps({
# 'actuator_request_type': {
# 'logging': {
# 'log_level': 'LOG_WARNING',
# 'log_type': 'IEM',
# 'log_msg': '{}'.format(jsonMsg)}}})
#self._write_internal_msgQ(LoggingMsgHandler.name(), internal_json_msg)
logger.info(f"RAAL: {jsonMsg}")
# Transmit it out over rabbitMQ channel
self._write_internal_msgQ(RabbitMQegressProcessor.name(), jsonMsg)
def load_storage_fru_list(self):
"""Get Storage FRU list and merge it with storage_fru_list,
maintained in global config, with which FRU list can be extended
for a solution.
Ex: Storage Enclosures not listing disk as FRU,
though its most common FRU in storage, and
practically it can be replaced easily.
So if for a solution, FRU list needs to be extended
beyond what publishes, 'storage_fru_list' from global config
can be used.
Some of the usual FRU examples are:- disk.
"""
self.fru_list = []
# Read FRU data using /show/frus cli api.
fru_data = singleton_realstorencl.get_realstor_encl_data("frus")
for fru_dict in fru_data:
if "name" in fru_dict.keys():
self.fru_list.append(fru_dict["name"])
self.fru_list = list(set(self.fru_list))
try:
self.hot_swapped_frus = Conf.get(
GLOBAL_CONF,
"storage_enclosure>storage_fru_list>hot_swappable",
['disk', 'controller'])
self.cold_swapped_frus = Conf.get(
GLOBAL_CONF, "storage_enclosure>storage_fru_list", [])
except ValueError as e:
logger.error("Failed to get storage_fru_list from config."
f"Error:{e}")
self.fru_list = list(
set(self.fru_list + self.hot_swapped_frus +
self.cold_swapped_frus))
logger.info(f"Fetched Enclosure FRU list:{self.fru_list}")
def _run_ipmitool_subcommand(self, subcommand, grep_args=None, out_file=subprocess.PIPE):
"""executes ipmitool sub-commands, and optionally greps the output"""
ipmi_tool = self.IPMITOOL
# A dummy file path check to select ipmi simulator if
# simulator is required, otherwise default ipmitool.
if os.path.exists("/tmp/activate_ipmisimtool"):
res, retcode = self._run_command(command=f"{self.IPMISIMTOOL} sel info")
if retcode == 0:
ipmi_tool = self.IPMISIMTOOL
logger.info("IPMI simulator is activated")
command = ipmi_tool + subcommand
if grep_args is not None:
command += " | grep "
if isinstance(grep_args, list):
grep_args_str = ""
for arg in grep_args:
grep_args_str = "'{}' ".format(arg)
command += grep_args_str
else:
command += "'{}'".format(grep_args)
res, retcode = self._run_command(command, out_file)
return res, retcode
def _generate_local_mount_data(self):
"""Create & transmit a local_mount_data message as defined
by the sensor response json schema"""
# Notify the node sensor to update its data required for the local_mount_data message
successful = self._node_sensor.read_data("local_mount_data",
self._get_debug(),
self._units)
if not successful:
logger.error(
"NodeDataMsgHandler, _generate_local_mount_data was NOT successful."
)
# Create the local mount data message and hand it over to the egress processor to transmit
localMountDataMsg = LocalMountDataMsg(
self._node_sensor.host_id, self._epoch_time,
self._node_sensor.free_space, self._node_sensor.free_inodes,
self._node_sensor.free_swap, self._node_sensor.total_space,
self._node_sensor.total_swap, self._units)
# Add in uuid if it was present in the json request
if self._uuid is not None:
localMountDataMsg.set_uuid(self._uuid)
jsonMsg = localMountDataMsg.getJson()
logger.info(f"RAAL: {jsonMsg}")
# Transmit it out over rabbitMQ channel
self._write_internal_msgQ(RabbitMQegressProcessor.name(), jsonMsg)
def _get_hostname(self):
try:
return self.os_utils.get_fqdn()
except Exception as e:
logger.exception("Got exception {} when trying to get hostname"
" using getfqdn().".format(e))
logger.info(" Trying with ip addr command")
try:
from subprocess import run, PIPE
from re import findall
IP_CMD = "ip -f inet addr show scope global up | grep inet"
IP_REGEX = b'\\b(\\d{1,3}(?:\\.\d{1,3}){3})/\d{1,2}\\b'
ip_out = run(IP_CMD, stdout=PIPE, shell=True, check=True)
ip_list = re.findall(IP_REGEX, ip_out.stdout)
if ip_list:
return ip_list[0]
except Exception as e:
logger.exception("Got exception {} when trying to get hostname"
" using ip addr command.".format(e))
# Ultimate fallback, when we are completely out of options
logger.info("Using localhost")
return "localhost"
def update_memcache_faults(self):
self.memcache_faults = self.latest_faults
#Update faults in persistent cache
logger.info("Updating faults persistent cache!!")
store.put(self.memcache_faults,
self.faults_persistent_cache)
def shutdown(self):
"""Clean up scheduler queue and gracefully shutdown thread"""
super(DriveManager, self).shutdown()
try:
self._blocking_notifier.stop()
except Exception:
logger.info("DriveManager, shutting down.")
def _raid_health_monitor(self):
try:
devices = self._get_devices()
logger.debug("Fetched devices:{}".format(devices))
for device in devices:
# Update the state as 'check' for RAID device file
result = self._update_raid_device_file(device)
if result == "failed":
self._retry_execution(self._update_raid_device_file, device)
logger.info("RAID device state is changed to 'check'")
# Check RAID device array state is 'idle' or not
result = self._check_raid_state(device)
if result == "failed":
self._retry_execution(self._check_raid_state, device)
logger.debug("'idle' state is found in Raid device:{}."
.format(device))
# Check Mismatch count in RAID device files.
result = self._check_mismatch_count(device)
if result == "failed":
self._retry_execution(self._check_mismatch_count, device)
logger.debug("No mismatch count is found in Raid device:{}"
.format(device))
if self._alert_resolved:
self.alert_type = self.FAULT_RESOLVED
self._alert_msg = "null"
self._send_json_msg(self.alert_type, device, self._alert_msg)
except Exception as ae:
logger.error("Failed in monitoring RAID health. ERROR:{}"
.format(str(ae)))
def run(self):
"""Run the module periodically on its own thread. """
self._log_debug("Start accepting requests")
try:
result = self._channel.queue_declare(exclusive=True)
self._channel.queue_bind(exchange=self._exchange_name,
queue=result.method.queue,
routing_key=self._routing_key)
self._channel.basic_consume(self._process_msg,
queue=result.method.queue)
self._channel.start_consuming()
except Exception:
if self.is_running() == True:
logger.info("RabbitMQingressProcessorTests ungracefully breaking out of run loop, restarting.")
# Configure RabbitMQ Exchange to receive messages
self._configure_exchange()
self._scheduler.enter(1, self._priority, self.run, ())
else:
logger.info("RabbitMQingressProcessorTests gracefully breaking out of run Loop, not restarting.")
self._log_debug("Finished processing successfully")
def get_health_info(self, rpath):
"""
Fetch health information for given rpath
rpath: Resource path to fetch its health
Examples:
node>storage[0]
node>storage[0]>fw
node>storage[0]>fw>logical_volumes
"""
logger.info(self.log.svc_log(f"Get Health data for rpath:{rpath}"))
info = {}
resource_found = False
nodes = rpath.strip().split(">")
leaf_node, _ = self.get_node_details(nodes[-1])
# Fetch health information for all sub nodes
if leaf_node == "storage":
resource_found = True
info = self.get_storage_health_info()
elif leaf_node in self.storage_resources:
resource_found = True
for resource, method in self.storage_resources[leaf_node].items():
try:
info.update({resource: method()})
resource_found = True
except Exception as err:
logger.error(
self.log.svc_log(f"{err.__class__.__name__}:{err}"))
info = None
else:
for node in nodes:
resource, _ = self.get_node_details(node)
for res_type in self.storage_resources:
method = self.storage_resources[res_type].get(resource)
if not method:
logger.error(
self.log.svc_log(
f"No mapping function found for {res_type}"))
continue
try:
resource_found = True
info = method()
break
except Exception as err:
logger.error(
self.log.svc_log(
f"{err.__class__.__name__}: {err}"))
info = None
if resource_found:
break
if not resource_found:
msg = f"Invalid rpath or health provider doesn't have support for'{rpath}'."
logger.error(self.log.svc_log(f"{msg}"))
raise ResourceMapError(errno.EINVAL, msg)
return info
def _read_config(self):
"""Configure the RabbitMQ exchange with defaults available"""
try:
self._virtual_host = self._conf_reader._get_value_with_default(
self.RABBITMQPROCESSOR, self.VIRT_HOST, 'SSPL')
# Read common RabbitMQ configuration
self._primary_rabbitmq_host = self._conf_reader._get_value_with_default(
self.RABBITMQPROCESSOR, self.PRIMARY_RABBITMQ_HOST,
'localhost')
# Read RabbitMQ configuration for sensor messages
self._queue_name = self._conf_reader._get_value_with_default(
self.RABBITMQPROCESSOR, self.QUEUE_NAME, 'sensor-queue')
self._exchange_name = self._conf_reader._get_value_with_default(
self.RABBITMQPROCESSOR, self.EXCHANGE_NAME, 'sspl-out')
self._routing_key = self._conf_reader._get_value_with_default(
self.RABBITMQPROCESSOR, self.ROUTING_KEY, 'sensor-key')
# Read RabbitMQ configuration for Ack messages
self._ack_queue_name = self._conf_reader._get_value_with_default(
self.RABBITMQPROCESSOR, self.ACK_QUEUE_NAME, 'sensor-queue')
self._ack_routing_key = self._conf_reader._get_value_with_default(
self.RABBITMQPROCESSOR, self.ACK_ROUTING_KEY, 'sensor-key')
self._username = self._conf_reader._get_value_with_default(
self.RABBITMQPROCESSOR, self.USER_NAME, 'sspluser')
self._password = self._conf_reader._get_value_with_default(
self.RABBITMQPROCESSOR, self.PASSWORD, '')
self._signature_user = self._conf_reader._get_value_with_default(
self.RABBITMQPROCESSOR, self.SIGNATURE_USERNAME, 'sspl-ll')
self._signature_token = self._conf_reader._get_value_with_default(
self.RABBITMQPROCESSOR, self.SIGNATURE_TOKEN, 'FAKETOKEN1234')
self._signature_expires = self._conf_reader._get_value_with_default(
self.RABBITMQPROCESSOR, self.SIGNATURE_EXPIRES, "3600")
self._iem_route_addr = self._conf_reader._get_value_with_default(
self.RABBITMQPROCESSOR, self.IEM_ROUTE_ADDR, '')
self._iem_route_exchange_name = self._conf_reader._get_value_with_default(
self.RABBITMQPROCESSOR, self.IEM_ROUTE_EXCHANGE_NAME,
'sspl-in')
cluster_id = self._conf_reader._get_value_with_default(
self.SYSTEM_INFORMATION_KEY,
COMMON_CONFIGS.get(self.SYSTEM_INFORMATION_KEY).get(
self.CLUSTER_ID_KEY), '')
# Decrypt RabbitMQ Password
decryption_key = encryptor.gen_key(cluster_id,
ServiceTypes.RABBITMQ.value)
self._password = encryptor.decrypt(decryption_key,
self._password.encode('ascii'),
"RabbitMQegressProcessor")
if self._iem_route_addr != "":
logger.info(" Routing IEMs to host: %s" %
self._iem_route_addr)
logger.info(" Using IEM exchange: %s" %
self._iem_route_exchange_name)
except Exception as ex:
logger.error("RabbitMQegressProcessor, _read_config: %r" % ex)
def _generate_alert(self, alert_type, port):
"""Queues the message to NodeData Message Handler"""
json_msg = self._create_json_message(alert_type, port)
if json_msg:
# RAAL stands for - RAise ALert
logger.info(f"RAAL: {json_msg}")
self._write_internal_msgQ(NodeDataMsgHandler.name(), json_msg)
def shutdown(self):
"""Clean up scheduler queue and gracefully shutdown thread"""
super(LoggingProcessor, self).shutdown()
try:
self._connection.cleanup()
except pika.exceptions.ConnectionClosed:
logger.info(
"LoggingProcessor, shutdown, RabbitMQ ConnectionClosed")
def _log_debug(self, message):
"""Logging messages"""
if self._debug:
log_msg = self.name() + ", " + message
if use_journal:
journal.send(log_msg, PRIORITY=7, SYSLOG_IDENTIFIER="sspl-ll")
else:
logger.info(log_msg)
def _raid_health_monitor(self):
try:
devices = self._get_devices()
if len(devices) == 0:
return
logger.debug("Fetched devices:{}".format(devices))
for device in devices:
# Update the state as 'check' for RAID device file
result = self._update_raid_device_file(device)
if result == "failed":
self._retry_execution(self._update_raid_device_file,
device)
logger.info("RAID device state is changed to 'check'")
# Check RAID device array state is 'idle' or not
result = self._check_raid_state(device)
if result == "failed":
logger.warning(
"'Idle' state not found for RAID device:{}".format(
device))
# Retry to check RAID state
self._retry_execution(self._check_raid_state, device)
logger.info(
"'idle' state is found in Raid device:{}.".format(device))
# Check Mismatch count in RAID device files.
result = self._check_mismatch_count(device)
if result == "failed":
# Persist RAID device fault state and send alert
fault_status_file = self.DEFAULT_RAID_DATA_PATH + device + "_" + RaidDataConfig.RAID_MISMATCH_FAULT_STATUS.value
if os.path.exists(fault_status_file):
with open(fault_status_file, 'r') as fs:
data = fs.read().rstrip()
if self.FAULT_RESOLVED in data:
self.alert_type = self.FAULT
self._alert_msg = RaidAlertMsgs.MISMATCH_MSG.value
self._send_json_msg(self.alert_type, device,
self._alert_msg)
self._update_fault_state_file(
device, self.FAULT, fault_status_file)
else:
self.alert_type = self.FAULT
self._alert_msg = RaidAlertMsgs.MISMATCH_MSG.value
self._send_json_msg(self.alert_type, device,
self._alert_msg)
self._update_fault_state_file(device, self.FAULT,
fault_status_file)
# Retry to check mismatch_cnt
self._retry_execution(self._check_mismatch_count, device)
logger.debug(
"No mismatch count is found in Raid device:{}".format(
device))
except Exception as ae:
logger.error("Failed in monitoring RAID health. ERROR:{}".format(
str(ae)))
def shutdown(self):
"""Clean up scheduler queue and gracefully shutdown thread"""
super(RabbitMQingressProcessor, self).shutdown()
try:
self._connection.cleanup()
except pika.exceptions.ConnectionClosed:
logger.info("RabbitMQingressProcessorTests, shutdown, RabbitMQ ConnectionClosed")
except Exception as err:
logger.info("RabbitMQingressProcessorTests, shutdown, RabbitMQ {}".format(str(err)))
def _send_json_msg(self, json_msg):
"""Sends JSON message to Handler"""
if not json_msg:
return
self._event.clear()
# RAAL stands for - RAise ALert
logger.info(f"RAAL: {json_msg}")
self._write_internal_msgQ(RealStorEnclMsgHandler.name(), json_msg,
self._event)
def shutdown(self):
"""Clean up scheduler queue and gracefully shutdown thread"""
super(RabbitMQingressProcessorTests, self).shutdown()
try:
if self._connection is not None:
self._connection.close()
self._channel.stop_consuming()
except pika.exceptions.ConnectionClosed:
logger.info("RabbitMQingressProcessorTests, shutdown, RabbitMQ ConnectionClosed")
def get_data(self, rpath):
"""Fetch Manifest information for given rpath."""
logger.info(self.log.svc_log(f"Get Manifest data for rpath:{rpath}"))
info = {}
resource_found = False
nodes = rpath.strip().split(">")
leaf_node, _ = StorageResourceMap.get_node_info(nodes[-1])
# Fetch health information for all sub nodes
if leaf_node == "storage":
# Example rpath: 'node>storage[0]'
resource_found = True
info = self.get_storage_manifest_info()
elif leaf_node in self.storage_resources:
# Example rpath: 'node>storage[0]>hw'
resource_found = True
for resource, method in self.storage_resources[leaf_node].items():
try:
info.update({resource: method()})
resource_found = True
except Exception as err:
logger.error(
self.log.svc_log(f"{err.__class__.__name__}:{err}"))
info = None
else:
# Example rpath: 'node>storage[0]>hw>disk'
for node in nodes:
resource, _ = StorageResourceMap.get_node_info(node)
for res_type in self.storage_resources:
method = self.storage_resources[res_type].get(resource)
if not method:
logger.error(
self.log.svc_log(
f"No mapping function found for {res_type}"))
continue
try:
resource_found = True
info = method()
break
except Exception as err:
logger.error(
self.log.svc_log(
f"{err.__class__.__name__}: {err}"))
info = None
if resource_found:
break
if not resource_found:
msg = f"Invalid rpath or manifest provider doesn't have support for'{rpath}'."
logger.error(self.log.svc_log(f"{msg}"))
raise ResourceMapError(errno.EINVAL, msg)
info = MonUtils.normalize_kv(info, HEALTH_UNDESIRED_VALS,
"Not Available")
return info
def _read_config(self):
"""Read in configuration values"""
try:
self._user = self._conf_reader._get_value_with_default(
self.IPMI, self.USER, 'admin')
self._pass = self._conf_reader._get_value_with_default(
self.IPMI, self.PASS, 'admin')
logger.info("IPMI Config: user: %s" % self._user)
except Exception as e:
logger.exception(e)
def _send_json_message(self, json_msg):
"""Transmit data to RealStorMsgHandler to be processed and sent out"""
self._event.clear()
# RAAL stands for - RAise ALert
logger.info(f"RAAL: {json_msg}")
# Send the event to real stor message handler
# to generate json message and send out
self._write_internal_msgQ(RealStorEnclMsgHandler.name(), json_msg,
self._event)
def _process_msg(self, body):
"""Parses the incoming message and hands off to the appropriate module"""
try:
# Encode and remove blankspace,\n,\t - Leaving as it might be useful
# ingressMsgTxt = json.dumps(body, ensure_ascii=True).encode('utf8')
# ingressMsg = json.loads(' '.join(ingressMsgTxt.split()))
# Enable debugging if it's found in the message
if "debug" in body.lower():
self._set_debug(True)
# Try encoding message to handle escape chars if present
try:
log_msg = body.encode('utf8')
except Exception as de:
logger.debug(
"_process_msg, no encoding applied, writing to syslog")
log_msg = body
# See if there is log level at the beginning
log_level = log_msg[0:log_msg.index(" ")]
if LOGLEVELS.get(log_level) is not None:
priority = LOGLEVELS[log_level]
log_msg = log_msg[log_msg.index(" "):]
else:
priority = LOG_INFO # Default to info log level
log_level = "LOG_INFO"
# See if there is an id available
event_code = None
try:
event_code_start = log_msg.index("IEC:") + 4
event_code_stop = log_msg.index(":", event_code_start)
# Parse out the event code and remove any blank spaces
event_code = log_msg[event_code_start:event_code_stop].strip()
self._log_debug("log_msg, event_code: %s" % event_code)
except Exception as e:
# Log message has no IEC to use as message_id in journal, ignoring
self._log_debug(
'Log message has no IEC to use as message_id in journal, ignoring: error: {}'
.format(e))
# Not an IEM so just dump it to the journal and don't worry about email and routing back to CMU
if event_code is None:
if use_journal:
journal.send(log_msg,
MESSAGE_ID=event_code,
PRIORITY=priority,
SYSLOG_IDENTIFIER="sspl-ll")
else:
logger.info(log_msg)
except Exception as ex:
logger.error("_process_msg: %r" % ex)
def generate_iem(module, event_code, severity, description):
"""Generate iem and send it to a MessgaeBroker."""
try:
logger.info(f"Sending IEM alert for module:{module}"
f" and event_code:{event_code}")
EventMessage.send(module=module,
event_id=event_code,
severity=severity,
message_blob=description)
except EventMessageError as e:
logger.error("Failed to send IEM alert." f"Error:{e}")
def _read_config(self):
"""Read in configuration values"""
try:
self._iem_routing_enabled = Conf.get(SSPL_CONF, f"{self.LOGGINGMSGHANDLER}>{self.IEM_ROUTING_ENABLED}",
'false')
self._iem_log_locally = Conf.get(SSPL_CONF, f"{self.LOGGINGMSGHANDLER}>{self.IEM_LOG_LOCALLY}",
'true')
logger.info(f"IEM routing enabled: {str(self._iem_routing_enabled)}")
logger.info(f"IEM log locally: {str(self._iem_log_locally)}")
except Exception as ex:
logger.exception(f"_read_config: {ex}")
def _debug_off_globally(self):
"""Turns debug mode off on all threads"""
jsonMsg = {'sspl_ll_debug': {'debug_component':'all', 'debug_enabled' : False}}
for _msgQ in self._msgQlist:
if _msgQ != "ThreadController":
logger.info("_debug_off_globally, notifying: %s" % _msgQ)
self._write_internal_msgQ(_msgQ, jsonMsg)
# Notify the ThreadController to bounce all threads so that blocking ones switch debug mode
jsonMsg = {'sspl_ll_debug': {'debug_component':'all'}}
self._write_internal_msgQ("ThreadController", jsonMsg)
