class ServiceMsgHandler(ScheduledModuleThread, InternalMsgQ):
"""Message Handler for service request messages"""
MODULE_NAME = "ServiceMsgHandler"
PRIORITY = 2
RESOURCE_TYPE = "node:sw:os:service"
ALERT_TYPE = "UPDATE"
# Dependency list
DEPENDENCIES = {"plugins": ["EgressProcessor"], "rpms": []}
@staticmethod
def dependencies():
"""Returns a list of plugins and RPMs this module requires
to function.
"""
return ServiceMsgHandler.DEPENDENCIES
@staticmethod
def name():
""" @return: name of the module."""
return ServiceMsgHandler.MODULE_NAME
def __init__(self):
super(ServiceMsgHandler, self).__init__(self.MODULE_NAME,
self.PRIORITY)
self._service_actuator = None
self._query_utility = None
self._dbus_service = DbusServiceHandler()
# Flag to indicate suspension of module
self._suspended = False
def initialize(self, conf_reader, msgQlist, product):
"""initialize configuration reader and internal msg queues"""
# Initialize ScheduledMonitorThread
super(ServiceMsgHandler, self).initialize(conf_reader)
# Initialize internal message queues for this module
super(ServiceMsgHandler, self).initialize_msgQ(msgQlist)
self._import_products(product)
self.host_id = socket.getfqdn()
self.site_id = Conf.get(GLOBAL_CONF, SITE_ID_KEY, "DC01")
self.rack_id = Conf.get(GLOBAL_CONF, RACK_ID_KEY, "RC01")
self.node_id = Conf.get(GLOBAL_CONF, NODE_ID_KEY, "SN01")
self.cluster_id = Conf.get(GLOBAL_CONF, CLUSTER_ID_KEY, "CC01")
self.storage_set_id = Conf.get(GLOBAL_CONF, STORAGE_SET_ID_KEY, "ST01")
self.monitored_services = Conf.get(
SSPL_CONF, f'{SERVICEMONITOR}>{MONITORED_SERVICES}')
def _import_products(self, product):
"""Import classes based on which product is being used"""
if product.lower() in [x.lower() for x in enabled_products]:
from zope.component import queryUtility
self._query_utility = queryUtility
def run(self):
"""Run the module periodically on its own thread."""
logger.debug("Start accepting requests")
# Do not proceed if module is suspended
if self._suspended == True:
self._scheduler.enter(1, self._priority, self.run, ())
return
# self._set_debug(True)
# self._set_debug_persist(True)
try:
# Block on message queue until it contains an entry
json_msg, _ = self._read_my_msgQ()
if json_msg is not None:
self._process_msg(json_msg)
# Keep processing until the message queue is empty
while not self._is_my_msgQ_empty():
json_msg, _ = self._read_my_msgQ()
if json_msg is not None:
self._process_msg(json_msg)
except Exception as ae:
# Log it and restart the whole process when a failure occurs
logger.exception(f"ServiceMsgHandler restarting: {ae}")
self._scheduler.enter(1, self._priority, self.run, ())
logger.debug("Finished processing successfully")
def _process_msg(self, jsonMsg):
"""Parses the incoming message and hands off to the appropriate logger
"""
logger.debug(f"_process_msg, jsonMsg: {jsonMsg}")
if isinstance(jsonMsg, dict) is False:
jsonMsg = json.loads(jsonMsg)
# Parse out the uuid so that it can be sent back in Ack message
uuid = None
if jsonMsg.get("sspl_ll_msg_header") is not None and \
jsonMsg.get("sspl_ll_msg_header").get("uuid") is not None:
uuid = jsonMsg.get("sspl_ll_msg_header").get("uuid")
logger.debug(f"_processMsg, uuid: {uuid}")
# Handle service start, stop, restart, status requests
if "actuator_request_type" in jsonMsg and \
"service_controller" in jsonMsg["actuator_request_type"]:
logger.debug("_processMsg, msg_type: service_controller")
service_name = jsonMsg.get("actuator_request_type") \
.get("service_controller").get("service_name")
service_request = jsonMsg.get("actuator_request_type") \
.get("service_controller").get("service_request")
request = f"{service_request}:{service_name}"
if service_name not in self.monitored_services:
logger.error(f"{service_name} - service not monitored")
msg = ("Check if supplied service name is valid, %s is not "
"monitored or managed." % service_name)
self.send_error_response(service_request, service_name, msg,
errno.EINVAL)
return
elif service_request not in ["disable", "enable"]:
status = self._dbus_service.is_enabled(service_name)
if status == "disabled":
logger.error(f"{service_name} - service is disabled")
msg = ("%s is disabled, enable request needed before "
"current - %s request can be processed." %
(service_name, service_request))
self.send_error_response(service_request, service_name,
msg, errno.EPERM)
return
# If the state is INITIALIZED, We can assume that actuator is
# ready to perform operation.
if actuator_state_manager.is_initialized("Service"):
logger.debug(f"_process_msg, service_actuator name: \
{self._service_actuator.name()}")
self._execute_request(self._service_actuator, jsonMsg, uuid)
# If the state is INITIALIZING, need to send message
elif actuator_state_manager.is_initializing("Service"):
# This state will not be reached. Kept here for consistency.
logger.info("Service actuator is initializing")
self.send_error_response(service_request, service_name, \
"BUSY - Service actuator is initializing.", errno.EBUSY)
elif actuator_state_manager.is_imported("Service"):
# This case will be for first request only. Subsequent
# requests will go to INITIALIZED state case.
logger.info("Service actuator is imported and initializing")
from actuators.IService import IService
actuator_state_manager.set_state(
"Service", actuator_state_manager.INITIALIZING)
service_actuator_class = self._query_utility(IService)
if service_actuator_class:
# NOTE: Instantiation part should not time consuming
# otherwise ServiceMsgHandler will get block and will
# not be able serve any subsequent requests. This applies
# to instantiation of evey actuator.
self._service_actuator = service_actuator_class()
logger.info(f"_process_msg, service_actuator name: \
{self._service_actuator.name()}")
self._execute_request(self._service_actuator, jsonMsg,
uuid)
actuator_state_manager.set_state(
"Service", actuator_state_manager.INITIALIZED)
else:
logger.info("Service actuator is not instantiated")
# If there is no entry for actuator in table, We can assume
# that it is not loaded for some reason.
else:
logger.warn("Service actuator is not loaded or not supported")
# Handle events generated by the service monitor
elif "sensor_request_type" in jsonMsg and \
"service_status_alert" in jsonMsg["sensor_request_type"]:
logger.debug(f"Received alert from ServiceMonitor : {jsonMsg}")
jsonMsg1 = ServiceMonitorMsg(
jsonMsg["sensor_request_type"]).getJson()
self._write_internal_msgQ("EgressProcessor", jsonMsg1)
# ... handle other service message types
def _execute_request(self, actuator_instance, json_msg, uuid):
"""Calls perform_request method of an actuator and sends response to
output channel.
"""
service_name = json_msg.get("actuator_request_type").\
get("service_controller").get("service_name")
service_request = json_msg.get("actuator_request_type").\
get("service_controller").get("service_request")
service_info = {}
service_name, result, error = actuator_instance.\
perform_request(json_msg)
if error:
self.send_error_response(service_request, service_name, result)
return
logger.debug(f"_processMsg, service_name: {service_name}, \
result: {result}")
service_info["service_name"] = service_name
service_info.update(result)
# Create an actuator response and send it out
response = self._create_actuator_response(service_info)
service_controller_msg = ServiceControllerMsg(response)
if uuid is not None:
service_controller_msg.set_uuid(uuid)
json_msg = service_controller_msg.getJson()
self._write_internal_msgQ("EgressProcessor", json_msg)
def send_error_response(self, request, service_name, err_msg, err_no=None):
"""Send error in response."""
error_info = {}
error_response = True
error_info["service_name"] = service_name
error_info["request"] = request
error_info["error_msg"] = err_msg
if err_no is not None:
str_err = errno_to_str_mapping(err_no)
error_info["error_no"] = f"{err_no} - {str_err}"
response = self._create_actuator_response(error_info, error_response)
service_controller_msg = ServiceControllerMsg(response).getJson()
self._write_internal_msgQ("EgressProcessor", service_controller_msg)
def _create_actuator_response(self, service_info, is_error=False):
"""Create JSON msg."""
if is_error:
severity = "warning"
else:
severity = "informational"
resource_id = service_info.get("service_name")
epoch_time = str(int(time.time()))
specific_info = []
info = {
"cluster_id": self.cluster_id,
"site_id": self.site_id,
"rack_id": self.rack_id,
"storage_set_id": self.storage_set_id,
"node_id": self.node_id,
"resource_id": resource_id,
"resource_type": self.RESOURCE_TYPE,
"event_time": epoch_time
}
specific_info.append(service_info)
response = {
"alert_type": self.ALERT_TYPE,
"severity": severity,
"host_id": self.host_id,
"instance_id": resource_id,
"info": info,
"specific_info": specific_info
}
return response
def suspend(self):
"""Suspends the module thread. It should be non-blocking"""
super(ServiceMsgHandler, self).suspend()
self._suspended = True
def resume(self):
"""Resumes the module thread. It should be non-blocking"""
super(ServiceMsgHandler, self).resume()
self._suspended = False
def shutdown(self):
"""Clean up scheduler queue and gracefully shutdown thread"""
super(ServiceMsgHandler, self).shutdown()
class SystemdService(Debug):
"""Handles service request messages to systemd"""
ACTUATOR_NAME = "SystemdService"
@staticmethod
def name():
""" @return: name of the module."""
return SystemdService.ACTUATOR_NAME
def __init__(self):
super(SystemdService, self).__init__()
# Use d-bus to communicate with systemd
# Described at: http://www.freedesktop.org/wiki/Software/systemd/dbus/
# Obtain an instance of d-bus to communicate with systemd
self._bus = SystemBus()
# Obtain a manager interface to d-bus for communications with systemd
systemd = self._bus.get_object('org.freedesktop.systemd1',
'/org/freedesktop/systemd1')
self._manager = Interface(
systemd, dbus_interface='org.freedesktop.systemd1.Manager')
# Subscribe to signal changes
self._manager.Subscribe()
# create service cls obj.
self._service = DbusServiceHandler()
def perform_request(self, jsonMsg):
"""Performs the service request"""
self._check_debug(jsonMsg)
# Parse out the service name and request to perform on it
if jsonMsg.get("actuator_request_type").get("service_controller") \
is not None:
self._service_name = jsonMsg.get("actuator_request_type").\
get("service_controller").get("service_name")
self._service_request = jsonMsg.get("actuator_request_type").\
get("service_controller").get("service_request")
else:
self._service_name = jsonMsg.get("actuator_request_type").\
get("service_watchdog_controller").get("service_name")
self._service_request = jsonMsg.get("actuator_request_type").\
get("service_watchdog_controller").get("service_request")
logger.debug("perform_request, service_name: %s, service_request: %s" % \
(self._service_name, self._service_request))
try:
# Load the systemd unit for the service
systemd_unit = self._manager.LoadUnit(self._service_name)
# Get a proxy to systemd for accessing properties of units
self._proxy = self._bus.get_object("org.freedesktop.systemd1", \
str(systemd_unit))
# The returned result of the desired action
result = {}
is_err_response = False
if self._service_request in ['restart', 'start']:
# Before restart/start the service, check service state.
# If it is not active or activating then only process
# restart/start request.
service_state = self._service.get_state(self._service_name)
state = service_state.state
if state not in ['active', 'activating']:
if self._service_request == "restart":
self._service.restart(self._service_name)
elif self._service_request == "start":
self._service.start(self._service_name)
# Ensure we get an "active" state and not "activating"
service_state = self._service.get_state(self._service_name)
state = service_state.state
max_wait = 0
while state != "active":
logger.debug(
"%s status is activating, needs 'active' "
"state after %s request has been processed, retrying"
% (self._service_name, self._service_request))
time.sleep(1)
max_wait += 1
if max_wait > 20:
logger.debug("maximum wait - %s seconds, for "
"service restart reached." % max_wait)
break
service_state = self._service.get_state(
self._service_name)
state = service_state.state
else:
is_err_response = True
err_msg = (
"Can not process %s request, for %s, as service "
"is already in %s state." %
(self._service_request, self._service_name, state))
logger.error(err_msg)
return (self._service_name, err_msg, is_err_response)
elif self._service_request == "stop":
self._service.stop(self._service_name)
elif self._service_request == "status":
# Return the status below
service_status = self._service.get_state(self._service_name)
# TODO: Use cortx.utils Service class methods for
# enable/disable services.
elif self._service_request == "enable":
service_list = []
service_list.append(self._service_name)
# EnableUnitFiles() function takes second argument as boolean.
# 'True' will enable a service for runtime only(creates symlink
# in /run/.. directory) 'False' will enable a service
# persistently (creates symlink in /etc/.. directory)
_, dbus_result = self._manager.EnableUnitFiles(
service_list, False, True)
res = parse_enable_disable_dbus_result(dbus_result)
result.update(res)
logger.debug("perform_request, result for enable request: "
"result: %s" % (result))
elif self._service_request == "disable":
service_list = []
service_list.append(self._service_name)
# DisableUnitFiles() function takes second argument as boolean.
# 'True' will disable a service for runtime only(removes symlink
# from /run/.. directory) 'False' will disable a service
# persistently(removes symlink from /etc/.. directory)
dbus_result = self._manager.DisableUnitFiles(
service_list, False)
res = parse_enable_disable_dbus_result(dbus_result)
result.update(res)
logger.debug(
"perform_request, result for disable request: %s" % result)
else:
logger.error("perform_request, Unknown service request - %s "
"for service - %s" %
(self._service_request, self._service_name))
is_err_response = True
return (self._service_name, "Unknown service request",
is_err_response)
except debus_exceptions.DBusException as error:
is_err_response = True
logger.exception("DBus Exception: %r" % error)
return (self._service_name, str(error), is_err_response)
except Exception as ae:
logger.exception("SystemD Exception: %r" % ae)
is_err_response = True
return (self._service_name, str(ae), is_err_response)
# Give the unit some time to finish starting/stopping to get final status
time.sleep(5)
# Get the current status of the process and return it back:
service_status = self._service.get_state(self._service_name)
pid = service_status.pid
state = service_status.state
substate = service_status.substate
status = self._service.is_enabled(self._service_name)
uptime = get_service_uptime(self._service_name)
# Parse dbus output to fetch command line path with args.
command_line = service_status.command_line_path
command_line_path_with_args = []
for field in list(command_line[0][1]):
command_line_path_with_args.append(str(field))
result["pid"] = pid
result["state"] = state
result["substate"] = substate
result["status"] = status
result["uptime"] = uptime
result["command_line_path"] = command_line_path_with_args
logger.debug("perform_request, state: %s, substate: %s" %
(str(state), str(substate)))
return (self._service_name, result, is_err_response)
