class ServiceMsgHandler(ScheduledModuleThread, InternalMsgQ): """Message Handler for service request messages""" MODULE_NAME = "ServiceMsgHandler" PRIORITY = 2 RESOURCE_TYPE = "node:sw:os:service" ALERT_TYPE = "UPDATE" # Dependency list DEPENDENCIES = {"plugins": ["EgressProcessor"], "rpms": []} @staticmethod def dependencies(): """Returns a list of plugins and RPMs this module requires to function. """ return ServiceMsgHandler.DEPENDENCIES @staticmethod def name(): """ @return: name of the module.""" return ServiceMsgHandler.MODULE_NAME def __init__(self): super(ServiceMsgHandler, self).__init__(self.MODULE_NAME, self.PRIORITY) self._service_actuator = None self._query_utility = None self._dbus_service = DbusServiceHandler() # Flag to indicate suspension of module self._suspended = False def initialize(self, conf_reader, msgQlist, product): """initialize configuration reader and internal msg queues""" # Initialize ScheduledMonitorThread super(ServiceMsgHandler, self).initialize(conf_reader) # Initialize internal message queues for this module super(ServiceMsgHandler, self).initialize_msgQ(msgQlist) self._import_products(product) self.host_id = socket.getfqdn() self.site_id = Conf.get(GLOBAL_CONF, SITE_ID_KEY, "DC01") self.rack_id = Conf.get(GLOBAL_CONF, RACK_ID_KEY, "RC01") self.node_id = Conf.get(GLOBAL_CONF, NODE_ID_KEY, "SN01") self.cluster_id = Conf.get(GLOBAL_CONF, CLUSTER_ID_KEY, "CC01") self.storage_set_id = Conf.get(GLOBAL_CONF, STORAGE_SET_ID_KEY, "ST01") self.monitored_services = Conf.get( SSPL_CONF, f'{SERVICEMONITOR}>{MONITORED_SERVICES}') def _import_products(self, product): """Import classes based on which product is being used""" if product.lower() in [x.lower() for x in enabled_products]: from zope.component import queryUtility self._query_utility = queryUtility def run(self): """Run the module periodically on its own thread.""" logger.debug("Start accepting requests") # Do not proceed if module is suspended if self._suspended == True: self._scheduler.enter(1, self._priority, self.run, ()) return # self._set_debug(True) # self._set_debug_persist(True) try: # Block on message queue until it contains an entry json_msg, _ = self._read_my_msgQ() if json_msg is not None: self._process_msg(json_msg) # Keep processing until the message queue is empty while not self._is_my_msgQ_empty(): json_msg, _ = self._read_my_msgQ() if json_msg is not None: self._process_msg(json_msg) except Exception as ae: # Log it and restart the whole process when a failure occurs logger.exception(f"ServiceMsgHandler restarting: {ae}") self._scheduler.enter(1, self._priority, self.run, ()) logger.debug("Finished processing successfully") def _process_msg(self, jsonMsg): """Parses the incoming message and hands off to the appropriate logger """ logger.debug(f"_process_msg, jsonMsg: {jsonMsg}") if isinstance(jsonMsg, dict) is False: jsonMsg = json.loads(jsonMsg) # Parse out the uuid so that it can be sent back in Ack message uuid = None if jsonMsg.get("sspl_ll_msg_header") is not None and \ jsonMsg.get("sspl_ll_msg_header").get("uuid") is not None: uuid = jsonMsg.get("sspl_ll_msg_header").get("uuid") logger.debug(f"_processMsg, uuid: {uuid}") # Handle service start, stop, restart, status requests if "actuator_request_type" in jsonMsg and \ "service_controller" in jsonMsg["actuator_request_type"]: logger.debug("_processMsg, msg_type: service_controller") service_name = jsonMsg.get("actuator_request_type") \ .get("service_controller").get("service_name") service_request = jsonMsg.get("actuator_request_type") \ .get("service_controller").get("service_request") request = f"{service_request}:{service_name}" if service_name not in self.monitored_services: logger.error(f"{service_name} - service not monitored") msg = ("Check if supplied service name is valid, %s is not " "monitored or managed." % service_name) self.send_error_response(service_request, service_name, msg, errno.EINVAL) return elif service_request not in ["disable", "enable"]: status = self._dbus_service.is_enabled(service_name) if status == "disabled": logger.error(f"{service_name} - service is disabled") msg = ("%s is disabled, enable request needed before " "current - %s request can be processed." % (service_name, service_request)) self.send_error_response(service_request, service_name, msg, errno.EPERM) return # If the state is INITIALIZED, We can assume that actuator is # ready to perform operation. if actuator_state_manager.is_initialized("Service"): logger.debug(f"_process_msg, service_actuator name: \ {self._service_actuator.name()}") self._execute_request(self._service_actuator, jsonMsg, uuid) # If the state is INITIALIZING, need to send message elif actuator_state_manager.is_initializing("Service"): # This state will not be reached. Kept here for consistency. logger.info("Service actuator is initializing") self.send_error_response(service_request, service_name, \ "BUSY - Service actuator is initializing.", errno.EBUSY) elif actuator_state_manager.is_imported("Service"): # This case will be for first request only. Subsequent # requests will go to INITIALIZED state case. logger.info("Service actuator is imported and initializing") from actuators.IService import IService actuator_state_manager.set_state( "Service", actuator_state_manager.INITIALIZING) service_actuator_class = self._query_utility(IService) if service_actuator_class: # NOTE: Instantiation part should not time consuming # otherwise ServiceMsgHandler will get block and will # not be able serve any subsequent requests. This applies # to instantiation of evey actuator. self._service_actuator = service_actuator_class() logger.info(f"_process_msg, service_actuator name: \ {self._service_actuator.name()}") self._execute_request(self._service_actuator, jsonMsg, uuid) actuator_state_manager.set_state( "Service", actuator_state_manager.INITIALIZED) else: logger.info("Service actuator is not instantiated") # If there is no entry for actuator in table, We can assume # that it is not loaded for some reason. else: logger.warn("Service actuator is not loaded or not supported") # Handle events generated by the service monitor elif "sensor_request_type" in jsonMsg and \ "service_status_alert" in jsonMsg["sensor_request_type"]: logger.debug(f"Received alert from ServiceMonitor : {jsonMsg}") jsonMsg1 = ServiceMonitorMsg( jsonMsg["sensor_request_type"]).getJson() self._write_internal_msgQ("EgressProcessor", jsonMsg1) # ... handle other service message types def _execute_request(self, actuator_instance, json_msg, uuid): """Calls perform_request method of an actuator and sends response to output channel. """ service_name = json_msg.get("actuator_request_type").\ get("service_controller").get("service_name") service_request = json_msg.get("actuator_request_type").\ get("service_controller").get("service_request") service_info = {} service_name, result, error = actuator_instance.\ perform_request(json_msg) if error: self.send_error_response(service_request, service_name, result) return logger.debug(f"_processMsg, service_name: {service_name}, \ result: {result}") service_info["service_name"] = service_name service_info.update(result) # Create an actuator response and send it out response = self._create_actuator_response(service_info) service_controller_msg = ServiceControllerMsg(response) if uuid is not None: service_controller_msg.set_uuid(uuid) json_msg = service_controller_msg.getJson() self._write_internal_msgQ("EgressProcessor", json_msg) def send_error_response(self, request, service_name, err_msg, err_no=None): """Send error in response.""" error_info = {} error_response = True error_info["service_name"] = service_name error_info["request"] = request error_info["error_msg"] = err_msg if err_no is not None: str_err = errno_to_str_mapping(err_no) error_info["error_no"] = f"{err_no} - {str_err}" response = self._create_actuator_response(error_info, error_response) service_controller_msg = ServiceControllerMsg(response).getJson() self._write_internal_msgQ("EgressProcessor", service_controller_msg) def _create_actuator_response(self, service_info, is_error=False): """Create JSON msg.""" if is_error: severity = "warning" else: severity = "informational" resource_id = service_info.get("service_name") epoch_time = str(int(time.time())) specific_info = [] info = { "cluster_id": self.cluster_id, "site_id": self.site_id, "rack_id": self.rack_id, "storage_set_id": self.storage_set_id, "node_id": self.node_id, "resource_id": resource_id, "resource_type": self.RESOURCE_TYPE, "event_time": epoch_time } specific_info.append(service_info) response = { "alert_type": self.ALERT_TYPE, "severity": severity, "host_id": self.host_id, "instance_id": resource_id, "info": info, "specific_info": specific_info } return response def suspend(self): """Suspends the module thread. It should be non-blocking""" super(ServiceMsgHandler, self).suspend() self._suspended = True def resume(self): """Resumes the module thread. It should be non-blocking""" super(ServiceMsgHandler, self).resume() self._suspended = False def shutdown(self): """Clean up scheduler queue and gracefully shutdown thread""" super(ServiceMsgHandler, self).shutdown()
class SystemdService(Debug): """Handles service request messages to systemd""" ACTUATOR_NAME = "SystemdService" @staticmethod def name(): """ @return: name of the module.""" return SystemdService.ACTUATOR_NAME def __init__(self): super(SystemdService, self).__init__() # Use d-bus to communicate with systemd # Described at: http://www.freedesktop.org/wiki/Software/systemd/dbus/ # Obtain an instance of d-bus to communicate with systemd self._bus = SystemBus() # Obtain a manager interface to d-bus for communications with systemd systemd = self._bus.get_object('org.freedesktop.systemd1', '/org/freedesktop/systemd1') self._manager = Interface( systemd, dbus_interface='org.freedesktop.systemd1.Manager') # Subscribe to signal changes self._manager.Subscribe() # create service cls obj. self._service = DbusServiceHandler() def perform_request(self, jsonMsg): """Performs the service request""" self._check_debug(jsonMsg) # Parse out the service name and request to perform on it if jsonMsg.get("actuator_request_type").get("service_controller") \ is not None: self._service_name = jsonMsg.get("actuator_request_type").\ get("service_controller").get("service_name") self._service_request = jsonMsg.get("actuator_request_type").\ get("service_controller").get("service_request") else: self._service_name = jsonMsg.get("actuator_request_type").\ get("service_watchdog_controller").get("service_name") self._service_request = jsonMsg.get("actuator_request_type").\ get("service_watchdog_controller").get("service_request") logger.debug("perform_request, service_name: %s, service_request: %s" % \ (self._service_name, self._service_request)) try: # Load the systemd unit for the service systemd_unit = self._manager.LoadUnit(self._service_name) # Get a proxy to systemd for accessing properties of units self._proxy = self._bus.get_object("org.freedesktop.systemd1", \ str(systemd_unit)) # The returned result of the desired action result = {} is_err_response = False if self._service_request in ['restart', 'start']: # Before restart/start the service, check service state. # If it is not active or activating then only process # restart/start request. service_state = self._service.get_state(self._service_name) state = service_state.state if state not in ['active', 'activating']: if self._service_request == "restart": self._service.restart(self._service_name) elif self._service_request == "start": self._service.start(self._service_name) # Ensure we get an "active" state and not "activating" service_state = self._service.get_state(self._service_name) state = service_state.state max_wait = 0 while state != "active": logger.debug( "%s status is activating, needs 'active' " "state after %s request has been processed, retrying" % (self._service_name, self._service_request)) time.sleep(1) max_wait += 1 if max_wait > 20: logger.debug("maximum wait - %s seconds, for " "service restart reached." % max_wait) break service_state = self._service.get_state( self._service_name) state = service_state.state else: is_err_response = True err_msg = ( "Can not process %s request, for %s, as service " "is already in %s state." % (self._service_request, self._service_name, state)) logger.error(err_msg) return (self._service_name, err_msg, is_err_response) elif self._service_request == "stop": self._service.stop(self._service_name) elif self._service_request == "status": # Return the status below service_status = self._service.get_state(self._service_name) # TODO: Use cortx.utils Service class methods for # enable/disable services. elif self._service_request == "enable": service_list = [] service_list.append(self._service_name) # EnableUnitFiles() function takes second argument as boolean. # 'True' will enable a service for runtime only(creates symlink # in /run/.. directory) 'False' will enable a service # persistently (creates symlink in /etc/.. directory) _, dbus_result = self._manager.EnableUnitFiles( service_list, False, True) res = parse_enable_disable_dbus_result(dbus_result) result.update(res) logger.debug("perform_request, result for enable request: " "result: %s" % (result)) elif self._service_request == "disable": service_list = [] service_list.append(self._service_name) # DisableUnitFiles() function takes second argument as boolean. # 'True' will disable a service for runtime only(removes symlink # from /run/.. directory) 'False' will disable a service # persistently(removes symlink from /etc/.. directory) dbus_result = self._manager.DisableUnitFiles( service_list, False) res = parse_enable_disable_dbus_result(dbus_result) result.update(res) logger.debug( "perform_request, result for disable request: %s" % result) else: logger.error("perform_request, Unknown service request - %s " "for service - %s" % (self._service_request, self._service_name)) is_err_response = True return (self._service_name, "Unknown service request", is_err_response) except debus_exceptions.DBusException as error: is_err_response = True logger.exception("DBus Exception: %r" % error) return (self._service_name, str(error), is_err_response) except Exception as ae: logger.exception("SystemD Exception: %r" % ae) is_err_response = True return (self._service_name, str(ae), is_err_response) # Give the unit some time to finish starting/stopping to get final status time.sleep(5) # Get the current status of the process and return it back: service_status = self._service.get_state(self._service_name) pid = service_status.pid state = service_status.state substate = service_status.substate status = self._service.is_enabled(self._service_name) uptime = get_service_uptime(self._service_name) # Parse dbus output to fetch command line path with args. command_line = service_status.command_line_path command_line_path_with_args = [] for field in list(command_line[0][1]): command_line_path_with_args.append(str(field)) result["pid"] = pid result["state"] = state result["substate"] = substate result["status"] = status result["uptime"] = uptime result["command_line_path"] = command_line_path_with_args logger.debug("perform_request, state: %s, substate: %s" % (str(state), str(substate))) return (self._service_name, result, is_err_response)