def _process_job_status(self, uuid, job_uuid):
"""Send the current job status requested"""
self._log_debug(
"_process_job_status, job_status requested on uuid: %s" % job_uuid)
try:
# Get a copy of the job tasks in the PlaneCntrlMsgHandler msg queue pending being worked
plane_cntrl_jobs = self._get_msgQ_copy("PlaneCntrlMsgHandler")
response = "Not Found"
# See if the requested uuid is in the list of pending jobs
for plane_cntrl_job in plane_cntrl_jobs:
self._log_debug("_process_job_status, plane_cntrl_job: %s" %
str(plane_cntrl_job))
if job_uuid in str(plane_cntrl_job):
response = "In Queue"
break
ack_type = {}
ack_type["hostname"] = gethostname()
ack_type["command"] = "job_status"
ack_type["arguments"] = str(job_uuid)
# The uuid is either not found or it's in the queue to be worked
ack_msg = AckResponseMsg(json.dumps(ack_type), response,
uuid).getJson()
self._write_internal_msgQ(PlaneCntrlRMQegressProcessor.name(),
ack_msg)
except Exception as ex:
logger.exception(
"PlaneCntrlRMQingressProcessor, _process_job_status exception: %s"
% str(ex))
def _run_thread_capture_errors(curr_module, sspl_modules, msgQlist,
conf_reader, product):
"""Run the given thread and log any errors that happen on it.
Will stop all sspl_modules if one of them fails."""
try:
# Each module is passed a reference list to message queues so it can transmit
# internal messages to other modules as desired
curr_module.start_thread(conf_reader, msgQlist, product)
except BaseException as ex:
logger.critical(
"SSPL-LL encountered a fatal error, terminating service Error: %s"
% ex)
logger.exception(ex)
# Populate an actuator response message and transmit back to HAlon
error_msg = "SSPL-LL encountered an error, terminating service Error: " + \
", Exception: " + logger.exception(ex)
json_msg = ThreadControllerMsg(curr_module.name(), error_msg).getJson()
if product.lower() in [x.lower() for x in enabled_products]:
self._write_internal_msgQ(RabbitMQegressProcessor.name(), json_msg)
elif product.lower() in [x.lower() for x in cs_legacy_products]:
self._write_internal_msgQ(PlaneCntrlRMQegressProcessor.name(),
json_msg)
# Shut it down, error is non-recoverable
for name, other_module in list(sspl_modules.items()):
if other_module is not curr_module:
other_module.shutdown()
def check_RabbitMQegressProcessor_is_running(self):
"""Used by the shutdown_handler to allow queued egress msgs to complete"""
if self._product.lower() in [x.lower() for x in enabled_products]:
return self._sspl_modules[
PlaneCntrlRMQegressProcessor.name()].is_running()
elif self._product.lower() in [x.lower() for x in cs_legacy_products]:
return self._sspl_modules[
RabbitMQegressProcessor.name()].is_running()
def _send_response(self, status, hostname, response, errors):
"""Transmit the response back as an Ack json msg"""
ack_type = {}
ack_type["hostname"], ack_type["command"], ack_type["parameters"], ack_type["status"], \
ack_type["errors"] = str(hostname, 'utf-8'), self._command, self._parameters, \
status, str(errors, 'utf-8')
ack_msg = AckResponseMsg(json.dumps(ack_type), \
str(response), self._uuid).getJson()
self._write_internal_msgQ(PlaneCntrlRMQegressProcessor.name(), ack_msg)
def _process_msg(self, jsonMsg):
"""Parses the incoming message and calls the appropriate method"""
self._log_debug("_process_msg, jsonMsg: %s" % jsonMsg)
# Check to see if debug mode is being globally turned off on all modules
if self._check_reset_all_modules(jsonMsg) is True:
return
# Parse out the module name and request
module_name = jsonMsg.get("actuator_request_type").get(
"thread_controller").get("module_name")
thread_request = jsonMsg.get("actuator_request_type").get(
"thread_controller").get("thread_request")
# Parse out the uuid so that it can be sent back in Ack message
uuid = None
if jsonMsg.get("sspl_ll_msg_header") is not None and \
jsonMsg.get("sspl_ll_msg_header").get("uuid") is not None:
uuid = jsonMsg.get("sspl_ll_msg_header").get("uuid")
self._log_debug("_processMsg, uuid: %s" % uuid)
# Pass along the debug section to the module
if jsonMsg.get("sspl_ll_debug") is not None:
self.debug_section = {"sspl_ll_debug": {}}
self.debug_section["sspl_ll_debug"] = jsonMsg.get("sspl_ll_debug")
else:
self.debug_section = None
self._log_debug("_process_msg, self.debug_section: %s" %
self.debug_section)
# Parse out thread request and call the appropriate method
if thread_request == "restart":
self._restart_module(module_name)
elif thread_request == "start":
self._start_module(module_name)
elif thread_request == "stop":
# Don't let the outside world stop us from using RabbitMQ connection or shut down this thread
if module_name == "RabbitMQegressProcessor" or \
module_name == "RabbitMQingressProcessor" or \
module_name == "ThreadController":
logger.warn(
"Attempt to stop RabbitMQ or ThreadController Processors, \
ignoring. Please try 'restart' instead.")
return
self._stop_module(module_name)
elif thread_request == "status":
self._status_module(module_name)
elif thread_request == "degrade":
if module_name.lower() != "all":
logger.warn(
"Invalid module_name {0}. Need 'all' in module_name".
format(module_name))
return
self._switch_to_degraded_state(self._sspl_modules)
elif thread_request == "active":
if module_name.lower() != "all":
logger.warn(
"Invalid module_name {0}. Need 'all' in module_name".
format(module_name))
return
self._switch_to_active_state(self._sspl_modules)
else:
self._thread_response = "Error, unrecognized thread request"
node_id = []
if jsonMsg.get("actuator_request_type").get("thread_controller").get("parameters") is not None and \
jsonMsg.get("actuator_request_type").get("thread_controller").get("parameters").get("node_id"):
node_id = jsonMsg.get("actuator_request_type").get(
"thread_controller").get("parameters").get("node_id")
ack_type = {}
ack_type["hostname"] = self._hostname
ack_type["node_id"] = node_id
# Populate an actuator response message and transmit
threadControllerMsg = ThreadControllerMsg(module_name, self._thread_response, \
json.dumps(ack_type))
if uuid is not None:
threadControllerMsg.set_uuid(uuid)
msgString = threadControllerMsg.getJson()
logger.info("ThreadController, response: %s" % str(msgString))
if self._product.lower() in [x.lower() for x in enabled_products]:
self._write_internal_msgQ(RabbitMQegressProcessor.name(),
msgString)
elif self._product.lower() in [x.lower() for x in cs_legacy_products]:
self._write_internal_msgQ(PlaneCntrlRMQegressProcessor.name(),
msgString)
def _process_msg(self, ch, method, properties, body):
"""Parses the incoming message and hands off to the appropriate module"""
ingressMsg = {}
try:
if isinstance(body, dict) is False:
ingressMsg = json.loads(body)
else:
ingressMsg = body
# Authenticate message using username and signature fields
username = ingressMsg.get("username")
signature = ingressMsg.get("signature")
message = ingressMsg.get("message")
uuid = message.get("sspl_ll_msg_header").get("uuid")
msg_len = len(message) + 1
if uuid is None:
uuid = "N/A"
if use_security_lib and \
SSPL_SEC.sspl_verify_message(msg_len, str(message), username, signature) != 0:
logger.warn(
"PlaneCntrlRMQingressProcessor, Authentication failed on message: %s"
% ingressMsg)
return
# Get the incoming message type
if message.get("actuator_request_type") is not None:
msgType = message.get("actuator_request_type")
# Validate against the actuator schema
validate(ingressMsg, self._actuator_schema)
elif message.get("sensor_request_type") is not None:
msgType = message.get("sensor_request_type")
# Validate against the sensor schema
validate(ingressMsg, self._sensor_schema)
else:
# We only handle incoming actuator and sensor requests, ignore everything else
return
# Check for debugging being activated in the message header
self._check_debug(message)
self._log_debug("_process_msg, ingressMsg: %s" % ingressMsg)
# Hand off to appropriate actuator message handler
if msgType.get("plane_controller") is not None:
command = message.get("actuator_request_type").get(
"plane_controller").get("command")
# For a job status request forward over to PlaneCntrlRMQegressProcessor
if command is not None and \
command == "job_status":
job_uuid = message.get("actuator_request_type").get(
"plane_controller").get("arguments").get("uuid", None)
node_id = None
if message.get("actuator_request_type").get("plane_controller").get("parameters") is not None and \
message.get("actuator_request_type").get("plane_controller").get("parameters").get("node_id") is not None:
node_id = message.get("actuator_request_type").get(
"plane_controller").get("parameters").get(
"node_id")
# node_id set to None is a broadcast otherwise see if it applies to this node
if node_id is None or \
self._hostname in str(node_id):
self._process_job_status(uuid, job_uuid)
else:
self._write_internal_msgQ("PlaneCntrlMsgHandler", message)
# Handle restarting of internal threads
elif msgType.get("thread_controller") is not None:
node_id = None
if message.get("actuator_request_type").get("thread_controller").get("parameters") is not None and \
message.get("actuator_request_type").get("thread_controller").get("parameters").get("node_id") is not None:
node_id = message.get("actuator_request_type").get(
"thread_controller").get("parameters").get(
"node_id", None)
# node_id set to None is a broadcast otherwise see if it applies to this node
if node_id is None or \
self._hostname in str(node_id):
self._write_internal_msgQ("ThreadController", message)
# ... handle other incoming messages that have been validated
else:
# Log a msg about not being able to process the message
logger.info(
"PlaneCntrlRMQingressProcessor, _process_msg, unrecognized message: %s"
% str(ingressMsg))
except Exception as ex:
logger.exception(
"PlaneCntrlRMQingressProcessor, _process_msg unrecognized message: %r"
% ingressMsg)
ack_msg = AckResponseMsg("Error Processing Msg",
"Msg Handler Not Found", uuid).getJson()
self._write_internal_msgQ(PlaneCntrlRMQegressProcessor.name(),
ack_msg)
# Acknowledge message was received
self._connection.ack(ch, delivery_tag=method.delivery_tag)
def _process_msg(self, jsonMsg):
"""Parses the incoming message and process"""
if isinstance(jsonMsg, dict) is False:
jsonMsg = json.loads(jsonMsg)
# Parse json msg into usable fields
success = self._parse_jsonMsg(jsonMsg)
if not success:
response = "An error occurred parsing JSON fields"
self._send_response(response)
return
status = -1
response = "N/A"
errors = "N/A"
hostname = "N/A"
try:
self._sedOpDispatch = self._SedOpDispatch(self._command,
self._parameters,
self._arguments)
status = self._sedOpDispatch.status
# Don't continue on init errors, invalid command or doesn't apply to this node
if self._sedOpDispatch.status != 0:
if self._sedOpDispatch.status == 2:
self._log_debug(
"_process_msg, request is not for this node, ignoring."
)
else:
errors = self._sedOpDispatch.errors
self._log_debug(
f"_process_msg, status: {str(self._sedOpDispatch.status)}, errors: {str(errors)}"
)
return
# Let the egress processor know the current task being worked
self._write_internal_msgQ(PlaneCntrlRMQegressProcessor.name(),
jsonMsg)
hostname = self._sedOpDispatch.hostname
# Run the command with the parameters and arguments and retrive the response and any errors
status = self._sedOpDispatch.run()
response = self._sedOpDispatch.output
errors = self._sedOpDispatch.errors
self._log_debug(
f"PlaneCntrlMsgHandler, _process_msg, status: {str(status)}, \
command: {str(self._command)}, parameters: {str(self._parameters)}, \
args: {str(self._arguments)}")
self._log_debug(
f"PlaneCntrlMsgHandler, _process_msg, response: {str(response)}"
)
self._log_debug(
f"PlaneCntrlMsgHandler, _process_msg, errors: {str(errors)}")
except Exception as ae:
errors = str(ae)
logger.warn(
f"PlaneCntrlMsgHandler, _process_msg exception: {errors}")
response = "There was an error processing the request. Please refer to the logs for details."
# No need to enable self._sedOpDispatch.interrupt() in the shutdown()
self._sedOpDispatch = None
# Transmit the response back as an Ack json msg
self._send_response(status, hostname, response, errors)