def occur_in_response_to(clazz: str, action: str, payload: object,
result: Result, n_priors: int) -> int:
"""
Evaluates the result of an action and performs additional actions as necessary
Parameters
----------
clazz: str
The class name that implemented the method handling the action
action: str
The name of the method implemented by clazz that was called
payload: object
The argument that were passed to the method
result: Result
The result of the method call
n_priors: int
A value indicating the number of reactions that have occurred during the requested operation. I.e., this
value is set to 0 as the router begins calling methods of each manager implementing the current request
action, and it increases by one each time a Result from a manager operation triggers a reaction.
"""
global config
if result.is_error():
send_email(
config.get("REACTION_NOTIFY"), "metaroot operation failed",
"<table>" + "<tr><td>Class</td><td>" + clazz + "</td></tr>"
"<tr><td>Action</td><td>" + action + "</td></tr>" +
"<tr><td>Payload</td><td>" + str(payload) + "</td></tr>" +
"<tr><td>Result Status</td><td>" + str(result.status) +
"</td></tr>" + "<tr><td>Result Payload</td><td>" +
str(result.response) + "</td></tr>" + "</table>")
return 0
def call_method(self, obj: object, message: dict):
"""
Calls a method of an object
Parameters
----------
obj: object
An object to invoke a method of
message: dict
Parameters specifying the method to invoke and arguments to pass
Returns
----------
dict
key status is 0 for success, and >0 on error
key response is response from method call, or None is server is returning internal error
"""
# Validate that the message defines an 'action' attribute which maps to a method name
if 'action' not in message:
self._logger.error("The message does not define an 'action' -> %s",
message)
return self.get_error_response(450)
# Lookup the requested method in the handler object
try:
method = getattr(obj, message['action'])
except AttributeError:
self._logger.error(
"The method %s is not defined on the argument object %s",
message['action'],
type(obj).__name__)
return self.get_error_response(451)
# Validate arguments match the method signature
arguments = inspect.signature(method).parameters
args = []
for argument in arguments:
if argument not in message:
self._logger.error(
"Call to method %s.%s%s, no parameter %r in message",
type(obj).__name__, message['action'],
inspect.signature(method), argument)
return self.get_error_response(452)
args.append(message[argument])
# Call the method, returning its Result. This is wrapped by a try/except so that exception raise by method
# calls do not cause the server to stop
try:
return method(*args).to_transport_format()
except Exception as e:
self._logger.exception(e)
send_email(
self._config.get("NOTIFY_ON_ERROR"),
"Method call error: " + self._config.get_mq_handler_class(),
str(e))
return self.get_error_response(455)
def test_send_email_fail_address_unresolveable(self):
self.assertEqual(
False, send_email("foo", "test email", "<i>Test Content</i>"))
def send(self, obj: object) -> Result:
"""
Method to initiate an RPC request
Parameters
----------
obj: object
A dictionary specifying a remote method name and arguments to invoke
Returns
----------
Result
Result.status is 0 for success, >0 on error
Result.response is any object returned by the remote method invocation or None
"""
# Encode the request dict as YAML
try:
message = yaml.safe_dump(obj)
except yaml.YAMLError as exc:
self.logger.error("YAML serialization error: %s", exc)
self.logger.error("{0}".format(obj))
return Result(453, None)
self.response = None
self.corr_id = str(uuid.uuid4())
# Send RPC request to server
not_sent = True
attempts = 1
while not_sent and attempts < 10:
try:
self.channel.basic_publish(exchange='',
routing_key=self.queue,
body=message,
properties=pika.BasicProperties(
reply_to=self.callback_queue,
correlation_id=self.corr_id))
not_sent = False
except Exception as e:
self.logger.info("Failed to send on attempt %d because connection closed. Reconnecting...", attempts)
time.sleep((attempts-1)*5)
if self.connection.is_closed:
self.connect()
attempts = attempts + 1
if not_sent:
self.logger.error("Failed to deliver message %s:%s", self.queue, message.rstrip())
send_email(self.config.get("NOTIFY_ON_ERROR"),
"Message delivery failure: " + self.__class__.__name__,
"Failed to deliver message {0}:{1}".format(self.queue, message.rstrip()))
return Result(470, "Message could not be delivered")
# Wait for response
attempts = 1
while self.response is None and attempts < 36:
self.logger.debug("Waiting for callback response to %s", str(obj))
# Process events in
self.connection.process_data_events(time_limit=5)
attempts = attempts + 1
self.corr_id = None
# If timed out waiting for response
if attempts == 36:
self.logger.error("Operation timed out waiting for a response to %s:%s", self.queue, message.rstrip())
send_email(self.config.get("NOTIFY_ON_ERROR"),
"RPC timeout failure: " + self.__class__.__name__,
"No response received for message {0}:{1}".format(self.queue, message.rstrip()))
return Result(471, "Operation timed out waiting for a response")
# Decode the response dict as YAML
try:
res_obj = yaml.safe_load(self.response)
return Result.from_transport_format(res_obj)
except yaml.YAMLError as exc:
self.logger.error("YAML serialization error: %s", exc)
self.logger.error("{0}".format(obj))
return Result(454, None)
def start(self, config_key):
"""
Instantiates the hosted manager object and consumes messages that map to methods of the manager
Parameters
----------
config_key: str
Key identifying which node of the configuration tree to use
Returns
----------
int
Returns 1 on exit
"""
self._config = metaroot.config.get_config(config_key)
# Setup our custom logging to use the class name processing the messages as its tag
self._logger = metaroot.utils.get_logger(
self.__class__.__name__, self._config.get_log_file(),
self._config.get_file_verbosity(),
self._config.get_screen_verbosity())
# Output debug logging
self._logger.debug("VVVVVV RPCServer Config VVVVVV")
metaroot.config.debug_config(self._config)
# Instantiate an instance of the class specified in the config file that will process messages
self._handler = metaroot.utils.instantiate_object_from_class_path(
self._config.get_mq_handler_class())
self._handler.initialize()
# We want to exit gracefully if a SIGTERM is sent, so configure a handler
# signal.signal(signal.SIGTERM, self.shutdown)
# Consume messages, attempting to recover from network dropout
self._logger.info('starting consume loop for messages of type "%s"...',
self._config.get_mq_queue_name())
connect_attempts = 1
while connect_attempts < 30 and not self._exit_requested:
if not self.connect():
self._logger.info(
"Failed to connect on attempt %d. Will try again after sleeping %d seconds",
connect_attempts, connect_attempts * 5)
time.sleep(connect_attempts * 5)
connect_attempts = connect_attempts + 1
else:
self._logger.info(
"Connected to message host %s:%d after %d attempts",
self._config.get_mq_host(), self._config.get_mq_port(),
connect_attempts)
send_email(
self._config.get("NOTIFY_ON_ERROR"),
"RPC Server (re)connect for " +
self._config.get_mq_handler_class(),
"Connected to message host {0}:{1} after {2} attempts".
format(self._config.get_mq_host(),
self._config.get_mq_port(), connect_attempts))
connect_attempts = 1
try:
self.start_consuming()
except KeyboardInterrupt as e:
self._logger.warning("Interrupted by keyboard input.")
break
except (pika.exceptions.AMQPConnectionError,
pika.exceptions.ConnectionClosed,
pika.exceptions.ChannelClosed,
pika.exceptions.StreamLostError) as e:
self._logger.exception(e)
self._logger.error(
"Consume loop broken...will attempt to reconnect")
except Exception as e:
self._logger.exception(e)
self._logger.error(
"Consume loop broken...will NOT attempt to reconnect")
send_email(
self._config.get("NOTIFY_ON_ERROR"),
"RPC Server Exited: " +
self._config.get_mq_handler_class(), str(e))
break
return 1