def _get_response(self, ctx, proxy, topic, data):
"""Process a curried message and cast the result to topic."""
LOG.debug("Running func with context: %s", ctx.to_dict())
data.setdefault('version', None)
data.setdefault('args', {})
try:
result = proxy.dispatch(ctx, data['version'], data['method'],
data.get('namespace'), **data['args'])
return ConsumerBase.normalize_reply(result, ctx.replies)
except greenlet.GreenletExit:
# ignore these since they are just from shutdowns
pass
except rpc_common.ClientException as e:
LOG.debug("Expected exception during message handling (%s)",
e._exc_info[1])
return {
'exc':
rpc_common.serialize_remote_exception(e._exc_info,
log_failure=False)
}
except Exception:
LOG.error(_("Exception during message handling"))
return {
'exc': rpc_common.serialize_remote_exception(sys.exc_info())
}
def _send_reply(self,
conn,
reply=None,
failure=None,
ending=False,
log_failure=True):
if (self.reply_q and not self._obsolete_reply_queues.reply_q_valid(
self.reply_q, self.msg_id)):
return
if failure:
failure = rpc_common.serialize_remote_exception(
failure, log_failure)
msg = {'result': reply, 'failure': failure}
if ending:
msg['ending'] = True
rpc_amqp._add_unique_id(msg)
# If a reply_q exists, add the msg_id to the reply and pass the
# reply_q to direct_send() to use it as the response queue.
# Otherwise use the msg_id for backward compatibility.
if self.reply_q:
msg['_msg_id'] = self.msg_id
try:
conn.direct_send(self.reply_q, rpc_common.serialize_msg(msg))
except rpc_amqp.AMQPDestinationNotFound:
self._obsolete_reply_queues.add(self.reply_q, self.msg_id)
else:
# TODO(sileht): look at which version of oslo-incubator rpc
# send need this, but I guess this is older than icehouse
# if this is icehouse, we can drop this at M
# if this is havana, we can drop this now.
conn.direct_send(self.msg_id, rpc_common.serialize_msg(msg))
def _send_reply(self, conn, reply=None, failure=None, ending=False, log_failure=True):
if self.reply_q and not self._obsolete_reply_queues.reply_q_valid(self.reply_q, self.msg_id):
return
if failure:
failure = rpc_common.serialize_remote_exception(failure, log_failure)
msg = {"result": reply, "failure": failure}
if ending:
msg["ending"] = True
rpc_amqp._add_unique_id(msg)
unique_id = msg[rpc_amqp.UNIQUE_ID]
# If a reply_q exists, add the msg_id to the reply and pass the
# reply_q to direct_send() to use it as the response queue.
# Otherwise use the msg_id for backward compatibility.
if self.reply_q:
msg["_msg_id"] = self.msg_id
try:
if ending:
LOG.debug(
"sending reply msg_id: %(msg_id)s "
"reply queue: %(reply_q)s"
% {"msg_id": self.msg_id, "unique_id": unique_id, "reply_q": self.reply_q}
)
conn.direct_send(self.reply_q, rpc_common.serialize_msg(msg))
except rpc_amqp.AMQPDestinationNotFound:
self._obsolete_reply_queues.add(self.reply_q, self.msg_id)
else:
# TODO(sileht): look at which version of oslo-incubator rpc
# send need this, but I guess this is older than icehouse
# if this is icehouse, we can drop this at Mitaka
# if this is havana, we can drop this now.
conn.direct_send(self.msg_id, rpc_common.serialize_msg(msg))
def test_remote_exception(self):
# We want tracebacks
cfg.CONF.set_override('debug', True)
error = validator_exc.InvalidContentType(content_type='a')
exc_info = (type(error), error, None)
serialized = rpc_common.serialize_remote_exception(exc_info)
remote_error = rpc_common.deserialize_remote_exception(
serialized, ["validator.common.exception"])
wrapper = fault.FaultWrapper(None)
msg = wrapper._error(remote_error)
expected_message, expected_traceback = six.text_type(
remote_error).split('\n', 1)
error = {u'code': 500,
u'error': {u'message': u'Invalid content type %(content_type)s',
u'traceback': u'InvalidContentType: Invalid content type a\n',
u'type': 'InvalidContentType'},
u'explanation': 'The server has either erred or is incapable of performing the requested operation.',
u'title': 'Internal Server Error'}
expected = {
'success': False,
'response': error,
'install': {
'success': False,
'response': ''
},
'test': {
'success': False,
'response': ''
},
'deploy': {
'success': False,
'response': ''
}
}
self.assertEqual(expected, msg)
def test_should_not_ignore_parent_classes_even_for_remote_ones(self):
# We want tracebacks
cfg.CONF.set_override('debug', True)
error = ClusterNotFoundChild(cluster='a')
exc_info = (type(error), error, None)
serialized = rpc_common.serialize_remote_exception(exc_info)
remote_error = rpc_common.deserialize_remote_exception(
serialized, ["senlin.tests.middleware.test_fault_middleware"])
wrapper = fault.FaultWrapper(None)
msg = wrapper._error(remote_error)
expected_message, expected_traceback = six.text_type(remote_error).\
split('\n', 1)
expected = {
'code': 404,
'error': {
'code': 404,
'message': expected_message,
'traceback': expected_traceback,
'type': 'ClusterNotFoundChild'
},
'explanation': 'The resource could not be found.',
'title': 'Not Found'
}
self.assertEqual(expected, msg)
def _to_remote_error(self, error):
"""Converts the given exception to the one with the _Remote suffix."""
exc_info = (type(error), error, None)
serialized = rpc_common.serialize_remote_exception(exc_info)
remote_error = rpc_common.deserialize_remote_exception(
serialized, ["heat.common.exception"])
return remote_error
def _to_remote_error(self, error):
'''Converts the given exception to one with the _Remote suffix.'''
exc_info = (type(error), error, None)
serialized = rpc_common.serialize_remote_exception(exc_info)
remote_error = rpc_common.deserialize_remote_exception(
serialized, ["senlin.common.exception"])
return remote_error
def _send_reply(self, conn, reply=None, failure=None, ending=True):
if not self._obsolete_reply_queues.reply_q_valid(
self.reply_q, self.msg_id):
return
if failure:
failure = rpc_common.serialize_remote_exception(failure)
# NOTE(sileht): ending can be removed in N*, see Listener.wait()
# for more detail.
msg = {
'result': reply,
'failure': failure,
'ending': ending,
'_msg_id': self.msg_id
}
rpc_amqp._add_unique_id(msg)
unique_id = msg[rpc_amqp.UNIQUE_ID]
LOG.debug(
"sending reply msg_id: %(msg_id)s "
"reply queue: %(reply_q)s "
"time elapsed: %(elapsed)ss", {
'msg_id': self.msg_id,
'unique_id': unique_id,
'reply_q': self.reply_q,
'elapsed': self.stopwatch.elapsed()
})
conn.direct_send(self.reply_q, rpc_common.serialize_msg(msg))
def _send_reply(self, conn, reply=None, failure=None,
ending=False, log_failure=True):
if (self.reply_q and
not self._obsolete_reply_queues.reply_q_valid(self.reply_q,
self.msg_id)):
return
if failure:
failure = rpc_common.serialize_remote_exception(failure,
log_failure)
msg = {'result': reply, 'failure': failure}
if ending:
msg['ending'] = True
rpc_amqp._add_unique_id(msg)
# If a reply_q exists, add the msg_id to the reply and pass the
# reply_q to direct_send() to use it as the response queue.
# Otherwise use the msg_id for backward compatibility.
if self.reply_q:
msg['_msg_id'] = self.msg_id
try:
conn.direct_send(self.reply_q, rpc_common.serialize_msg(msg))
except rpc_amqp.AMQPDestinationNotFound:
self._obsolete_reply_queues.add(self.reply_q, self.msg_id)
else:
# TODO(sileht): look at which version of oslo-incubator rpc
# send need this, but I guess this is older than icehouse
# if this is icehouse, we can drop this at Mitaka
# if this is havana, we can drop this now.
conn.direct_send(self.msg_id, rpc_common.serialize_msg(msg))
def _to_remote_error(self, error):
'''Converts the given exception to one with the _Remote suffix.'''
exc_info = (type(error), error, None)
serialized = rpc_common.serialize_remote_exception(exc_info)
remote_error = rpc_common.deserialize_remote_exception(
serialized, ["senlin.common.exception"])
return remote_error
def test_serialize_remote_exception(self):
try:
try:
raise self.cls(*self.args, **self.kwargs)
except Exception as ex:
# Note: in Python 3 ex variable will be cleared at the end of
# the except clause, so explicitly make an extra copy of it
cls_error = ex
if self.add_remote:
ex = add_remote_postfix(ex)
raise ex
except Exception:
exc_info = sys.exc_info()
serialized = exceptions.serialize_remote_exception(exc_info)
failure = jsonutils.loads(serialized)
self.assertEqual(self.clsname, failure['class'], failure)
self.assertEqual(self.modname, failure['module'])
self.assertEqual(self.msg, failure['message'])
self.assertEqual([self.msg], failure['args'])
self.assertEqual(self.kwargs, failure['kwargs'])
# Note: _Remote prefix not stripped from tracebacks
tb = cls_error.__class__.__name__ + ': ' + self.msg
self.assertIn(tb, ''.join(failure['tb']))
def _to_remote_error(self, error):
"""Converts the given exception to the one with the _Remote suffix."""
exc_info = (type(error), error, None)
serialized = rpc_common.serialize_remote_exception(exc_info)
remote_error = rpc_common.deserialize_remote_exception(
serialized, ["heat.common.exception"])
return remote_error
def test_should_not_ignore_parent_classes_even_for_remote_ones(self):
# We want tracebacks
cfg.CONF.set_override('debug', True)
error = ClusterNotFoundChild(cluster='a')
exc_info = (type(error), error, None)
serialized = rpc_common.serialize_remote_exception(exc_info)
remote_error = rpc_common.deserialize_remote_exception(
serialized, ["senlin.tests.unit.middleware.test_fault_middleware"])
wrapper = fault.FaultWrapper(None)
msg = wrapper._error(remote_error)
expected_message, expected_traceback = six.text_type(remote_error).\
split('\n', 1)
expected = {
'code': 404,
'error': {
'code': 404,
'message': expected_message,
'traceback': expected_traceback,
'type': 'ClusterNotFoundChild'
},
'explanation': 'The resource could not be found.',
'title': 'Not Found'
}
self.assertEqual(expected, msg)
def test_serialize_remote_exception(self):
try:
try:
raise self.cls(*self.args, **self.kwargs)
except Exception as ex:
# Note: in Python 3 ex variable will be cleared at the end of
# the except clause, so explicitly make an extra copy of it
cls_error = ex
if self.add_remote:
ex = add_remote_postfix(ex)
raise ex
except Exception:
exc_info = sys.exc_info()
serialized = exceptions.serialize_remote_exception(exc_info)
failure = jsonutils.loads(serialized)
self.assertEqual(self.clsname, failure['class'], failure)
self.assertEqual(self.modname, failure['module'])
self.assertEqual(self.msg, failure['message'])
self.assertEqual([self.msg], failure['args'])
self.assertEqual(self.kwargs, failure['kwargs'])
# Note: _Remote prefix not stripped from tracebacks
tb = cls_error.__class__.__name__ + ': ' + self.msg
self.assertIn(tb, ''.join(failure['tb']))
def remote_exception_helper(self, name, error):
exc_info = (type(error), error, None)
serialized = rpc_common.serialize_remote_exception(exc_info)
remote_error = rpc_common.deserialize_remote_exception(
serialized, name)
wrapper = fault.FaultWrapper(None)
msg = wrapper._error(remote_error)
print msg
errmsg = {
'code': 500,
'error': {'message': msg['response']['error']['message'],
'traceback': None,
'type': 'RemoteError'},
'explanation': msg['response']['explanation'],
'title': 'Internal Server Error'}
expected = {
'success': False,
'response': errmsg,
'install': {
'success': False,
'response': ''
},
'test': {
'success': False,
'response': ''
},
'deploy': {
'success': False,
'response': ''
}
}
self.assertEqual(expected, msg)
def to_remote_error(error):
'''Prepend the given exception with the _Remote suffix.'''
exc_info = (type(error), error, None)
serialized = rpc_common.serialize_remote_exception(exc_info)
remote_error = rpc_common.deserialize_remote_exception(
serialized, ["senlin.common.exception"])
return remote_error
def to_remote_error(error):
'''Prepend the given exception with the _Remote suffix.'''
exc_info = (type(error), error, None)
serialized = rpc_common.serialize_remote_exception(exc_info)
remote_error = rpc_common.deserialize_remote_exception(
serialized, ["senlin.common.exception"])
return remote_error
def _reply(self, rpc_message, reply, failure):
if failure is not None:
failure = rpc_common.serialize_remote_exception(failure)
reply = zmq_response.Reply(message_id=rpc_message.message_id,
reply_id=rpc_message.reply_id,
reply_body=reply,
failure=failure)
self.reply_sender.send(rpc_message.socket, reply)
return reply
def marshal_response(reply=None, failure=None):
# TODO(grs): do replies have a context?
msg = proton.Message()
if failure:
failure = common.serialize_remote_exception(failure)
data = {"failure": failure}
else:
data = {"response": reply}
msg.body = jsonutils.dumps(data)
return msg
def _reply(self, rpc_message, reply, failure):
if failure is not None:
failure = rpc_common.serialize_remote_exception(failure)
reply = zmq_response.Reply(message_id=rpc_message.message_id,
reply_id=rpc_message.reply_id,
message_version=rpc_message.message_version,
reply_body=reply,
failure=failure)
self.reply_sender.send(rpc_message.socket, reply)
return reply
def reply(self, reply=None, failure=None):
if self.sender is not None:
if failure is not None:
failure = rpc_common.serialize_remote_exception(failure)
reply = zmq_response.Response(msg_type=zmq_names.REPLY_TYPE,
message_id=self.message_id,
reply_id=self.reply_id,
reply_body=reply,
failure=failure)
self.sender.send(self.socket, reply)
def marshal_response(reply=None, failure=None):
# TODO(grs): do replies have a context?
msg = proton.Message()
if failure:
failure = common.serialize_remote_exception(failure)
data = {"failure": failure}
else:
data = {"response": reply}
msg.body = jsonutils.dumps(data)
return msg
def reply(self, reply=None, failure=None):
if self.sender is not None:
if failure is not None:
failure = rpc_common.serialize_remote_exception(failure)
reply = zmq_response.Response(msg_type=zmq_names.REPLY_TYPE,
message_id=self.message_id,
reply_id=self.reply_id,
reply_body=reply,
failure=failure)
self.sender.send(self.socket, reply)
def reply(self, reply=None, failure=None):
if self.reply_sender is not None:
if failure is not None:
failure = rpc_common.serialize_remote_exception(failure)
reply = zmq_response.Reply(message_id=self.message_id,
reply_id=self.reply_id,
reply_body=reply,
failure=failure)
self.reply_sender.send(self.socket, reply)
if self.replies_cache is not None:
self.replies_cache.add(self.message_id, reply)
def marshal_response(reply=None, failure=None):
# TODO(grs): do replies have a context?
# NOTE(flaper87): Set inferred to True since rabbitmq-amqp-1.0 doesn't
# have support for vbin8.
msg = proton.Message(inferred=True)
if failure:
failure = common.serialize_remote_exception(failure)
data = {"failure": failure}
else:
data = {"response": reply}
msg.body = jsonutils.dumps(data)
return msg
def reply(self, reply=None, failure=None):
if failure is not None:
failure = rpc_common.serialize_remote_exception(failure)
response = zmq_response.Response(type=zmq_names.REPLY_TYPE,
message_id=self.request.message_id,
reply_id=self.reply_id,
reply_body=reply,
failure=failure)
LOG.debug("Replying %s", (str(self.request.message_id)))
self.received = True
self.reply_socket.send(self.reply_id, zmq.SNDMORE)
self.reply_socket.send(b'', zmq.SNDMORE)
self.reply_socket.send_pyobj(response)
def reply(self, reply=None, failure=None, log_failure=True):
if failure is not None:
failure = rpc_common.serialize_remote_exception(failure,
log_failure)
message_reply = {zmq_names.FIELD_REPLY: reply,
zmq_names.FIELD_FAILURE: failure,
zmq_names.FIELD_LOG_FAILURE: log_failure}
LOG.info("Replying %s REP", (str(message_reply)))
self.received = True
self.reply_socket.send(self.reply_id, zmq.SNDMORE)
self.reply_socket.send(b'', zmq.SNDMORE)
self.reply_socket.send_pyobj(message_reply)
self.poller.resume_polling(self.reply_socket)
def remote_exception_helper(self, name, error):
error.args = ()
exc_info = (type(error), error, None)
serialized = rpc_common.serialize_remote_exception(exc_info)
remote_error = rpc_common.deserialize_remote_exception(
serialized, name)
wrapper = fault.FaultWrapper(None)
msg = wrapper._error(remote_error)
expected = {'code': 500,
'error': {'traceback': None,
'type': 'RemoteError'},
'explanation': msg['explanation'],
'title': 'Internal Server Error'}
self.assertEqual(expected, msg)
def _get_response(self, ctx, proxy, topic, data):
"""Process a curried message and cast the result to topic."""
LOG.debug("Running func with context: %s", ctx.to_dict())
data.setdefault('version', None)
data.setdefault('args', {})
try:
if not data.get("method"):
raise KeyError
result = proxy.dispatch(ctx, data)
return ConsumerBase.normalize_reply(result, ctx.replies)
except greenlet.GreenletExit:
# ignore these since they are just from shutdowns
pass
except rpc_common.ClientException as e:
LOG.debug("Expected exception during message handling (%s)",
e._exc_info[1])
return {'exc':
rpc_common.serialize_remote_exception(e._exc_info,
log_failure=False)}
except Exception:
LOG.error(_("Exception during message handling"))
return {'exc':
rpc_common.serialize_remote_exception(sys.exc_info())}
def reply(self, reply=None, failure=None):
if failure is not None:
failure = rpc_common.serialize_remote_exception(failure)
response = zmq_response.Response(type=zmq_names.REPLY_TYPE,
message_id=self.request.message_id,
reply_id=self.reply_id,
reply_body=reply,
failure=failure)
LOG.debug("Replying %s", (str(self.request.message_id)))
self.received = True
self.reply_socket.send(self.reply_id, zmq.SNDMORE)
self.reply_socket.send(b'', zmq.SNDMORE)
self.reply_socket.send_pyobj(response)
def remote_exception_helper(self, name, error):
if six.PY3:
error.args = ()
exc_info = (type(error), error, None)
serialized = rpc_common.serialize_remote_exception(exc_info)
remote_error = rpc_common.deserialize_remote_exception(
serialized, name)
wrapper = fault.FaultWrapper(None)
msg = wrapper._error(remote_error)
expected = {'code': 500,
'error': {'traceback': None,
'type': 'RemoteError'},
'explanation': msg['explanation'],
'title': 'Internal Server Error'}
self.assertEqual(expected, msg)
def remote_exception_helper(self, name, error):
if six.PY3:
error.args = ()
exc_info = (type(error), error, None)
serialized = rpc_common.serialize_remote_exception(exc_info)
remote_error = rpc_common.deserialize_remote_exception(serialized, name)
wrapper = fault.FaultWrapper(None)
msg = wrapper._error(remote_error)
expected = {
"code": 500,
"error": {"message": msg["error"]["message"], "traceback": None, "type": "RemoteError"},
"explanation": msg["explanation"],
"title": "Internal Server Error",
}
self.assertEqual(expected, msg)
def reply(self, reply=None, failure=None, log_failure=True):
if failure is not None:
failure = rpc_common.serialize_remote_exception(failure, log_failure)
message_reply = {
zmq_names.FIELD_REPLY: reply,
zmq_names.FIELD_FAILURE: failure,
zmq_names.FIELD_LOG_FAILURE: log_failure,
}
LOG.info("Replying %s REP", (str(message_reply)))
self.received = True
self.reply_socket.send(self.reply_id, zmq.SNDMORE)
self.reply_socket.send(b"", zmq.SNDMORE)
self.reply_socket.send_pyobj(message_reply)
self.poller.resume_polling(self.reply_socket)
def test_remote_exception(self):
# We want tracebacks
cfg.CONF.set_override("debug", True)
error = heat_exc.EntityNotFound(entity="Stack", name="a")
exc_info = (type(error), error, None)
serialized = rpc_common.serialize_remote_exception(exc_info)
remote_error = rpc_common.deserialize_remote_exception(serialized, ["heat.common.exception"])
wrapper = fault.FaultWrapper(None)
msg = wrapper._error(remote_error)
expected_message, expected_traceback = six.text_type(remote_error).split("\n", 1)
expected = {
"code": 404,
"error": {"message": expected_message, "traceback": expected_traceback, "type": "EntityNotFound"},
"explanation": "The resource could not be found.",
"title": "Not Found",
}
self.assertEqual(expected, msg)
def reply(self, reply=None, failure=None, log_failure=True):
if failure is not None:
failure = rpc_common.serialize_remote_exception(failure,
log_failure)
message_reply = {zmq_names.FIELD_TYPE: zmq_names.REPLY_TYPE,
zmq_names.FIELD_REPLY: reply,
zmq_names.FIELD_FAILURE: failure,
zmq_names.FIELD_LOG_FAILURE: log_failure,
zmq_names.FIELD_MSG_ID: self.request.message_id}
LOG.debug("Replying %s", (str(self.request.message_id)))
self.received = True
self.reply_socket.send(self.reply_id, zmq.SNDMORE)
self.reply_socket.send(b'', zmq.SNDMORE)
self.reply_socket.send_pyobj(message_reply)
self.poller.resume_polling(self.reply_socket)
def _send_reply(self, conn, reply=None, failure=None, log_failure=True):
if not self._obsolete_reply_queues.reply_q_valid(self.reply_q, self.msg_id):
return
if failure:
failure = rpc_common.serialize_remote_exception(failure, log_failure)
# NOTE(sileht): ending can be removed in N*, see Listener.wait()
# for more detail.
msg = {"result": reply, "failure": failure, "ending": True, "_msg_id": self.msg_id}
rpc_amqp._add_unique_id(msg)
unique_id = msg[rpc_amqp.UNIQUE_ID]
LOG.debug(
"sending reply msg_id: %(msg_id)s " "reply queue: %(reply_q)s",
{"msg_id": self.msg_id, "unique_id": unique_id, "reply_q": self.reply_q},
)
conn.direct_send(self.reply_q, rpc_common.serialize_msg(msg))
def test_remote_exception(self):
# We want tracebacks
cfg.CONF.set_override('debug', True)
error = heat_exc.EntityNotFound(entity='Stack', name='a')
exc_info = (type(error), error, None)
serialized = rpc_common.serialize_remote_exception(exc_info)
remote_error = rpc_common.deserialize_remote_exception(
serialized, ["heat.common.exception"])
wrapper = fault.FaultWrapper(None)
msg = wrapper._error(remote_error)
expected_message, expected_traceback = six.text_type(
remote_error).split('\n', 1)
expected = {'code': 404,
'error': {'message': expected_message,
'traceback': expected_traceback,
'type': 'EntityNotFound'},
'explanation': 'The resource could not be found.',
'title': 'Not Found'}
self.assertEqual(expected, msg)
def test_remote_exception(self):
# We want tracebacks
cfg.CONF.set_override('debug', True)
error = heat_exc.EntityNotFound(entity='Stack', name='a')
exc_info = (type(error), error, None)
serialized = rpc_common.serialize_remote_exception(exc_info)
remote_error = rpc_common.deserialize_remote_exception(
serialized, ["heat.common.exception"])
wrapper = fault.FaultWrapper(None)
msg = wrapper._error(remote_error)
expected_message, expected_traceback = str(
remote_error).split('\n', 1)
expected = {'code': 404,
'error': {'message': expected_message,
'traceback': expected_traceback,
'type': 'EntityNotFound'},
'explanation': 'The resource could not be found.',
'title': 'Not Found'}
self.assertEqual(expected, msg)
def test_should_not_ignore_parent_classes_even_for_remote_ones(self):
# We want tracebacks
cfg.CONF.set_override('debug', True)
error = StackNotFoundChild(stack_name='a')
exc_info = (type(error), error, None)
serialized = rpc_common.serialize_remote_exception(exc_info)
remote_error = rpc_common.deserialize_remote_exception(
serialized, ["validator.tests.test_fault_middleware"])
wrapper = fault.FaultWrapper(None)
msg = wrapper._error(remote_error)
expected_message, expected_traceback = six.text_type(
remote_error).split('\n', 1)
error = {u'code': 500,
u'error': {
u'message': u"Remote error: StackNotFoundChild "
u"The %(entity)s (%(name)s) could not be "
u"found.\n[u'StackNotFoundChild: "
u"The %(entity)s (%(name)s) "
u"could not be found.\\n'].",
u'traceback': 'None\n',
u'type': 'RemoteError'},
u'explanation': 'The server has either erred or is incapable '
'of performing the requested operation.',
u'title': 'Internal Server Error'}
expected = {
'success': False,
'response': error,
'install': {
'success': False,
'response': ''
},
'test': {
'success': False,
'response': ''
},
'deploy': {
'success': False,
'response': ''
}
}
self.assertEqual(expected, msg)
def _send_reply(self, conn, reply=None, failure=None,
ending=False, log_failure=True):
if failure:
failure = rpc_common.serialize_remote_exception(failure,
log_failure)
msg = {'result': reply, 'failure': failure}
if ending:
msg['ending'] = True
rpc_amqp._add_unique_id(msg)
# If a reply_q exists, add the msg_id to the reply and pass the
# reply_q to direct_send() to use it as the response queue.
# Otherwise use the msg_id for backward compatibility.
if self.reply_q:
msg['_msg_id'] = self.msg_id
conn.direct_send(self.reply_q, rpc_common.serialize_msg(msg))
else:
conn.direct_send(self.msg_id, rpc_common.serialize_msg(msg))
def test_remote_exception(self):
cfg.CONF.set_override('debug', True, enforce_type=True)
error = senlin_exc.ClusterNotFound(cluster='a')
exc_info = (type(error), error, None)
serialized = rpc_common.serialize_remote_exception(exc_info)
remote_error = rpc_common.deserialize_remote_exception(
serialized, ["senlin.common.exception"])
wrapper = fault.FaultWrapper(None)
msg = wrapper._error(remote_error)
expected_message = six.text_type(remote_error).split('\n', 1)[0]
expected = {
'code': 404,
'error': {
'code': 404,
'message': expected_message,
'type': 'ClusterNotFound'
},
'explanation': 'The resource could not be found.',
'title': 'Not Found'
}
self.assertEqual(expected, msg)
def test_remote_exception(self):
cfg.CONF.set_override('debug', True)
error = senlin_exc.ResourceNotFound(type='cluster', id='a')
exc_info = (type(error), error, None)
serialized = rpc_common.serialize_remote_exception(exc_info)
remote_error = rpc_common.deserialize_remote_exception(
serialized, ["senlin.common.exception"])
wrapper = fault.FaultWrapper(None)
msg = wrapper._error(remote_error)
expected_message = six.text_type(remote_error).split('\n', 1)[0]
expected = {
'code': 404,
'error': {
'code': 404,
'message': expected_message,
'type': 'ResourceNotFound'
},
'explanation': 'The resource could not be found.',
'title': 'Not Found'
}
self.assertEqual(expected, msg)
def test_serialize_remote_exception(self):
errors = []
def stub_error(msg, *a, **kw):
if (a and len(a) == 1 and isinstance(a[0], dict) and a[0]):
a = a[0]
errors.append(str(msg) % a)
self.stubs.Set(exceptions.LOG, 'error', stub_error)
try:
try:
raise self.cls(*self.args, **self.kwargs)
except Exception as ex:
cls_error = ex
if self.add_remote:
ex = add_remote_postfix(ex)
raise ex
except Exception:
exc_info = sys.exc_info()
serialized = exceptions.serialize_remote_exception(
exc_info, log_failure=self.log_failure)
failure = jsonutils.loads(serialized)
self.assertEqual(self.clsname, failure['class'], failure)
self.assertEqual(self.modname, failure['module'])
self.assertEqual(self.msg, failure['message'])
self.assertEqual([self.msg], failure['args'])
self.assertEqual(self.kwargs, failure['kwargs'])
# Note: _Remote prefix not stripped from tracebacks
tb = cls_error.__class__.__name__ + ': ' + self.msg
self.assertIn(tb, ''.join(failure['tb']))
if self.log_failure:
self.assertTrue(len(errors) > 0, errors)
else:
self.assertEqual(0, len(errors), errors)
def test_serialize_remote_exception(self):
errors = []
def stub_error(msg, *a, **kw):
if (a and len(a) == 1 and isinstance(a[0], dict) and a[0]):
a = a[0]
errors.append(str(msg) % a)
self.stubs.Set(exceptions.LOG, 'error', stub_error)
try:
try:
raise self.cls(*self.args, **self.kwargs)
except Exception as ex:
cls_error = ex
if self.add_remote:
ex = add_remote_postfix(ex)
raise ex
except Exception:
exc_info = sys.exc_info()
serialized = exceptions.serialize_remote_exception(
exc_info, log_failure=self.log_failure)
failure = jsonutils.loads(serialized)
self.assertEqual(self.clsname, failure['class'], failure)
self.assertEqual(self.modname, failure['module'])
self.assertEqual(self.msg, failure['message'])
self.assertEqual([self.msg], failure['args'])
self.assertEqual(self.kwargs, failure['kwargs'])
# Note: _Remote prefix not stripped from tracebacks
tb = cls_error.__class__.__name__ + ': ' + self.msg
self.assertIn(tb, ''.join(failure['tb']))
if self.log_failure:
self.assertTrue(len(errors) > 0, errors)
else:
self.assertEqual(0, len(errors), errors)
def _send_reply(self, conn, reply=None, failure=None, ending=True):
if not self._obsolete_reply_queues.reply_q_valid(self.reply_q,
self.msg_id):
return
if failure:
failure = rpc_common.serialize_remote_exception(failure)
# NOTE(sileht): ending can be removed in N*, see Listener.wait()
# for more detail.
msg = {'result': reply, 'failure': failure, 'ending': ending,
'_msg_id': self.msg_id}
rpc_amqp._add_unique_id(msg)
unique_id = msg[rpc_amqp.UNIQUE_ID]
LOG.debug("sending reply msg_id: %(msg_id)s "
"reply queue: %(reply_q)s "
"time elapsed: %(elapsed)ss", {
'msg_id': self.msg_id,
'unique_id': unique_id,
'reply_q': self.reply_q,
'elapsed': self.stopwatch.elapsed()})
conn.direct_send(self.reply_q, rpc_common.serialize_msg(msg))
def _send_reply(self,
conn,
reply=None,
failure=None,
ending=False,
log_failure=True):
if failure:
failure = rpc_common.serialize_remote_exception(
failure, log_failure)
msg = {'result': reply, 'failure': failure}
if ending:
msg['ending'] = True
rpc_amqp._add_unique_id(msg)
# If a reply_q exists, add the msg_id to the reply and pass the
# reply_q to direct_send() to use it as the response queue.
# Otherwise use the msg_id for backward compatibility.
if self.reply_q:
msg['_msg_id'] = self.msg_id
conn.direct_send(self.reply_q, rpc_common.serialize_msg(msg))
else:
conn.direct_send(self.msg_id, rpc_common.serialize_msg(msg))
def reply(self, reply=None, failure=None, log_failure=True):
if failure is not None:
failure = rpc_common.serialize_remote_exception(
failure, log_failure)
message_reply = {
zmq_names.FIELD_TYPE: zmq_names.REPLY_TYPE,
zmq_names.FIELD_REPLY: reply,
zmq_names.FIELD_FAILURE: failure,
zmq_names.FIELD_LOG_FAILURE: log_failure,
zmq_names.FIELD_ID: self.request.proxy_reply_id,
zmq_names.FIELD_MSG_ID: self.request.message_id
}
LOG.debug("Replying %s", (str(self.request.message_id)))
self.received = True
self.reply_socket.send(self.reply_id, zmq.SNDMORE)
self.reply_socket.send(b'', zmq.SNDMORE)
if self.request.proxy_reply_id:
self.reply_socket.send_string(zmq_names.REPLY_TYPE, zmq.SNDMORE)
self.reply_socket.send(self.request.proxy_reply_id, zmq.SNDMORE)
self.reply_socket.send(b'', zmq.SNDMORE)
self.reply_socket.send_pyobj(message_reply)
self.poller.resume_polling(self.reply_socket)
def _do(self):
"""Look for a new RPC call and serve it"""
# Get all the messages in queue
msgs = self.RPC.query.all()
for msg in msgs:
# Find the first msg marked as enqueued.
if msg.working and \
(self.current_time_seconds() - self.millisec_to_sec(msg.updated))\
> self.conf.messaging_server.response_timeout:
msg.status = message.Message.ENQUEUED
msg.update(condition=self.working_status_condition)
if not msg.enqueued:
continue
if 'plan_name' in msg.ctxt.keys():
LOG.info('Plan name: {}'.format(msg.ctxt['plan_name']))
elif 'plan_name' in msg.args.keys():
LOG.info('Plan name: {}'.format(msg.args['plan_name']))
# Change the status to WORKING (operation with a lock)
msg.status = message.Message.WORKING
msg.owner = socket.gethostname()
# All update should have a condition (status == enqueued)
_is_updated = msg.update(condition=self.enqueued_status_condition)
if not _is_updated or 'FAILURE' in _is_updated:
continue
# RPC methods must not start/end with an underscore.
if msg.method.startswith('_') or msg.method.endswith('_'):
error_msg = _LE("Method {} must not start or end"
"with underscores").format(msg.method)
self._log_error_and_update_msg(msg, error_msg)
return
# The first endpoint that supports the method wins.
method = None
for endpoint in self.endpoints:
if msg.method not in dir(endpoint):
continue
endpoint_method = getattr(endpoint, msg.method)
if callable(endpoint_method):
method = endpoint_method
if self.conf.messaging_server.debug:
LOG.debug("Message {} method {} is "
"handled by endpoint {}".format(
msg.id, msg.method,
method.__str__.__name__))
break
if not method:
error_msg = _LE("Message {} method {} unsupported "
"in endpoints.").format(msg.id, msg.method)
self._log_error_and_update_msg(msg, error_msg)
return
# All methods must take a ctxt and args param.
if inspect.getargspec(method).args != ['self', 'ctx', 'arg']:
error_msg = _LE("Method {} must take three args: "
"self, ctx, arg").format(msg.method)
self._log_error_and_update_msg(msg, error_msg)
return
LOG.info(
_LI("Message {} method {} received").format(
msg.id, msg.method))
if self.conf.messaging_server.debug:
LOG.debug(
_LI("Message {} method {} context: {}, args: {}").format(
msg.id, msg.method, msg.ctxt, msg.args))
failure = None
try:
# Add the template to conductor.plan table
# Methods return an opaque dictionary
result = method(msg.ctxt, msg.args)
# FIXME(jdandrea): Remove response/error and make it opaque.
# That means this would just be assigned result outright.
msg.response = result.get('response', result)
except Exception:
# Current sys.exc_info() content can be overridden
# by another exception raised by a log handler during
# LOG.exception(). So keep a copy and delete it later.
failure = sys.exc_info()
# Do not log details about the failure here. It will
# be returned later upstream.
LOG.exception(_LE('Exception during message handling'))
try:
if failure is None:
msg.status = message.Message.COMPLETED
else:
msg.failure = \
rpc_common.serialize_remote_exception(failure)
msg.status = message.Message.ERROR
LOG.info(
_LI("Message {} method {}, status: {}").format(
msg.id, msg.method, msg.status))
if self.conf.messaging_server.debug:
LOG.debug("Message {} method {}, response: {}".format(
msg.id, msg.method, msg.response))
_is_success = 'FAILURE'
while 'FAILURE' in _is_success and (
self.current_time_seconds() -
self.millisec_to_sec(msg.updated)
) <= self.conf.messaging_server.response_timeout:
_is_success = msg.update()
LOG.info(
_LI("updating the message status from working to {}, "
"atomic update response from MUSIC {}").format(
msg.status, _is_success))
except Exception:
LOG.exception(
_LE("Can not send reply for message {} "
"method {}").format(msg.id, msg.method))
finally:
# Remove circular object reference between the current
# stack frame and the traceback in exc_info.
del failure