def call(self, request, rpc_timeout=RESPONSE_TIMEOUT):
request_id = request["request_id"]
if not self._lightweight:
self.response_thread.start_wait(request_id, rpc_timeout)
with tx_connections[_amqp_connection()].acquire(block=True) as connection:
self._send(connection, request)
return self.response_thread.complete_wait(request_id)
else:
self._response_routing_key = "%s.responses_%s_%s_%s" % (
self._service_name,
os.uname()[1],
os.getpid(),
request_id,
)
self._complete = False
def callback(body, message):
# log.debug(body)
try:
jsonschema.validate(body, RESPONSE_SCHEMA)
except jsonschema.ValidationError as e:
log.debug("Malformed response: %s" % e)
else:
self._result = body
self._complete = True
finally:
message.ack()
with lw_connections[_amqp_connection()].acquire(block=True) as connection:
with connection.Consumer(
queues=[
kombu.messaging.Queue(
self._response_routing_key,
_amqp_exchange(),
routing_key=self._response_routing_key,
auto_delete=True,
durable=False,
)
],
callbacks=[callback],
):
self._send(connection, request)
timeout_at = time.time() + rpc_timeout
while not self._complete:
try:
connection.drain_events(timeout=1)
except socket.timeout:
pass
except IOError as e:
# See HYD-2551
if e.errno != errno.EINTR:
# if not [Errno 4] Interrupted system call
raise
if time.time() > timeout_at:
raise RpcTimeout()
return self._result
def put(self, body):
with _amqp_connection() as conn:
q = conn.SimpleQueue(self.name,
serializer='json',
exchange_opts={'durable': False},
queue_opts={'durable': False})
q.put(body)
def _send_one_amqp(self, message):
with _amqp_connection() as conn:
q = conn.SimpleQueue(self.TX_QUEUE_NAME,
serializer="json",
exchange_opts={"durable": False},
queue_opts={"durable": False})
q.put(message)
def _flush_queue(self, queue):
with _amqp_connection() as conn:
conn.SimpleQueue(queue,
exchange_opts={
"durable": False
},
queue_opts={
"durable": False
}).consumer.purge()
def run(self):
try:
result = {
"result": self.rpc._local_call(self.body["method"], *self.body["args"], **self.body["kwargs"]),
"request_id": self.body["request_id"],
"exception": None,
}
except Exception as e:
import sys
import traceback
exc_info = sys.exc_info()
backtrace = "\n".join(traceback.format_exception(*(exc_info or sys.exc_info())))
# Utility to generate human readable errors
def translate_error(err):
from socket import error as socket_error
if type(err) == socket_error:
return "Cannot reach server"
return str(err)
result = {
"request_id": self.body["request_id"],
"result": None,
"exception": translate_error(e),
"exception_type": type(e).__name__,
"traceback": backtrace,
}
log.error("RunOneRpc: exception calling %s: %s" % (self.body["method"], backtrace))
finally:
django.db.connection.close()
with self._response_conn_pool[_amqp_connection()].acquire(block=True) as connection:
def errback(exc, _):
log.info("RabbitMQ rpc got a temporary error. May retry. Error: %r", exc, exc_info=1)
retry_policy = {"max_retries": 10, "errback": errback}
connection.ensure_connection(**retry_policy)
with Producer(connection) as producer:
maybe_declare(_amqp_exchange(), producer.channel, True, **retry_policy)
producer.publish(
result,
serializer="json",
routing_key=self.body["response_routing_key"],
delivery_mode=TRANSIENT_DELIVERY_MODE,
retry=True,
retry_policy=retry_policy,
immedate=True,
mandatory=True,
)
def purge(self):
with _amqp_connection() as conn:
purged = conn.SimpleQueue(self.name,
exchange_opts={
'durable': False
},
queue_opts={
'durable': False
}).consumer.purge()
log.info("Purged %s messages from '%s' queue" %
(purged, self.name))
def _receive_one_amqp(self):
TIMEOUT = RABBITMQ_GRACE_PERIOD
# Data message should be forwarded to AMQP
with _amqp_connection() as conn:
q = conn.SimpleQueue(self.RX_QUEUE_NAME, serializer = 'json',
exchange_opts={'durable': False}, queue_opts={'durable': False})
try:
message = q.get(timeout = TIMEOUT)
message.ack()
except Empty:
raise AssertionError("No message received in %s seconds from queue %s" % (TIMEOUT, self.RX_QUEUE_NAME))
else:
return message.decode()
def run(self):
log.debug("ResponseThread.run")
def callback(body, message):
# log.debug(body)
try:
jsonschema.validate(body, RESPONSE_SCHEMA)
except jsonschema.ValidationError as e:
log.error("Malformed response: %s" % e)
else:
try:
state = self._response_states[body["request_id"]]
except KeyError:
log.debug("Unknown request ID %s" % body["request_id"])
else:
state.result = body
state.complete.set()
finally:
message.ack()
with rx_connections[_amqp_connection()].acquire(
block=True) as connection:
# Prepare the response queue
with connection.Consumer(
queues=[
kombu.messaging.Queue(
self._response_routing_key,
_amqp_exchange(),
routing_key=self._response_routing_key,
auto_delete=True,
durable=False,
)
],
callbacks=[callback],
):
self._started.set()
while not self._stopping:
try:
connection.drain_events(timeout=1)
except socket.timeout:
pass
except IOError as e:
# See HYD-2551
if e.errno != errno.EINTR:
# if not [Errno 4] Interrupted system call
raise
self._age_response_states()
log.debug("%s stopped" % self.__class__.__name__)
def run(self):
with _amqp_connection() as conn:
while not self._stopping.is_set():
try:
msg = self._queue_collection.plugin_rx_queue.get(
block=True, timeout=1)
except Queue.Empty:
pass
else:
plugin_name = msg['plugin']
rx_queue_name = "agent_%s_rx" % plugin_name
q = conn.SimpleQueue(rx_queue_name,
serializer='json',
exchange_opts={'durable': False},
queue_opts={'durable': False})
q.put(msg)
def serve(self, callback):
from Queue import Empty as QueueEmpty
with _amqp_connection() as conn:
q = conn.SimpleQueue(self.name,
serializer='json',
exchange_opts={'durable': False},
queue_opts={'durable': False})
# FIXME: it would be preferable to avoid waking up so often: really what is wanted
# here is to sleep on messages or a stop event.
while not self._stopping.is_set():
try:
message = q.get(timeout=1)
message.ack()
message = message.decode()
callback(message)
except QueueEmpty:
pass
def run(self):
with _amqp_connection() as connection:
self.worker = RpcServer(self, connection, self.__class__.__name__)
self.worker.run()
'traceback': backtrace
}
log.error("RunOneRpc: exception calling %s: %s" % (self.body['method'], backtrace))
finally:
try:
if rpc_throttle:
try:
rpc_complete_event = RunOneRpc._throttled_locks.pop(0)
rpc_complete_event.set()
except IndexError:
pass
RunOneRpc._outstanding -= 1
finally:
django.db.connection.close()
with self._response_conn_pool[_amqp_connection()].acquire(block=True) as connection:
def errback(exc, _):
log.info('RabbitMQ rpc got a temporary error. May retry. Error: %r', exc, exc_info=1)
retry_policy = {
'max_retries': 10,
'errback': errback
}
connection.ensure_connection(**retry_policy)
with Producer(connection) as producer:
maybe_declare(_amqp_exchange(), producer.channel, True, **retry_policy)
producer.publish(result, serializer="json", routing_key=self.body['response_routing_key'],
delivery_mode=TRANSIENT_DELIVERY_MODE, retry=True, retry_policy=retry_policy, immedate=True, mandatory=True)
}
log.error("RunOneRpc: exception calling %s: %s" %
(self.body['method'], backtrace))
finally:
try:
if rpc_throttle:
try:
rpc_complete_event = RunOneRpc._throttled_locks.pop(0)
rpc_complete_event.set()
except IndexError:
pass
RunOneRpc._outstanding -= 1
finally:
django.db.connection.close()
with self._response_conn_pool[_amqp_connection()].acquire(
block=True) as connection:
with Producer(connection) as producer:
maybe_declare(_amqp_exchange(), producer.channel)
producer.publish(result,
serializer="json",
routing_key=self.body['response_routing_key'],
delivery_mode=1,
immedate=True,
mandatory=True)
class RpcServer(ConsumerMixin):
def __init__(self, rpc, connection, service_name, serialize=False):
"""
:param rpc: A ServiceRpcInterface instance
