def _send(self, connection, request):
"""
:param request: JSON serializable dict
"""
dbutils.exit_if_in_transaction(log)
log.debug("send %s" % request["request_id"])
request["response_routing_key"] = self._response_routing_key
def errback(exc, _):
log.info(
"RabbitMQ rpc got a temporary error. May retry. Error: %r",
exc,
exc_info=1)
retry_policy = {"max_retries": 10, "errback": errback}
with Producer(connection) as producer:
maybe_declare(_amqp_exchange(), producer.channel, True,
**retry_policy)
producer.publish(
request,
serializer="json",
routing_key=self._request_routing_key,
delivery_mode=TRANSIENT_DELIVERY_MODE,
retry=True,
retry_policy=retry_policy,
)
def call(self, request, rpc_timeout=RESPONSE_TIMEOUT):
request_id = request["request_id"]
if not self._lightweight:
self.response_thread.start_wait(request_id, rpc_timeout)
with tx_connections[_amqp_connection()].acquire(block=True) as connection:
self._send(connection, request)
return self.response_thread.complete_wait(request_id)
else:
self._response_routing_key = "%s.responses_%s_%s_%s" % (
self._service_name,
os.uname()[1],
os.getpid(),
request_id,
)
self._complete = False
def callback(body, message):
# log.debug(body)
try:
jsonschema.validate(body, RESPONSE_SCHEMA)
except jsonschema.ValidationError as e:
log.debug("Malformed response: %s" % e)
else:
self._result = body
self._complete = True
finally:
message.ack()
with lw_connections[_amqp_connection()].acquire(block=True) as connection:
with connection.Consumer(
queues=[
kombu.messaging.Queue(
self._response_routing_key,
_amqp_exchange(),
routing_key=self._response_routing_key,
auto_delete=True,
durable=False,
)
],
callbacks=[callback],
):
self._send(connection, request)
timeout_at = time.time() + rpc_timeout
while not self._complete:
try:
connection.drain_events(timeout=1)
except socket.timeout:
pass
except IOError as e:
# See HYD-2551
if e.errno != errno.EINTR:
# if not [Errno 4] Interrupted system call
raise
if time.time() > timeout_at:
raise RpcTimeout()
return self._result
def run(self):
try:
result = {
"result": self.rpc._local_call(self.body["method"], *self.body["args"], **self.body["kwargs"]),
"request_id": self.body["request_id"],
"exception": None,
}
except Exception as e:
import sys
import traceback
exc_info = sys.exc_info()
backtrace = "\n".join(traceback.format_exception(*(exc_info or sys.exc_info())))
# Utility to generate human readable errors
def translate_error(err):
from socket import error as socket_error
if type(err) == socket_error:
return "Cannot reach server"
return str(err)
result = {
"request_id": self.body["request_id"],
"result": None,
"exception": translate_error(e),
"exception_type": type(e).__name__,
"traceback": backtrace,
}
log.error("RunOneRpc: exception calling %s: %s" % (self.body["method"], backtrace))
finally:
django.db.connection.close()
with self._response_conn_pool[_amqp_connection()].acquire(block=True) as connection:
def errback(exc, _):
log.info("RabbitMQ rpc got a temporary error. May retry. Error: %r", exc, exc_info=1)
retry_policy = {"max_retries": 10, "errback": errback}
connection.ensure_connection(**retry_policy)
with Producer(connection) as producer:
maybe_declare(_amqp_exchange(), producer.channel, True, **retry_policy)
producer.publish(
result,
serializer="json",
routing_key=self.body["response_routing_key"],
delivery_mode=TRANSIENT_DELIVERY_MODE,
retry=True,
retry_policy=retry_policy,
immedate=True,
mandatory=True,
)
def get_consumers(self, Consumer, channel):
return [
Consumer(queues=[
Queue(self.request_routing_key,
_amqp_exchange(),
routing_key=self.request_routing_key,
durable=False)
],
callbacks=[self.process_task])
]
def _send(self, connection, request):
"""
:param request: JSON serializable dict
"""
log.debug("send %s" % request['request_id'])
request['response_routing_key'] = self._response_routing_key
with Producer(connection) as producer:
maybe_declare(_amqp_exchange(), producer.channel)
producer.publish(request,
serializer="json",
routing_key=self._request_routing_key,
delivery_mode=1)
def run(self):
log.debug("ResponseThread.run")
def callback(body, message):
# log.debug(body)
try:
jsonschema.validate(body, RESPONSE_SCHEMA)
except jsonschema.ValidationError as e:
log.error("Malformed response: %s" % e)
else:
try:
state = self._response_states[body["request_id"]]
except KeyError:
log.debug("Unknown request ID %s" % body["request_id"])
else:
state.result = body
state.complete.set()
finally:
message.ack()
with rx_connections[_amqp_connection()].acquire(
block=True) as connection:
# Prepare the response queue
with connection.Consumer(
queues=[
kombu.messaging.Queue(
self._response_routing_key,
_amqp_exchange(),
routing_key=self._response_routing_key,
auto_delete=True,
durable=False,
)
],
callbacks=[callback],
):
self._started.set()
while not self._stopping:
try:
connection.drain_events(timeout=1)
except socket.timeout:
pass
except IOError as e:
# See HYD-2551
if e.errno != errno.EINTR:
# if not [Errno 4] Interrupted system call
raise
self._age_response_states()
log.debug("%s stopped" % self.__class__.__name__)
django.db.connection.close()
with self._response_conn_pool[_amqp_connection()].acquire(block=True) as connection:
def errback(exc, _):
log.info('RabbitMQ rpc got a temporary error. May retry. Error: %r', exc, exc_info=1)
retry_policy = {
'max_retries': 10,
'errback': errback
}
connection.ensure_connection(**retry_policy)
with Producer(connection) as producer:
maybe_declare(_amqp_exchange(), producer.channel, True, **retry_policy)
producer.publish(result, serializer="json", routing_key=self.body['response_routing_key'],
delivery_mode=TRANSIENT_DELIVERY_MODE, retry=True, retry_policy=retry_policy, immedate=True, mandatory=True)
class RpcServer(ConsumerMixin):
def __init__(self, rpc, connection, service_name, serialize = False):
"""
:param rpc: A ServiceRpcInterface instance
:param serialize: If True, then process RPCs one after another in a single thread
rather than running a thread for each RPC.
"""
super(RpcServer, self).__init__()
self.serialize = serialize
self.rpc = rpc
self.connection = connection
finally:
try:
if rpc_throttle:
try:
rpc_complete_event = RunOneRpc._throttled_locks.pop(0)
rpc_complete_event.set()
except IndexError:
pass
RunOneRpc._outstanding -= 1
finally:
django.db.connection.close()
with self._response_conn_pool[_amqp_connection()].acquire(
block=True) as connection:
with Producer(connection) as producer:
maybe_declare(_amqp_exchange(), producer.channel)
producer.publish(result,
serializer="json",
routing_key=self.body['response_routing_key'],
delivery_mode=1,
immedate=True,
mandatory=True)
class RpcServer(ConsumerMixin):
def __init__(self, rpc, connection, service_name, serialize=False):
"""
:param rpc: A ServiceRpcInterface instance
:param serialize: If True, then process RPCs one after another in a single thread
rather than running a thread for each RPC.
"""
