def _send(self, connection, request): """ :param request: JSON serializable dict """ dbutils.exit_if_in_transaction(log) log.debug("send %s" % request["request_id"]) request["response_routing_key"] = self._response_routing_key def errback(exc, _): log.info( "RabbitMQ rpc got a temporary error. May retry. Error: %r", exc, exc_info=1) retry_policy = {"max_retries": 10, "errback": errback} with Producer(connection) as producer: maybe_declare(_amqp_exchange(), producer.channel, True, **retry_policy) producer.publish( request, serializer="json", routing_key=self._request_routing_key, delivery_mode=TRANSIENT_DELIVERY_MODE, retry=True, retry_policy=retry_policy, )
def call(self, request, rpc_timeout=RESPONSE_TIMEOUT): request_id = request["request_id"] if not self._lightweight: self.response_thread.start_wait(request_id, rpc_timeout) with tx_connections[_amqp_connection()].acquire(block=True) as connection: self._send(connection, request) return self.response_thread.complete_wait(request_id) else: self._response_routing_key = "%s.responses_%s_%s_%s" % ( self._service_name, os.uname()[1], os.getpid(), request_id, ) self._complete = False def callback(body, message): # log.debug(body) try: jsonschema.validate(body, RESPONSE_SCHEMA) except jsonschema.ValidationError as e: log.debug("Malformed response: %s" % e) else: self._result = body self._complete = True finally: message.ack() with lw_connections[_amqp_connection()].acquire(block=True) as connection: with connection.Consumer( queues=[ kombu.messaging.Queue( self._response_routing_key, _amqp_exchange(), routing_key=self._response_routing_key, auto_delete=True, durable=False, ) ], callbacks=[callback], ): self._send(connection, request) timeout_at = time.time() + rpc_timeout while not self._complete: try: connection.drain_events(timeout=1) except socket.timeout: pass except IOError as e: # See HYD-2551 if e.errno != errno.EINTR: # if not [Errno 4] Interrupted system call raise if time.time() > timeout_at: raise RpcTimeout() return self._result
def run(self): try: result = { "result": self.rpc._local_call(self.body["method"], *self.body["args"], **self.body["kwargs"]), "request_id": self.body["request_id"], "exception": None, } except Exception as e: import sys import traceback exc_info = sys.exc_info() backtrace = "\n".join(traceback.format_exception(*(exc_info or sys.exc_info()))) # Utility to generate human readable errors def translate_error(err): from socket import error as socket_error if type(err) == socket_error: return "Cannot reach server" return str(err) result = { "request_id": self.body["request_id"], "result": None, "exception": translate_error(e), "exception_type": type(e).__name__, "traceback": backtrace, } log.error("RunOneRpc: exception calling %s: %s" % (self.body["method"], backtrace)) finally: django.db.connection.close() with self._response_conn_pool[_amqp_connection()].acquire(block=True) as connection: def errback(exc, _): log.info("RabbitMQ rpc got a temporary error. May retry. Error: %r", exc, exc_info=1) retry_policy = {"max_retries": 10, "errback": errback} connection.ensure_connection(**retry_policy) with Producer(connection) as producer: maybe_declare(_amqp_exchange(), producer.channel, True, **retry_policy) producer.publish( result, serializer="json", routing_key=self.body["response_routing_key"], delivery_mode=TRANSIENT_DELIVERY_MODE, retry=True, retry_policy=retry_policy, immedate=True, mandatory=True, )
def get_consumers(self, Consumer, channel): return [ Consumer(queues=[ Queue(self.request_routing_key, _amqp_exchange(), routing_key=self.request_routing_key, durable=False) ], callbacks=[self.process_task]) ]
def _send(self, connection, request): """ :param request: JSON serializable dict """ log.debug("send %s" % request['request_id']) request['response_routing_key'] = self._response_routing_key with Producer(connection) as producer: maybe_declare(_amqp_exchange(), producer.channel) producer.publish(request, serializer="json", routing_key=self._request_routing_key, delivery_mode=1)
def run(self): log.debug("ResponseThread.run") def callback(body, message): # log.debug(body) try: jsonschema.validate(body, RESPONSE_SCHEMA) except jsonschema.ValidationError as e: log.error("Malformed response: %s" % e) else: try: state = self._response_states[body["request_id"]] except KeyError: log.debug("Unknown request ID %s" % body["request_id"]) else: state.result = body state.complete.set() finally: message.ack() with rx_connections[_amqp_connection()].acquire( block=True) as connection: # Prepare the response queue with connection.Consumer( queues=[ kombu.messaging.Queue( self._response_routing_key, _amqp_exchange(), routing_key=self._response_routing_key, auto_delete=True, durable=False, ) ], callbacks=[callback], ): self._started.set() while not self._stopping: try: connection.drain_events(timeout=1) except socket.timeout: pass except IOError as e: # See HYD-2551 if e.errno != errno.EINTR: # if not [Errno 4] Interrupted system call raise self._age_response_states() log.debug("%s stopped" % self.__class__.__name__)
django.db.connection.close() with self._response_conn_pool[_amqp_connection()].acquire(block=True) as connection: def errback(exc, _): log.info('RabbitMQ rpc got a temporary error. May retry. Error: %r', exc, exc_info=1) retry_policy = { 'max_retries': 10, 'errback': errback } connection.ensure_connection(**retry_policy) with Producer(connection) as producer: maybe_declare(_amqp_exchange(), producer.channel, True, **retry_policy) producer.publish(result, serializer="json", routing_key=self.body['response_routing_key'], delivery_mode=TRANSIENT_DELIVERY_MODE, retry=True, retry_policy=retry_policy, immedate=True, mandatory=True) class RpcServer(ConsumerMixin): def __init__(self, rpc, connection, service_name, serialize = False): """ :param rpc: A ServiceRpcInterface instance :param serialize: If True, then process RPCs one after another in a single thread rather than running a thread for each RPC. """ super(RpcServer, self).__init__() self.serialize = serialize self.rpc = rpc self.connection = connection
finally: try: if rpc_throttle: try: rpc_complete_event = RunOneRpc._throttled_locks.pop(0) rpc_complete_event.set() except IndexError: pass RunOneRpc._outstanding -= 1 finally: django.db.connection.close() with self._response_conn_pool[_amqp_connection()].acquire( block=True) as connection: with Producer(connection) as producer: maybe_declare(_amqp_exchange(), producer.channel) producer.publish(result, serializer="json", routing_key=self.body['response_routing_key'], delivery_mode=1, immedate=True, mandatory=True) class RpcServer(ConsumerMixin): def __init__(self, rpc, connection, service_name, serialize=False): """ :param rpc: A ServiceRpcInterface instance :param serialize: If True, then process RPCs one after another in a single thread rather than running a thread for each RPC. """