def call(self, request, rpc_timeout=RESPONSE_TIMEOUT): request_id = request["request_id"] if not self._lightweight: self.response_thread.start_wait(request_id, rpc_timeout) with tx_connections[_amqp_connection()].acquire(block=True) as connection: self._send(connection, request) return self.response_thread.complete_wait(request_id) else: self._response_routing_key = "%s.responses_%s_%s_%s" % ( self._service_name, os.uname()[1], os.getpid(), request_id, ) self._complete = False def callback(body, message): # log.debug(body) try: jsonschema.validate(body, RESPONSE_SCHEMA) except jsonschema.ValidationError as e: log.debug("Malformed response: %s" % e) else: self._result = body self._complete = True finally: message.ack() with lw_connections[_amqp_connection()].acquire(block=True) as connection: with connection.Consumer( queues=[ kombu.messaging.Queue( self._response_routing_key, _amqp_exchange(), routing_key=self._response_routing_key, auto_delete=True, durable=False, ) ], callbacks=[callback], ): self._send(connection, request) timeout_at = time.time() + rpc_timeout while not self._complete: try: connection.drain_events(timeout=1) except socket.timeout: pass except IOError as e: # See HYD-2551 if e.errno != errno.EINTR: # if not [Errno 4] Interrupted system call raise if time.time() > timeout_at: raise RpcTimeout() return self._result
def put(self, body): with _amqp_connection() as conn: q = conn.SimpleQueue(self.name, serializer='json', exchange_opts={'durable': False}, queue_opts={'durable': False}) q.put(body)
def _send_one_amqp(self, message): with _amqp_connection() as conn: q = conn.SimpleQueue(self.TX_QUEUE_NAME, serializer="json", exchange_opts={"durable": False}, queue_opts={"durable": False}) q.put(message)
def _flush_queue(self, queue): with _amqp_connection() as conn: conn.SimpleQueue(queue, exchange_opts={ "durable": False }, queue_opts={ "durable": False }).consumer.purge()
def run(self): try: result = { "result": self.rpc._local_call(self.body["method"], *self.body["args"], **self.body["kwargs"]), "request_id": self.body["request_id"], "exception": None, } except Exception as e: import sys import traceback exc_info = sys.exc_info() backtrace = "\n".join(traceback.format_exception(*(exc_info or sys.exc_info()))) # Utility to generate human readable errors def translate_error(err): from socket import error as socket_error if type(err) == socket_error: return "Cannot reach server" return str(err) result = { "request_id": self.body["request_id"], "result": None, "exception": translate_error(e), "exception_type": type(e).__name__, "traceback": backtrace, } log.error("RunOneRpc: exception calling %s: %s" % (self.body["method"], backtrace)) finally: django.db.connection.close() with self._response_conn_pool[_amqp_connection()].acquire(block=True) as connection: def errback(exc, _): log.info("RabbitMQ rpc got a temporary error. May retry. Error: %r", exc, exc_info=1) retry_policy = {"max_retries": 10, "errback": errback} connection.ensure_connection(**retry_policy) with Producer(connection) as producer: maybe_declare(_amqp_exchange(), producer.channel, True, **retry_policy) producer.publish( result, serializer="json", routing_key=self.body["response_routing_key"], delivery_mode=TRANSIENT_DELIVERY_MODE, retry=True, retry_policy=retry_policy, immedate=True, mandatory=True, )
def purge(self): with _amqp_connection() as conn: purged = conn.SimpleQueue(self.name, exchange_opts={ 'durable': False }, queue_opts={ 'durable': False }).consumer.purge() log.info("Purged %s messages from '%s' queue" % (purged, self.name))
def _receive_one_amqp(self): TIMEOUT = RABBITMQ_GRACE_PERIOD # Data message should be forwarded to AMQP with _amqp_connection() as conn: q = conn.SimpleQueue(self.RX_QUEUE_NAME, serializer = 'json', exchange_opts={'durable': False}, queue_opts={'durable': False}) try: message = q.get(timeout = TIMEOUT) message.ack() except Empty: raise AssertionError("No message received in %s seconds from queue %s" % (TIMEOUT, self.RX_QUEUE_NAME)) else: return message.decode()
def run(self): log.debug("ResponseThread.run") def callback(body, message): # log.debug(body) try: jsonschema.validate(body, RESPONSE_SCHEMA) except jsonschema.ValidationError as e: log.error("Malformed response: %s" % e) else: try: state = self._response_states[body["request_id"]] except KeyError: log.debug("Unknown request ID %s" % body["request_id"]) else: state.result = body state.complete.set() finally: message.ack() with rx_connections[_amqp_connection()].acquire( block=True) as connection: # Prepare the response queue with connection.Consumer( queues=[ kombu.messaging.Queue( self._response_routing_key, _amqp_exchange(), routing_key=self._response_routing_key, auto_delete=True, durable=False, ) ], callbacks=[callback], ): self._started.set() while not self._stopping: try: connection.drain_events(timeout=1) except socket.timeout: pass except IOError as e: # See HYD-2551 if e.errno != errno.EINTR: # if not [Errno 4] Interrupted system call raise self._age_response_states() log.debug("%s stopped" % self.__class__.__name__)
def run(self): with _amqp_connection() as conn: while not self._stopping.is_set(): try: msg = self._queue_collection.plugin_rx_queue.get( block=True, timeout=1) except Queue.Empty: pass else: plugin_name = msg['plugin'] rx_queue_name = "agent_%s_rx" % plugin_name q = conn.SimpleQueue(rx_queue_name, serializer='json', exchange_opts={'durable': False}, queue_opts={'durable': False}) q.put(msg)
def serve(self, callback): from Queue import Empty as QueueEmpty with _amqp_connection() as conn: q = conn.SimpleQueue(self.name, serializer='json', exchange_opts={'durable': False}, queue_opts={'durable': False}) # FIXME: it would be preferable to avoid waking up so often: really what is wanted # here is to sleep on messages or a stop event. while not self._stopping.is_set(): try: message = q.get(timeout=1) message.ack() message = message.decode() callback(message) except QueueEmpty: pass
def run(self): with _amqp_connection() as connection: self.worker = RpcServer(self, connection, self.__class__.__name__) self.worker.run()
'traceback': backtrace } log.error("RunOneRpc: exception calling %s: %s" % (self.body['method'], backtrace)) finally: try: if rpc_throttle: try: rpc_complete_event = RunOneRpc._throttled_locks.pop(0) rpc_complete_event.set() except IndexError: pass RunOneRpc._outstanding -= 1 finally: django.db.connection.close() with self._response_conn_pool[_amqp_connection()].acquire(block=True) as connection: def errback(exc, _): log.info('RabbitMQ rpc got a temporary error. May retry. Error: %r', exc, exc_info=1) retry_policy = { 'max_retries': 10, 'errback': errback } connection.ensure_connection(**retry_policy) with Producer(connection) as producer: maybe_declare(_amqp_exchange(), producer.channel, True, **retry_policy) producer.publish(result, serializer="json", routing_key=self.body['response_routing_key'], delivery_mode=TRANSIENT_DELIVERY_MODE, retry=True, retry_policy=retry_policy, immedate=True, mandatory=True)
} log.error("RunOneRpc: exception calling %s: %s" % (self.body['method'], backtrace)) finally: try: if rpc_throttle: try: rpc_complete_event = RunOneRpc._throttled_locks.pop(0) rpc_complete_event.set() except IndexError: pass RunOneRpc._outstanding -= 1 finally: django.db.connection.close() with self._response_conn_pool[_amqp_connection()].acquire( block=True) as connection: with Producer(connection) as producer: maybe_declare(_amqp_exchange(), producer.channel) producer.publish(result, serializer="json", routing_key=self.body['response_routing_key'], delivery_mode=1, immedate=True, mandatory=True) class RpcServer(ConsumerMixin): def __init__(self, rpc, connection, service_name, serialize=False): """ :param rpc: A ServiceRpcInterface instance