class BaseWorker(object): def __init__(self, ioloop, app): self.io_loop = ioloop self.app = app self.connected = False self.connecting = False self.connection = None self.channel = None self.consumer_tag = None self.event_listeners = set([]) def connect(self): if self.connecting: return self.connecting = True param = pika.ConnectionParameters(host=QUEUE_HOST) self.connection = TornadoConnection( param, on_open_callback=self.on_connected) self.connection.add_on_close_callback(self.on_closed) def on_connected(self, connection): self.connected = True self.connection = connection self.connection.channel(self.on_channel_open) def on_channel_open(self, channel): self.channel = channel channel.queue_declare( self.on_queue_declareok, queue=QUEUE_NAME, durable=True) def on_queue_declareok(self, method_frame): self.consumer_tag = self.channel.basic_consume(self.on_message, QUEUE_NAME) def on_closed(self, connection): self.io_loop.stop() def on_message(self, channel, method, header, body): logger.info('Message received: %s' % body) self.process(channel, method, header, body) self.channel.basic_ack(delivery_tag=method.delivery_tag) def process(self, channel, method, header, body): raise NotImplementedError
class QueueManager(object): """ Async RabbitMQ & Tornado websocket connector """ INPUT_QUEUE_NAME = 'in_queue' def __init__(self, io_loop=None): log.info('PikaClient: __init__') self.io_loop = io_loop self.connected = False self.connecting = False self.connection = None self.in_channel = None self.out_channels = {} self.out_channel = None self.websockets = {} # self.connect() def connect(self): """ Creates connection to RabbitMQ server """ if self.connecting: log.info('PikaClient: Already connecting to RabbitMQ') return log.info('PikaClient: Connecting to RabbitMQ') self.connecting = True self.connection = TornadoConnection(NON_BLOCKING_MQ_PARAMS, stop_ioloop_on_close=False, custom_ioloop=self.io_loop, on_open_callback=self.on_connected) def on_connected(self, connection): """ AMQP connection callback. Creates input channel. Args: connection: AMQP connection """ log.info('PikaClient: connected to RabbitMQ') self.connected = True self.in_channel = self.connection.channel(self.on_channel_open) def on_channel_open(self, channel): """ Input channel creation callback Queue declaration done here Args: channel: input channel """ self.in_channel.exchange_declare(exchange='input_exc', type='topic', durable=True) channel.queue_declare(callback=self.on_input_queue_declare, queue=self.INPUT_QUEUE_NAME) def on_input_queue_declare(self, queue): """ Input queue declaration callback. Input Queue/Exchange binding done here Args: queue: input queue """ self.in_channel.queue_bind(callback=None, exchange='input_exc', queue=self.INPUT_QUEUE_NAME, routing_key="#") def register_websocket(self, sess_id, ws): """ Args: sess_id: ws: """ self.websockets[sess_id] = ws self.create_out_channel(sess_id) def inform_disconnection(self, sess_id): self.in_channel.basic_publish(exchange='input_exc', routing_key=sess_id, body=json_encode( dict(data={ 'view': '_zops_mark_offline_user', 'sess_id': sess_id, }, _zops_source='Internal', _zops_remote_ip=''))) self.websockets[sess_id].write_message( json.dumps({ "cmd": "status", "status": "closing" })) def unregister_websocket(self, sess_id): # user_id = sys.sessid_to_userid.get(sess_id, None) try: self.inform_disconnection(sess_id) del self.websockets[sess_id] except KeyError: log.exception("Non-existent websocket for %s" % sess_id) if sess_id in self.out_channels: try: self.out_channels[sess_id].close() except ChannelClosed: log.exception("Pika client (out) channel already closed") def create_out_channel(self, sess_id): def _on_output_channel_creation(channel): def _on_output_queue_decleration(queue): # differentiate and identify incoming message with registered consumer channel.basic_consume( self.on_message, queue=sess_id, consumer_tag=sess_id, # no_ack=True ) log.debug("BINDED QUEUE TO WS Q.%s" % sess_id) self.out_channels[sess_id] = channel channel.queue_declare( callback=_on_output_queue_decleration, queue=sess_id, arguments={'x-expires': 40000}, # auto_delete=True, # exclusive=True ) self.connection.channel(_on_output_channel_creation) def redirect_incoming_message(self, sess_id, message, request): message = json_decode(message) message['_zops_sess_id'] = sess_id message['_zops_remote_ip'] = request.remote_ip message['_zops_source'] = 'Remote' self.in_channel.basic_publish(exchange='input_exc', routing_key=sess_id, body=json_encode(message)) def on_message(self, channel, method, header, body): sess_id = method.consumer_tag log.debug("WS RPLY for %s" % sess_id) log.debug("WS BODY for %s" % body) try: if sess_id in self.websockets: log.info("write msg to client") self.websockets[sess_id].write_message(body) log.debug("WS OBJ %s" % self.websockets[sess_id]) channel.basic_ack(delivery_tag=method.delivery_tag) except RuntimeError: log.exception("CANT WRITE TO HTTP OR WS: %s\n \n%s" % (sess_id, body)) except KeyError: self.unregister_websocket(sess_id) log.exception("CANT FIND WS OR HTTP: %s" % sess_id)
class PikaClient(): def __init__(self, ioloop): self.ioloop = ioloop self.connected = False self.connecting = False self.connection = None self.channel = None self.event_listeners = set([]) def connect(self): if self.connecting: LOGGER.info('Already connected to RabbitMQ server') return LOGGER.info('Opening a connection to RabbitMQ server') self.connecting = True self.connection = TornadoConnection( parameters=settings.pika_parameters, on_open_callback=self.on_connection_open) self.connection.add_on_close_callback(self.on_closed) def on_connection_open(self, connection): LOGGER.info('Opening a connection to RabbitMQ server completed') self.connected = True self.connection = connection self.open_channel() def open_channel(self): LOGGER.info('Opening a channel') self.connection.channel(on_open_callback=self.on_channel_open) def on_channel_open(self, channel): LOGGER.info('Opening a channel completed') self.channel = channel self.declare_exchange() # def on_channel_open(self, channel): # LOGGER.info('PikaClient: Channel open, Declaring exchange') # self.channel = channel # # declare exchanges, which in turn, declare # # queues, and bind exchange to queues def declare_exchange(self): LOGGER.info('Declaring an exchange') self.channel.exchange_declare( exchange=settings.RABBITMQ_NEWSFEED_ENTRY_EXCHANGE_NAME, exchange_type=settings.RABBITMQ_NEWSFEED_ENTRY_EXCHANGE_TYPE, passive=settings.RABBITMQ_NEWSFEED_ENTRY_EXCHANGE_PASSIVE, durable=settings.RABBITMQ_NEWSFEED_ENTRY_EXCHANGE_DURABLE, auto_delete=settings.RABBITMQ_NEWSFEED_ENTRY_EXCHANGE_AUTO_DELETE, internal=settings.RABBITMQ_NEWSFEED_ENTRY_EXCHANGE_INTERNAL, nowait=settings.RABBITMQ_NEWSFEED_ENTRY_EXCHANGE_NOWAIT, arguments=None, callback=self.on_exchange_declared) def on_exchange_declared(self, unused_frame): LOGGER.info('Declaring an exchange completed') self.declare_queue() def declare_queue(self): LOGGER.info('Declaring a queue') self.channel.queue_declare( callback=self.on_queue_declared, queue=settings.RABBITMQ_NEWSFEED_ENTRY_QUEUE_NAME, passive=settings.RABBITMQ_NEWSFEED_ENTRY_QUEUE_PASSIVE, durable=settings.RABBITMQ_NEWSFEED_ENTRY_QUEUE_DURABLE, exclusive=settings.RABBITMQ_NEWSFEED_ENTRY_QUEUE_EXCLUSIVE, auto_delete=settings.RABBITMQ_NEWSFEED_ENTRY_QUEUE_AUTO_DELETE, nowait=settings.RABBITMQ_NEWSFEED_ENTRY_QUEUE_NOWAIT, arguments=None) def on_queue_declared(self, method_frame): LOGGER.info('Declaring a queue completed') self.bind_queue() def bind_queue(self): LOGGER.info('Binding a queue') self.channel.queue_bind( callback=self.on_queue_binded, queue=settings.RABBITMQ_NEWSFEED_ENTRY_QUEUE_NAME, exchange=settings.RABBITMQ_NEWSFEED_ENTRY_EXCHANGE_NAME, routing_key=settings.RABBITMQ_NEWSFEED_ENTRY_ROUTING_KEY) def on_queue_binded(self, frame): LOGGER.info('Binding a queue completed') self.channel.basic_consume( consumer_callback=self.on_message, queue=settings.RABBITMQ_NEWSFEED_ENTRY_QUEUE_NAME, no_ack= False, # Set to True means tell the broker to not expect a response exclusive= False, # Set to True means don't allow other consumers on the queue consumer_tag=None) # Specify your own consumer tag def on_closed(self, connection): LOGGER.info('Connection to RabbitMQ server closed') self.ioloop.stop() #def on_message(self, channel, method, header, body): def on_message(self, channel, basic_deliver, properties, body): LOGGER.info('Received message # %s from %s', basic_deliver.delivery_tag, properties.app_id) self.ack_message(basic_deliver.delivery_tag) self.notify_listeners(body) def ack_message(self, delivery_tag): """Acknowledge the message delivery from RabbitMQ by sending a Basic.Ack RPC method for the delivery tag. :param int delivery_tag: The delivery tag from the Basic.Deliver frame """ LOGGER.info('Acknowledging message %s', delivery_tag) self.channel.basic_ack(delivery_tag) def notify_listeners(self, event_obj): # here we assume the message the sourcing app # post to the message queue is in JSON format #event_json = json.dumps(event_obj) event_json = event_obj for listener in self.event_listeners: listener.write_message(event_json) LOGGER.info('Notified %s' % repr(listener)) def add_event_listener(self, listener): self.event_listeners.add(listener) LOGGER.info('Added listener %s' % repr(listener)) def remove_event_listener(self, listener): try: self.event_listeners.remove(listener) LOGGER.info('Removed listener %s' % repr(listener)) except KeyError: pass
class PikaConnection(object): QUEUE = 'test' EXCHANGE = '' PUBLISH_INTERVAL = 1 def __init__(self, io_loop, amqp_url): self.io_loop = io_loop self.url = amqp_url self.is_connected = False self.is_connecting = False self.connection = None self.channel = None self.listeners = set([]) def connect(self): if self.is_connecting: logger.info("PikaConnection: Already connecting to RabbitMQ") return logger.info("PikaConnection: Connecting to RabbitMQ") self.connecting = True self.connection = TornadoConnection( pika.URLParameters(self.url), on_open_callback=self.on_connected) self.connection.add_on_close_callback(self.on_closed) def on_connected(self, connection): logger.info("PikaConnection: connected to RabbitMQ") self.connected = True self.connection = connection self.connection.channel(self.on_channel_open) def on_closed(self, connection): logger.info("PikaConnection: connection closed") self.io_loop.stop() def on_channel_open(self, channel): self.channel = channel self.channel.add_on_close_callback(self.on_channel_closed) self.channel.queue_declare(queue=self.QUEUE, callback=self.on_queue_declared) def on_channel_closed(self, channel, reply_code, reply_text): self.connection.close() def on_queue_declared(self, frame): print "subscribe frame:", frame self.channel.basic_consume(self.on_message, self.QUEUE) def on_message(self, channel, method, header, body): logger.info(body) for listener in self.listeners: listener.emit(body) def add_event_listener(self, listener): self.listeners.add(listener) def publish_message(self, *args, **kwargs): if 'message' in kwargs: self.channel.basic_publish(exchange=self.EXCHANGE, routing_key=self.QUEUE, body=kwargs['message'])
class PikaClient(object): INPUT_QUEUE_NAME = 'in_queue' def __init__(self, io_loop): self.io_loop = io_loop self.received_message_counter = 0 self.sent_message_counter = 0 self.start_time = -1 self.connected = False self.connecting = False self.connection = None self.in_channel = None self.out_channels = {} self.websockets = {} self.connect() def connect(self): if self.connecting: return self.connecting = True cred = pika.PlainCredentials('guest', 'guest') param = pika.ConnectionParameters( host='localhost', port=5672, virtual_host='/', credentials=cred ) self.connection = TornadoConnection(param, on_open_callback=self.on_connected) def on_connected(self, connection): self.connected = True self.connection = connection self.in_channel = self.connection.channel(self.on_conn_open) def on_conn_open(self, channel): self.in_channel.exchange_declare(exchange='tornado_input', type='topic') channel.queue_declare(callback=self.on_input_queue_declare, queue=self.INPUT_QUEUE_NAME) def on_input_queue_declare(self, queue): self.in_channel.queue_bind(callback=None, exchange='tornado_input', queue=self.INPUT_QUEUE_NAME, routing_key="#") def register_websocket(self, sess_id, ws): self.websockets[sess_id] = ws channel = self.create_out_channel(sess_id) def unregister_websocket(self, sess_id): del self.websockets[sess_id] if sess_id in self.out_channels: self.out_channels[sess_id].close() print("Time: %s, In: %s Out: %s" % (int(time.time() - self.start_time), self.received_message_counter, self.sent_message_counter) ) def create_out_channel(self, sess_id): def on_output_channel_creation(channel): def on_output_queue_decleration(queue): channel.basic_consume(self.on_message, queue=sess_id) self.out_channels[sess_id] = channel channel.queue_declare(callback=on_output_queue_decleration, queue=sess_id, auto_delete=True, exclusive=True) self.connection.channel(on_output_channel_creation) def redirect_incoming_message(self, sess_id, message): if not self.sent_message_counter: self.start_time = time.time() self.received_message_counter += 1 if not self.received_message_counter % 1000: print("Total Received: %s " % self.received_message_counter) self.in_channel.basic_publish(exchange='tornado_input', routing_key=sess_id, body=message) def on_message(self, channel, method, header, body): self.sent_message_counter += 1 sess_id = method.routing_key if sess_id in self.websockets: self.websockets[sess_id].write_message(body) channel.basic_ack(delivery_tag=method.delivery_tag) else: channel.basic_reject(delivery_tag=method.delivery_tag)
class PikaClient(object): INPUT_QUEUE_NAME = 'in_queue' def __init__(self, io_loop): self.io_loop = io_loop self.received_message_counter = 0 self.sent_message_counter = 0 self.start_time = -1 self.connected = False self.connecting = False self.connection = None self.in_channel = None self.out_channels = {} self.websockets = {} self.connect() def connect(self): if self.connecting: return self.connecting = True cred = pika.PlainCredentials('guest', 'guest') param = pika.ConnectionParameters( host='localhost', port=5672, virtual_host='/', credentials=cred ) self.connection = TornadoConnection(param, on_open_callback=self.on_connected) def on_connected(self, connection): self.connected = True self.connection = connection self.in_channel = self.connection.channel(self.on_conn_open) def on_conn_open(self, channel): self.in_channel.exchange_declare(exchange='tornado_input', type='topic') channel.queue_declare(callback=self.on_input_queue_declare, queue=self.INPUT_QUEUE_NAME) def on_input_queue_declare(self, queue): self.in_channel.queue_bind(callback=None, exchange='tornado_input', queue=self.INPUT_QUEUE_NAME, routing_key="#") def register_websocket(self, sess_id, ws): self.websockets[sess_id] = ws channel = self.create_out_channel(sess_id) def unregister_websocket(self, sess_id): del self.websockets[sess_id] if sess_id in self.out_channels: self.out_channels[sess_id].close() print("Time: %s, In: %s Out: %s" % (int(time.time() - self.start_time), self.received_message_counter, self.sent_message_counter) ) def create_out_channel(self, sess_id): def on_output_channel_creation(channel): def on_output_queue_decleration(queue): channel.basic_consume(self.on_message, queue=sess_id) self.out_channels[sess_id] = channel channel.queue_declare(callback=on_output_queue_decleration, queue=sess_id, auto_delete=True, exclusive=True) self.connection.channel(on_output_channel_creation) def redirect_incoming_message(self, sess_id, message): if not self.sent_message_counter: self.start_time = time.time() self.received_message_counter += 1 if not self.received_message_counter % 1000: print("Total Received: %s " % self.received_message_counter) self.in_channel.basic_publish(exchange='tornado_input', routing_key='in.%s' % sess_id, body=message) def on_message(self, channel, method, header, body): self.sent_message_counter += 1 sess_id = method.routing_key if sess_id in self.websockets: self.websockets[sess_id].write_message(body) channel.basic_ack(delivery_tag=method.delivery_tag) else: channel.basic_reject(delivery_tag=method.delivery_tag)
class Bus(Component): """ Apollo's message bus system. """ def __init__(self, core): super(Bus, self).__init__(core) self.ready = False self.busName = uuid.uuid4().hex self.parameters = ConnectionParameters( credentials=PlainCredentials( options.amqp_username, options.amqp_password ), host=options.amqp_host, port=options.amqp_port, virtual_host=options.amqp_vhost ) def go(self): self.amqp = TornadoConnection( parameters=self.parameters, on_open_callback=self.on_amqp_connection_open ) def on_amqp_connection_open(self, conn): self.amqp.channel(self.on_amqp_channel_open) def on_amqp_channel_open(self, channel): self.channel = channel self.declareQueue("inter", self.on_queue_declared) def on_queue_declared(self, something): self.channel.basic_consume( consumer_callback=self.on_inter_message, queue="inter", no_ack=False ) self.channel.queue_bind( exchange="amq.topic", queue="inter", routing_key="inter.#" ) self.ready = True logging.info("Message bus ready.") def send(self, dest, packet): """ Send a packet to a specific destination. :Parameters: * ``dest`` Destination. Prefixed with ``cross.`` for cross-server communication and ``user.`` for direct user messaging. * ``packet`` Packet to send. """ packet_dump = packet.dump() logging.debug("Sending to %s: %s" % (dest, packet_dump)) self.channel.basic_publish( exchange="amq.topic", routing_key=dest, body=packet_dump, properties=BasicProperties( delivery_mode=2 ) ) def declareQueue(self, queue, callback=None): self.channel.queue_declare( queue=queue, auto_delete=False, callback=callback ) def bindQueue(self, queue, dest, callback=None): logging.debug("Binding %s to %s" % (queue, dest)) self.channel.queue_bind( exchange="amq.topic", queue=queue, routing_key=dest, callback=callback ) def unbindQueue(self, queue, dest, callback=None): logging.debug("Unbinding %s from %s" % (queue, dest)) self.channel.queue_unbind( exchange="amq.topic", queue=queue, routing_key=dest, callback=callback ) def deleteQueue(self, queue): logging.debug("Deleting queue %s" % queue) self.channel.queue_delete(queue=queue) def on_inter_message(self, channel, method, header, body): """ Process an "inter" message. """ sess = meta.Session() channel.basic_ack(delivery_tag=method.delivery_tag) prefixparts = method.routing_key.split(".") packet = deserializePacket(body) packet._origin = ORIGIN_INTER if prefixparts[1] == "global": for user in sess.query(User): packet.dispatch(self.core, FakeSession(user.id)) return ident = uuid.UUID(hex=prefixparts[2]) if prefixparts[1] == "User": user = sess.query(User).get(ident) packet.dispatch(self.core, FakeSession(user.id)) elif prefixparts[1] == "Tile": for user in sess.query(User).filter(User.location_id == ident): packet.dispatch(self.core, FakeSession(user.id)) elif prefixparts[1] == "Group": for user in sess.query(User).filter(User.group_id == ident): packet.dispatch(self.core, FakeSession(user.id)) elif prefixparts[1] == "Realm": for user in sess.query(User).filter(User.location_id == Tile.id).filter(Tile.chunk_id == Chunk.id).filter(Chunk.realm_id == Realm.id).filter(Realm.id == ident): packet.dispatch(self.core, FakeSession(user.id)) def broadcastEx(self, packet): self.send("ex.global", packet) def broadcastInter(self, packet): self.send("inter.global", packet) def globalBind(self, session, callback=None): self.bindQueue("ex:%s" % session.id, "ex.global", callback) def globalUnbind(self, session, callback=None): self.unbindQueue("ex:%s" % session.id, "ex.global", callback)
class RabbitMQClient(BaseClient): """ Base RabbitMQ asynchronous adapter. """ def __init__(self, host=settings.HOST, port=settings.PORT, virtual_host=settings.VIRTUAL_HOST, username=settings.USERNAME, password=settings.PASSWORD, exchange_name='direct', exchange_type='direct'): """ Initialize RabbitMQ client with passed configuration or get parameters from django settings module. :param host: RabbitMQ host :type host: :class:`str` :param port: RabbitMQ port :type port: :class:`int` :param virtual_host: RabbitMQ virtual host :type virtual_host: :class:`str` :param username: RabbitMQ user :type username: :class:`str` :param password: RabbitMQ user's password :type password: :class:`str` :param exchange_name: Exchange name, see RabbitMQ docs for more info. :type exchange_name: :class:`str` :param exchange_type: Exchange type, see RabbitMQ docs for more info. :type exchange_type: :class:`str` """ self.host = host self.port = int(port) self.virtual_host = virtual_host self.username = username self.password = password self.exchange_name = exchange_name self.exchange_type = exchange_type self.connected = False self.connecting = False self.reconnecting = False self.closing = False self.connection = None self.channel = None ############################################################################ # CONNECTION METHODS ############################################################################ def connect(self): """ Asynchronous connection. Connects to RabbitMQ server and establish non-blocking connection. After connection is established on_connected callback will be executed which will try to reconnect if this function failed. """ if self.connecting and not self.reconnecting: return self.connecting = True credentials = PlainCredentials(self.username, self.password) param = ConnectionParameters(host=self.host, port=self.port, virtual_host=self.virtual_host, credentials=credentials) self.connection = TornadoConnection(param, on_open_callback=self.on_connected) self.connection.add_on_close_callback(self.on_close) def reconnect(self): """ Reconnect method. Basically you don't need to call this method manually. """ self.reconnecting = True self.connect() def disconnect(self): """ Disconnect method. Call this method after you end with messaging. """ self.closing = True self.connection.close() ############################################################################ # CALLBACKS ############################################################################ def on_connected(self, connection): """ Callback on connection. Reconnect if connection dropped. Otherwise it will open channel and on_channel_open callback will be executed. :param connection: RabbitMQ connection. :type connection: :class:`TornadoConnection` """ self.connected = True self.connection = connection self.connection.channel(self.on_channel_open) def on_channel_open(self, channel, callback=lambda frame: None): """ Callback on channel open. It will declare exchanges for messaging. See RabbitMQ docs for more information. :param channel: Opened channel. :type channel: :class:`Channel` :param callback: Callback that will be executed after channel open. :type callback: callable object """ self.channel = channel self.channel.exchange_declare(exchange=self.exchange_name, type=self.exchange_type, auto_delete=False, durable=True, callback=callback) def on_close(self, connection, *args, **kwargs): """ On close callback. You don't need to manually call this method. :param connection: Established connection. :type connection: :class:`TornadoConnection` :param args: Internal args :param kwargs: Internal kwargs """ self.channel = None if not self.closing: self.reconnect() self.connection.add_timeout(5, self.reconnect) else: connection.close()
class TAMQPTornadoServer(object): def __init__(self, processor, iprot_factory, oprot_factory=None, *args, **kwargs): self._processor = processor self._iprot_factory = iprot_factory self._oprot_factory = oprot_factory if oprot_factory else iprot_factory self._connection = None self._channel = None self._url = kwargs.get('url', constant.DEFAULT_URL) self._exchange_name = kwargs.get('exchange_name', constant.EXCHANGE_NAME) self._routing_key = kwargs.get('routing_key', constant.ROUTING_KEY) self._queue_name = kwargs.get('queue_name', constant.QUEUE_NAME) self._prefetch = kwargs.get('prefetch', 0) self._consumer_tag = kwargs.get('consumer_tag') self._error_logger = kwargs.get('error_logger') self._starting = False def start(self): if self._starting: return logger.info("Starting the connection") self._starting = True self._connection = TornadoConnection(pika.URLParameters(self._url), self.on_connection_open, self.on_connection_error, self.on_connection_error) @gen.coroutine def on_connection_error(self, *args, **kwargs): self._starting = False logger.info("Connection failed") yield gen.sleep(constant.TIMEOUT_RECONNECT) self.start() def on_connection_open(self, _): logger.info("Connection started") logger.info("Starting the channel") self._connection.channel(on_open_callback=self.on_channel_open) def on_channel_open(self, channel): logger.info("Channel started") logger.info( "Declaring exchange : {} of type {}".format( self._exchange_name, constant.EXCHANGE_TYPE)) self._channel = channel self._channel.add_on_close_callback(self.on_channel_close) self._channel.basic_qos(prefetch_count=self._prefetch) try: self._channel.exchange_declare(self.on_exchange_declared, self._exchange_name, constant.EXCHANGE_TYPE, False) except: self._channel.exchange_declare(self.on_exchange_declared, self._exchange_name, constant.EXCHANGE_TYPE, True) @gen.coroutine def on_channel_close(self, *args): logger.info("Channel closed") yield gen.sleep(constant.TIMEOUT_RECONNECT) if not self._starting: if self._connection and self._connection.is_open: self._connection.channel(on_open_callback=self.on_channel_open) else: self.start() def on_exchange_declared(self, _): logger.info( "Exchange declared : {} of type {}".format( self._exchange_name, constant.EXCHANGE_TYPE)) logger.info("Declaring queue: {}".format(self._queue_name)) self._channel.queue_declare(self.on_queue_declared, self._queue_name) def on_queue_declared(self, _queue): logger.info("Queue declared : {}".format(self._queue_name)) self._channel.queue_bind(self.on_binded, self._queue_name, self._exchange_name, self._routing_key) def on_binded(self, _bind): logger.info( "Queue {} binded to the {} exchange with the {} routing key".format( self._queue_name, self._exchange_name, self._routing_key)) self._channel.basic_consume(self.on_message, self._queue_name, consumer_tag=self._consumer_tag) self._starting = False @gen.coroutine def on_message(self, _channel, method, properties, body): iprot = self._iprot_factory.getProtocol(TMemoryBuffer(body)) iprot_dup = self._iprot_factory.getProtocol(TMemoryBuffer(body)) try: type = iprot_dup.readMessageBegin()[1] except: self._error_logger.capture_exception() self._channel.basic_ack(delivery_tag=method.delivery_tag) raise gen.Return() if type == TMessageType.ONEWAY: try: yield self._processor.process(iprot, None) except Exception as e: self._error_logger.capture_exception() logger.error(e, exc_info=True) self._channel.basic_ack(delivery_tag=method.delivery_tag) else: trans = TAMQPTornadoTransport(channel=self._channel, properties=properties, method=method) oprot = self._oprot_factory.getProtocol(trans) try: yield self._processor.process(iprot, oprot) except Exception as e: self._error_logger.capture_exception() logger.error(e, exc_info=True) self._channel.basic_ack(delivery_tag=method.delivery_tag)
class TAMQPTornadoServer(object): def __init__(self, processor, iprot_factory, oprot_factory=None, *args, **kwargs): self._processor = processor self._iprot_factory = iprot_factory self._oprot_factory = oprot_factory if oprot_factory else iprot_factory self._connection = None self._channel = None self._url = kwargs.get('url', constant.DEFAULT_URL) self._exchange_name = kwargs.get('exchange_name', constant.EXCHANGE_NAME) self._routing_key = kwargs.get('routing_key', constant.ROUTING_KEY) self._queue_name = kwargs.get('queue_name', constant.QUEUE_NAME) self._prefetch = kwargs.get('prefetch', 0) def start(self): logger.info("Starting the connection") self._connection = TornadoConnection(pika.URLParameters(self._url), self.on_connection_open, self.on_connection_error, self.on_connection_error) @gen.coroutine def on_connection_error(self, *args, **kwargs): logger.info("Connection failed") yield gen.sleep(constant.TIMEOUT_RECONNECT) self.start() def on_connection_open(self, _): logger.info("Connection started") logger.info("Starting the channel") self._connection.channel(on_open_callback=self.on_channel_open) def on_channel_open(self, channel): logger.info("Channel started") logger.info( "Declaring exchange : {} of type {}".format( self._exchange_name, constant.EXCHANGE_TYPE)) self._channel = channel self._channel.basic_qos(prefetch_count=self._prefetch) try: self._channel.exchange_declare(self.on_exchange_declared, self._exchange_name, constant.EXCHANGE_TYPE, False) except: self._channel.exchange_declare(self.on_exchange_declared, self._exchange_name, constant.EXCHANGE_TYPE, True) def on_exchange_declared(self, _): logger.info( "Exchange declared : {} of type {}".format( self._exchange_name, constant.EXCHANGE_TYPE)) logger.info("Declaring queue: {}".format(self._queue_name)) self._channel.queue_declare(self.on_queue_declared, self._queue_name) def on_queue_declared(self, _queue): logger.info("Queue declared : {}".format(self._queue_name)) self._channel.queue_bind(self.on_binded, self._queue_name, self._exchange_name, self._routing_key) def on_binded(self, _bind): logger.info( "Queue {} binded to the {} exchange with the {} routing key".format( self._queue_name, self._exchange_name, self._routing_key)) self._channel.basic_consume(self.on_message, self._queue_name) @gen.coroutine def on_message(self, _channel, method, properties, body): iprot = self._iprot_factory.getProtocol(TMemoryBuffer(body)) iprot_dup = self._iprot_factory.getProtocol(TMemoryBuffer(body)) try: type = iprot_dup.readMessageBegin()[1] except: self._channel.basic_ack(delivery_tag=method.delivery_tag) raise gen.Return() if type == TMessageType.ONEWAY: try: yield self._processor.process(iprot, None) except Exception as e: logger.error(e, exc_info=True) self._channel.basic_ack(delivery_tag=method.delivery_tag) else: trans = TAMQPTornadoTransport(channel=self._channel, properties=properties, method=method) oprot = self._oprot_factory.getProtocol(trans) try: yield self._processor.process(iprot, oprot) except Exception as e: logger.error(e, exc_info=True) self._channel.basic_ack(delivery_tag=method.delivery_tag)
class TAMQPTornadoTransport(TTransportBase): def __init__(self, channel=None, exchange_name=constant.EXCHANGE_NAME, routing_key=constant.ROUTING_KEY, properties=None, method=None, io_loop=None, **kwargs): self._channel = channel self._exchange_name = exchange_name self._routing_key = routing_key self._wbuf = StringIO() self._properties = properties self._method = method self._url = kwargs.get('url', constant.DEFAULT_URL) self._reply_to = None self._lock = _Lock() self._callback = None self._connection = None self._consumer_tag = None self._consumer_name = kwargs.get('consumer_tag') self._message_expiration = kwargs.get('message_expiration') self._reply_queue_name = kwargs.get('reply_queue_name', '') self._error_logger = kwargs.get('error_logger') self._callback_queue = Queue() self.io_loop = io_loop or ioloop.IOLoop.instance() self._closing = False self._starting = False @gen.coroutine def assign_queue(self): logger.info("Openning callback queue") result = yield gen.Task(self._channel.queue_declare, queue=self._reply_queue_name, exclusive=True, auto_delete=True) logger.info("Callback queue openned") if self._reply_queue_name == '': self._reply_to = result.method.queue else: self._reply_to = self._reply_queue_name if self._lock.acquired(): self._lock.release() self._consumer_tag = self._channel.basic_consume( self.on_reply_message, self._reply_to, consumer_tag=self._consumer_name) self._starting = False if self._callback: self._callback() def on_reply_message(self, _channel, method, properties, body): if method: self._channel.basic_ack(delivery_tag=method.delivery_tag) self._callback_queue.put(body) def on_connection_open(self, _connection): logger.info("Openning channel") self._connection.channel(on_open_callback=self.on_channel_open) def open(self, callback=None): if self._starting: if callback: callback() return self._starting = True logger.info("Openning AMQP transport") if self._channel is not None and self._channel.is_open: logger.info("Already set") if callback: callback() else: logger.info("Openning connection") self._callback = callback self._lock.acquire() self._connection = TornadoConnection(pika.URLParameters(self._url), self.on_connection_open, self.on_connection_error, self.on_connection_error) @gen.coroutine def on_connection_error(self, *args, **kwargs): self._starting = False logger.info("Connection failed") yield gen.sleep(constant.TIMEOUT_RECONNECT) self.open(self._callback) @gen.coroutine def readFrame(self): result = yield self._callback_queue.get() if issubclass(result.__class__, Exception): raise result raise gen.Return(result) @gen.coroutine def on_channel_close(self, *args): logger.info("Channel closed") yield gen.sleep(constant.TIMEOUT_RECONNECT) self._callback_queue.put( thrift.transport.TTransport.TTransportException( message='channel closed')) if not self._closing and not self._starting: if self._connection and self._connection.is_open: self._lock.acquire() self._connection.channel(on_open_callback=self.on_channel_open) else: self.open() def on_channel_open(self, channel): logger.info("Channel openned") self._channel = channel self._channel.add_on_close_callback(self.on_channel_close) self.assign_queue() def close(self): self._closing = True if self._channel: if self._consumer_tag: self._channel.basic_cancel(consumer_tag=self._consumer_tag, nowait=True) self._channel.close() def isOpen(self): return self._channel is not None def read(self, _): assert False, "wrong stuff" @gen.coroutine def write(self, buf): with (yield self._lock.acquire()): if type(buf) is unicode: self._wbuf.write(buf.encode("utf-8")) else: self._wbuf.write(buf) @gen.coroutine def flush(self, recovered=False): try: yield self.flush_once() self._wbuf = StringIO() except Exception as e: yield gen.sleep(constant.TIMEOUT_RECONNECT) if not recovered: yield self.flush(True) else: if self._error_logger: self._error_logger.capture_exception() logger.info(e, exc_info=True) self._wbuf = StringIO() raise thrift.transport.TTransport.TTransportException( message=str(e)) @gen.coroutine def flush_once(self): if self._properties is not None: props = pika.BasicProperties( correlation_id=self._properties.correlation_id) result = self._wbuf.getvalue() if type(result) is unicode: result = result.encode("utf-8") self._channel.basic_publish(exchange='', routing_key=self._properties.reply_to, properties=props, body=result) if self._method is not None: self._channel.basic_ack(delivery_tag=self._method.delivery_tag) else: with (yield self._lock.acquire()): props = pika.BasicProperties(correlation_id=str(uuid.uuid4()), reply_to=self._reply_to) if self._message_expiration: props.expiration = str(self._message_expiration * 1000) result = self._wbuf.getvalue() if type(result) is unicode: result = result.encode("utf-8") self._channel.basic_publish(exchange=self._exchange_name, routing_key=self._routing_key, properties=props, body=result) self._wbuf = StringIO()