def start(self):
if self._starting:
return
logger.info("Starting the connection")
self._starting = True
self._connection = TornadoConnection(pika.URLParameters(self._url),
self.on_connection_open,
self.on_connection_error,
self.on_connection_error)
def connect(self):
if self.connecting:
LOGGER.info('Already connected to RabbitMQ server')
return
LOGGER.info('Opening a connection to RabbitMQ server')
self.connecting = True
self.connection = TornadoConnection(
parameters=settings.pika_parameters,
on_open_callback=self.on_connection_open)
self.connection.add_on_close_callback(self.on_closed)
def connect(self):
if self.is_connecting:
logger.info("PikaConnection: Already connecting to RabbitMQ")
return
logger.info("PikaConnection: Connecting to RabbitMQ")
self.connecting = True
self.connection = TornadoConnection(
pika.URLParameters(self.url),
on_open_callback=self.on_connected)
self.connection.add_on_close_callback(self.on_closed)
class BaseWorker(object):
def __init__(self, ioloop, app):
self.io_loop = ioloop
self.app = app
self.connected = False
self.connecting = False
self.connection = None
self.channel = None
self.consumer_tag = None
self.event_listeners = set([])
def connect(self):
if self.connecting:
return
self.connecting = True
param = pika.ConnectionParameters(host=QUEUE_HOST)
self.connection = TornadoConnection(
param, on_open_callback=self.on_connected)
self.connection.add_on_close_callback(self.on_closed)
def on_connected(self, connection):
self.connected = True
self.connection = connection
self.connection.channel(self.on_channel_open)
def on_channel_open(self, channel):
self.channel = channel
channel.queue_declare(
self.on_queue_declareok, queue=QUEUE_NAME, durable=True)
def on_queue_declareok(self, method_frame):
self.consumer_tag = self.channel.basic_consume(self.on_message,
QUEUE_NAME)
def on_closed(self, connection):
self.io_loop.stop()
def on_message(self, channel, method, header, body):
logger.info('Message received: %s' % body)
self.process(channel, method, header, body)
self.channel.basic_ack(delivery_tag=method.delivery_tag)
def process(self, channel, method, header, body):
raise NotImplementedError
def connect(self):
"""
Creates connection to RabbitMQ server
"""
if self.connecting:
log.info('PikaClient: Already connecting to RabbitMQ')
return
log.info('PikaClient: Connecting to RabbitMQ')
self.connecting = True
self.connection = TornadoConnection(NON_BLOCKING_MQ_PARAMS,
stop_ioloop_on_close=False,
custom_ioloop=self.io_loop,
on_open_callback=self.on_connected)
def connect(self):
if self.connecting:
return
self.connecting = True
cred = pika.PlainCredentials('guest', 'guest')
param = pika.ConnectionParameters(
host='localhost',
port=5672,
virtual_host='/',
credentials=cred
)
self.connection = TornadoConnection(param,
on_open_callback=self.on_connected)
def reconnect(self, callback):
logger.info('Attempting to acquire the connect_lock.')
if not self.connect_lock.acquire(False):
logger.info('AsyncRabbitClient.reconnect is already '
'attempting to connect (the connect_lock '
'could not be acquired).')
callback()
return
try:
logger.info('AsyncRabbitClient.reconnect attempting to '
'connect to host: %s' % self.host,
extra={'host': self.host})
params = pika.ConnectionParameters(host=self.host)
key = str(uuid.uuid4())
TornadoConnection(parameters=params,
custom_ioloop=self.io_loop,
on_open_callback=(yield gen.Callback(key)))
logger.info('Waiting for TornadoConnection to return control '
'via on_open_callback.')
self.connection = yield gen.Wait(key)
logger.info('Control has been returned.')
logger.info('Opening a channel on the connection.')
key = str(uuid.uuid4())
self.connection.channel(on_open_callback=
(yield gen.Callback(key)))
logger.info('Waiting for connection.channel to return control '
'via on_open_callback.')
self.channel = yield gen.Wait(key)
logger.info('Control has been returned.')
logger.info('Adding callbacks to warn us when the connection '
'has been closed and when backpressure is being '
'applied.')
self.connection.add_on_close_callback(self.on_connection_closed)
self.connection.add_backpressure_callback(self.on_backpressure)
self.channel.add_on_close_callback(self.on_channel_closed)
logger.info('Adding callbacks that are waiting for an open '
'connection to the tornado queue.')
while self.connection_open_callbacks:
cb = self.connection_open_callbacks.pop()
self.io_loop.add_callback(cb)
logger.info('Done adding callbacks.')
except Exception as e:
logger.critical('An unknown exception was raised when trying '
'to open a connection to rabbit: %s' %
unicode(e))
raise
finally:
logger.info('Releasing the connect lock.')
self.connect_lock.release()
callback()
class BaseAMQPConnection(BaseConnection):
""" An object which does an actual job of (re-)connecting to the the AMQP broker.
Concrete subclasses implement either listening or publishing features.
"""
def __init__(self, conn_params, item_name, properties=None):
super(BaseAMQPConnection, self).__init__()
self.conn_params = conn_params
self.item_name = item_name
self.properties = properties
self.conn = None
self.channel = None
self.reconnect_exceptions = (TypeError, EnvironmentError)
def _start(self):
self.conn = TornadoConnection(self.conn_params, self._on_connected)
self.conn.ioloop.start()
def _close(self):
""" Actually close the connection.
"""
if self.conn:
self.conn.close()
def _conn_info(self):
return '{0}:{1}{2} ({3})'.format(self.conn_params.host,
self.conn_params.port, self.conn_params.virtual_host, self.item_name)
def _on_channel_open(self, channel):
self.channel = channel
msg = 'Got a channel for {0}'.format(self._conn_info())
self.logger.debug(msg)
def _keep_connecting(self, e):
# We need to catch TypeError because pika will sometimes erroneously raise
# it in self._start's self.conn.ioloop.start().
# Otherwise, it may one of the network errors we are hopefully able to recover from.
return isinstance(e, TypeError) or (isinstance(e, EnvironmentError)
and e.errno in self.reconnect_error_numbers)
def _on_connected(self, conn):
""" Invoked after establishing a successful connection to an AMQP broker.
Will report a diagnostic message regarding how many attempts there were
and how long it took if the connection hasn't been established straightaway.
"""
super(BaseAMQPConnection, self)._on_connected()
conn.channel(self._on_channel_open)
def connect(self):
if self.connecting:
return
self.connecting = True
param = pika.ConnectionParameters(host=QUEUE_HOST)
self.connection = TornadoConnection(
param, on_open_callback=self.on_connected)
self.connection.add_on_close_callback(self.on_closed)
def open(self, callback=None):
if self._starting:
if callback:
callback()
return
self._starting = True
logger.info("Openning AMQP transport")
if self._channel is not None and self._channel.is_open:
logger.info("Already set")
if callback:
callback()
else:
logger.info("Openning connection")
self._callback = callback
self._lock.acquire()
self._connection = TornadoConnection(pika.URLParameters(self._url),
self.on_connection_open,
self.on_connection_error,
self.on_connection_error)
def open(self, callback=None):
logger.info("Openning AMQP transport")
if self._channel is not None:
logger.info("Already set")
callback()
else:
logger.info("Openning connection")
self._callback = callback
self._connection = TornadoConnection(pika.URLParameters(self._url),
self.on_connection_open,
self.on_connection_error,
self.on_connection_error)
self._lock.acquire()
def connect(self):
if self.connecting:
return
self.connecting = True
cred = pika.PlainCredentials('guest', 'guest')
param = pika.ConnectionParameters(
host='localhost',
port=5672,
virtual_host='/',
credentials=cred
)
self.connection = TornadoConnection(param,
on_open_callback=self.on_connected)
def connect(cls, url):
f = Future()
def on_open(conn):
f.set_result(cls(conn))
def on_open_error(conn, err):
f.set_exception(AMQPConnectionError(err))
def on_close(conn):
LOGGER.debug('connection closed: %s', conn)
TornadoConnection(
URLParameters(url),
on_open_callback=on_open,
on_open_error_callback=on_open_error,
on_close_callback=on_close,
)
return f
def connect(self):
"""
Asynchronous connection. Connects to RabbitMQ server and establish
non-blocking connection. After connection is established on_connected
callback will be executed which will try to reconnect if this function
failed.
"""
if self.connecting and not self.reconnecting:
return
self.connecting = True
credentials = PlainCredentials(self.username, self.password)
param = ConnectionParameters(host=self.host,
port=self.port,
virtual_host=self.virtual_host,
credentials=credentials)
self.connection = TornadoConnection(param,
on_open_callback=self.on_connected)
self.connection.add_on_close_callback(self.on_close)
class TAMQPTornadoServer(object):
def __init__(self, processor, iprot_factory, oprot_factory=None, *args,
**kwargs):
self._processor = processor
self._iprot_factory = iprot_factory
self._oprot_factory = oprot_factory if oprot_factory else iprot_factory
self._connection = None
self._channel = None
self._url = kwargs.get('url', constant.DEFAULT_URL)
self._exchange_name = kwargs.get('exchange_name',
constant.EXCHANGE_NAME)
self._routing_key = kwargs.get('routing_key', constant.ROUTING_KEY)
self._queue_name = kwargs.get('queue_name', constant.QUEUE_NAME)
self._prefetch = kwargs.get('prefetch', 0)
self._consumer_tag = kwargs.get('consumer_tag')
self._error_logger = kwargs.get('error_logger')
self._starting = False
def start(self):
if self._starting:
return
logger.info("Starting the connection")
self._starting = True
self._connection = TornadoConnection(pika.URLParameters(self._url),
self.on_connection_open,
self.on_connection_error,
self.on_connection_error)
@gen.coroutine
def on_connection_error(self, *args, **kwargs):
self._starting = False
logger.info("Connection failed")
yield gen.sleep(constant.TIMEOUT_RECONNECT)
self.start()
def on_connection_open(self, _):
logger.info("Connection started")
logger.info("Starting the channel")
self._connection.channel(on_open_callback=self.on_channel_open)
def on_channel_open(self, channel):
logger.info("Channel started")
logger.info(
"Declaring exchange : {} of type {}".format(
self._exchange_name, constant.EXCHANGE_TYPE))
self._channel = channel
self._channel.add_on_close_callback(self.on_channel_close)
self._channel.basic_qos(prefetch_count=self._prefetch)
try:
self._channel.exchange_declare(self.on_exchange_declared,
self._exchange_name,
constant.EXCHANGE_TYPE, False)
except:
self._channel.exchange_declare(self.on_exchange_declared,
self._exchange_name,
constant.EXCHANGE_TYPE, True)
@gen.coroutine
def on_channel_close(self, *args):
logger.info("Channel closed")
yield gen.sleep(constant.TIMEOUT_RECONNECT)
if not self._starting:
if self._connection and self._connection.is_open:
self._connection.channel(on_open_callback=self.on_channel_open)
else:
self.start()
def on_exchange_declared(self, _):
logger.info(
"Exchange declared : {} of type {}".format(
self._exchange_name, constant.EXCHANGE_TYPE))
logger.info("Declaring queue: {}".format(self._queue_name))
self._channel.queue_declare(self.on_queue_declared, self._queue_name)
def on_queue_declared(self, _queue):
logger.info("Queue declared : {}".format(self._queue_name))
self._channel.queue_bind(self.on_binded, self._queue_name,
self._exchange_name, self._routing_key)
def on_binded(self, _bind):
logger.info(
"Queue {} binded to the {} exchange with the {} routing key".format(
self._queue_name, self._exchange_name, self._routing_key))
self._channel.basic_consume(self.on_message, self._queue_name,
consumer_tag=self._consumer_tag)
self._starting = False
@gen.coroutine
def on_message(self, _channel, method, properties, body):
iprot = self._iprot_factory.getProtocol(TMemoryBuffer(body))
iprot_dup = self._iprot_factory.getProtocol(TMemoryBuffer(body))
try:
type = iprot_dup.readMessageBegin()[1]
except:
self._error_logger.capture_exception()
self._channel.basic_ack(delivery_tag=method.delivery_tag)
raise gen.Return()
if type == TMessageType.ONEWAY:
try:
yield self._processor.process(iprot, None)
except Exception as e:
self._error_logger.capture_exception()
logger.error(e, exc_info=True)
self._channel.basic_ack(delivery_tag=method.delivery_tag)
else:
trans = TAMQPTornadoTransport(channel=self._channel,
properties=properties,
method=method)
oprot = self._oprot_factory.getProtocol(trans)
try:
yield self._processor.process(iprot, oprot)
except Exception as e:
self._error_logger.capture_exception()
logger.error(e, exc_info=True)
self._channel.basic_ack(delivery_tag=method.delivery_tag)
class PikaClient():
def __init__(self, ioloop):
self.ioloop = ioloop
self.connected = False
self.connecting = False
self.connection = None
self.channel = None
self.event_listeners = set([])
def connect(self):
if self.connecting:
LOGGER.info('Already connected to RabbitMQ server')
return
LOGGER.info('Opening a connection to RabbitMQ server')
self.connecting = True
self.connection = TornadoConnection(
parameters=settings.pika_parameters,
on_open_callback=self.on_connection_open)
self.connection.add_on_close_callback(self.on_closed)
def on_connection_open(self, connection):
LOGGER.info('Opening a connection to RabbitMQ server completed')
self.connected = True
self.connection = connection
self.open_channel()
def open_channel(self):
LOGGER.info('Opening a channel')
self.connection.channel(on_open_callback=self.on_channel_open)
def on_channel_open(self, channel):
LOGGER.info('Opening a channel completed')
self.channel = channel
self.declare_exchange()
# def on_channel_open(self, channel):
# LOGGER.info('PikaClient: Channel open, Declaring exchange')
# self.channel = channel
# # declare exchanges, which in turn, declare
# # queues, and bind exchange to queues
def declare_exchange(self):
LOGGER.info('Declaring an exchange')
self.channel.exchange_declare(
exchange=settings.RABBITMQ_NEWSFEED_ENTRY_EXCHANGE_NAME,
exchange_type=settings.RABBITMQ_NEWSFEED_ENTRY_EXCHANGE_TYPE,
passive=settings.RABBITMQ_NEWSFEED_ENTRY_EXCHANGE_PASSIVE,
durable=settings.RABBITMQ_NEWSFEED_ENTRY_EXCHANGE_DURABLE,
auto_delete=settings.RABBITMQ_NEWSFEED_ENTRY_EXCHANGE_AUTO_DELETE,
internal=settings.RABBITMQ_NEWSFEED_ENTRY_EXCHANGE_INTERNAL,
nowait=settings.RABBITMQ_NEWSFEED_ENTRY_EXCHANGE_NOWAIT,
arguments=None,
callback=self.on_exchange_declared)
def on_exchange_declared(self, unused_frame):
LOGGER.info('Declaring an exchange completed')
self.declare_queue()
def declare_queue(self):
LOGGER.info('Declaring a queue')
self.channel.queue_declare(
callback=self.on_queue_declared,
queue=settings.RABBITMQ_NEWSFEED_ENTRY_QUEUE_NAME,
passive=settings.RABBITMQ_NEWSFEED_ENTRY_QUEUE_PASSIVE,
durable=settings.RABBITMQ_NEWSFEED_ENTRY_QUEUE_DURABLE,
exclusive=settings.RABBITMQ_NEWSFEED_ENTRY_QUEUE_EXCLUSIVE,
auto_delete=settings.RABBITMQ_NEWSFEED_ENTRY_QUEUE_AUTO_DELETE,
nowait=settings.RABBITMQ_NEWSFEED_ENTRY_QUEUE_NOWAIT,
arguments=None)
def on_queue_declared(self, method_frame):
LOGGER.info('Declaring a queue completed')
self.bind_queue()
def bind_queue(self):
LOGGER.info('Binding a queue')
self.channel.queue_bind(
callback=self.on_queue_binded,
queue=settings.RABBITMQ_NEWSFEED_ENTRY_QUEUE_NAME,
exchange=settings.RABBITMQ_NEWSFEED_ENTRY_EXCHANGE_NAME,
routing_key=settings.RABBITMQ_NEWSFEED_ENTRY_ROUTING_KEY)
def on_queue_binded(self, frame):
LOGGER.info('Binding a queue completed')
self.channel.basic_consume(
consumer_callback=self.on_message,
queue=settings.RABBITMQ_NEWSFEED_ENTRY_QUEUE_NAME,
no_ack=
False, # Set to True means tell the broker to not expect a response
exclusive=
False, # Set to True means don't allow other consumers on the queue
consumer_tag=None) # Specify your own consumer tag
def on_closed(self, connection):
LOGGER.info('Connection to RabbitMQ server closed')
self.ioloop.stop()
#def on_message(self, channel, method, header, body):
def on_message(self, channel, basic_deliver, properties, body):
LOGGER.info('Received message # %s from %s',
basic_deliver.delivery_tag, properties.app_id)
self.ack_message(basic_deliver.delivery_tag)
self.notify_listeners(body)
def ack_message(self, delivery_tag):
"""Acknowledge the message delivery from RabbitMQ by sending a
Basic.Ack RPC method for the delivery tag.
:param int delivery_tag: The delivery tag from the Basic.Deliver frame
"""
LOGGER.info('Acknowledging message %s', delivery_tag)
self.channel.basic_ack(delivery_tag)
def notify_listeners(self, event_obj):
# here we assume the message the sourcing app
# post to the message queue is in JSON format
#event_json = json.dumps(event_obj)
event_json = event_obj
for listener in self.event_listeners:
listener.write_message(event_json)
LOGGER.info('Notified %s' % repr(listener))
def add_event_listener(self, listener):
self.event_listeners.add(listener)
LOGGER.info('Added listener %s' % repr(listener))
def remove_event_listener(self, listener):
try:
self.event_listeners.remove(listener)
LOGGER.info('Removed listener %s' % repr(listener))
except KeyError:
pass
class PikaConnection(object):
QUEUE = 'test'
EXCHANGE = ''
PUBLISH_INTERVAL = 1
def __init__(self, io_loop, amqp_url):
self.io_loop = io_loop
self.url = amqp_url
self.is_connected = False
self.is_connecting = False
self.connection = None
self.channel = None
self.listeners = set([])
def connect(self):
if self.is_connecting:
logger.info("PikaConnection: Already connecting to RabbitMQ")
return
logger.info("PikaConnection: Connecting to RabbitMQ")
self.connecting = True
self.connection = TornadoConnection(
pika.URLParameters(self.url),
on_open_callback=self.on_connected)
self.connection.add_on_close_callback(self.on_closed)
def on_connected(self, connection):
logger.info("PikaConnection: connected to RabbitMQ")
self.connected = True
self.connection = connection
self.connection.channel(self.on_channel_open)
def on_closed(self, connection):
logger.info("PikaConnection: connection closed")
self.io_loop.stop()
def on_channel_open(self, channel):
self.channel = channel
self.channel.add_on_close_callback(self.on_channel_closed)
self.channel.queue_declare(queue=self.QUEUE,
callback=self.on_queue_declared)
def on_channel_closed(self, channel, reply_code, reply_text):
self.connection.close()
def on_queue_declared(self, frame):
print "subscribe frame:", frame
self.channel.basic_consume(self.on_message, self.QUEUE)
def on_message(self, channel, method, header, body):
logger.info(body)
for listener in self.listeners:
listener.emit(body)
def add_event_listener(self, listener):
self.listeners.add(listener)
def publish_message(self, *args, **kwargs):
if 'message' in kwargs:
self.channel.basic_publish(exchange=self.EXCHANGE,
routing_key=self.QUEUE,
body=kwargs['message'])
def _start(self):
self.conn = TornadoConnection(self.conn_params, self._on_connected)
self.conn.ioloop.start()
class TAMQPTornadoServer(object):
def __init__(self, processor, iprot_factory, oprot_factory=None, *args,
**kwargs):
self._processor = processor
self._iprot_factory = iprot_factory
self._oprot_factory = oprot_factory if oprot_factory else iprot_factory
self._connection = None
self._channel = None
self._url = kwargs.get('url', constant.DEFAULT_URL)
self._exchange_name = kwargs.get('exchange_name',
constant.EXCHANGE_NAME)
self._routing_key = kwargs.get('routing_key', constant.ROUTING_KEY)
self._queue_name = kwargs.get('queue_name', constant.QUEUE_NAME)
self._prefetch = kwargs.get('prefetch', 0)
def start(self):
logger.info("Starting the connection")
self._connection = TornadoConnection(pika.URLParameters(self._url),
self.on_connection_open,
self.on_connection_error,
self.on_connection_error)
@gen.coroutine
def on_connection_error(self, *args, **kwargs):
logger.info("Connection failed")
yield gen.sleep(constant.TIMEOUT_RECONNECT)
self.start()
def on_connection_open(self, _):
logger.info("Connection started")
logger.info("Starting the channel")
self._connection.channel(on_open_callback=self.on_channel_open)
def on_channel_open(self, channel):
logger.info("Channel started")
logger.info(
"Declaring exchange : {} of type {}".format(
self._exchange_name, constant.EXCHANGE_TYPE))
self._channel = channel
self._channel.basic_qos(prefetch_count=self._prefetch)
try:
self._channel.exchange_declare(self.on_exchange_declared,
self._exchange_name,
constant.EXCHANGE_TYPE, False)
except:
self._channel.exchange_declare(self.on_exchange_declared,
self._exchange_name,
constant.EXCHANGE_TYPE, True)
def on_exchange_declared(self, _):
logger.info(
"Exchange declared : {} of type {}".format(
self._exchange_name, constant.EXCHANGE_TYPE))
logger.info("Declaring queue: {}".format(self._queue_name))
self._channel.queue_declare(self.on_queue_declared, self._queue_name)
def on_queue_declared(self, _queue):
logger.info("Queue declared : {}".format(self._queue_name))
self._channel.queue_bind(self.on_binded, self._queue_name,
self._exchange_name, self._routing_key)
def on_binded(self, _bind):
logger.info(
"Queue {} binded to the {} exchange with the {} routing key".format(
self._queue_name, self._exchange_name, self._routing_key))
self._channel.basic_consume(self.on_message, self._queue_name)
@gen.coroutine
def on_message(self, _channel, method, properties, body):
iprot = self._iprot_factory.getProtocol(TMemoryBuffer(body))
iprot_dup = self._iprot_factory.getProtocol(TMemoryBuffer(body))
try:
type = iprot_dup.readMessageBegin()[1]
except:
self._channel.basic_ack(delivery_tag=method.delivery_tag)
raise gen.Return()
if type == TMessageType.ONEWAY:
try:
yield self._processor.process(iprot, None)
except Exception as e:
logger.error(e, exc_info=True)
self._channel.basic_ack(delivery_tag=method.delivery_tag)
else:
trans = TAMQPTornadoTransport(channel=self._channel,
properties=properties,
method=method)
oprot = self._oprot_factory.getProtocol(trans)
try:
yield self._processor.process(iprot, oprot)
except Exception as e:
logger.error(e, exc_info=True)
self._channel.basic_ack(delivery_tag=method.delivery_tag)
def connect(self, ioloop=None, stop_ioloop=True):
logger.debug('Connecting to %s', self._url)
self.stop_ioloop = stop_ioloop
return TornadoConnection(URLParameters(self._url),
self.on_connection_open,
custom_ioloop=ioloop)
class PikaClient(object):
INPUT_QUEUE_NAME = 'in_queue'
def __init__(self, io_loop):
self.io_loop = io_loop
self.received_message_counter = 0
self.sent_message_counter = 0
self.start_time = -1
self.connected = False
self.connecting = False
self.connection = None
self.in_channel = None
self.out_channels = {}
self.websockets = {}
self.connect()
def connect(self):
if self.connecting:
return
self.connecting = True
cred = pika.PlainCredentials('guest', 'guest')
param = pika.ConnectionParameters(
host='localhost',
port=5672,
virtual_host='/',
credentials=cred
)
self.connection = TornadoConnection(param,
on_open_callback=self.on_connected)
def on_connected(self, connection):
self.connected = True
self.connection = connection
self.in_channel = self.connection.channel(self.on_conn_open)
def on_conn_open(self, channel):
self.in_channel.exchange_declare(exchange='tornado_input', type='topic')
channel.queue_declare(callback=self.on_input_queue_declare, queue=self.INPUT_QUEUE_NAME)
def on_input_queue_declare(self, queue):
self.in_channel.queue_bind(callback=None,
exchange='tornado_input',
queue=self.INPUT_QUEUE_NAME,
routing_key="#")
def register_websocket(self, sess_id, ws):
self.websockets[sess_id] = ws
channel = self.create_out_channel(sess_id)
def unregister_websocket(self, sess_id):
del self.websockets[sess_id]
if sess_id in self.out_channels:
self.out_channels[sess_id].close()
print("Time: %s, In: %s Out: %s" % (int(time.time() - self.start_time),
self.received_message_counter,
self.sent_message_counter) )
def create_out_channel(self, sess_id):
def on_output_channel_creation(channel):
def on_output_queue_decleration(queue):
channel.basic_consume(self.on_message, queue=sess_id)
self.out_channels[sess_id] = channel
channel.queue_declare(callback=on_output_queue_decleration,
queue=sess_id,
auto_delete=True,
exclusive=True)
self.connection.channel(on_output_channel_creation)
def redirect_incoming_message(self, sess_id, message):
if not self.sent_message_counter:
self.start_time = time.time()
self.received_message_counter += 1
if not self.received_message_counter % 1000:
print("Total Received: %s " % self.received_message_counter)
self.in_channel.basic_publish(exchange='tornado_input',
routing_key='in.%s' % sess_id,
body=message)
def on_message(self, channel, method, header, body):
self.sent_message_counter += 1
sess_id = method.routing_key
if sess_id in self.websockets:
self.websockets[sess_id].write_message(body)
channel.basic_ack(delivery_tag=method.delivery_tag)
else:
channel.basic_reject(delivery_tag=method.delivery_tag)
class PikaClient(object):
INPUT_QUEUE_NAME = 'in_queue'
def __init__(self, io_loop):
self.io_loop = io_loop
self.received_message_counter = 0
self.sent_message_counter = 0
self.start_time = -1
self.connected = False
self.connecting = False
self.connection = None
self.in_channel = None
self.out_channels = {}
self.websockets = {}
self.connect()
def connect(self):
if self.connecting:
return
self.connecting = True
cred = pika.PlainCredentials('guest', 'guest')
param = pika.ConnectionParameters(
host='localhost',
port=5672,
virtual_host='/',
credentials=cred
)
self.connection = TornadoConnection(param,
on_open_callback=self.on_connected)
def on_connected(self, connection):
self.connected = True
self.connection = connection
self.in_channel = self.connection.channel(self.on_conn_open)
def on_conn_open(self, channel):
self.in_channel.exchange_declare(exchange='tornado_input', type='topic')
channel.queue_declare(callback=self.on_input_queue_declare, queue=self.INPUT_QUEUE_NAME)
def on_input_queue_declare(self, queue):
self.in_channel.queue_bind(callback=None,
exchange='tornado_input',
queue=self.INPUT_QUEUE_NAME,
routing_key="#")
def register_websocket(self, sess_id, ws):
self.websockets[sess_id] = ws
channel = self.create_out_channel(sess_id)
def unregister_websocket(self, sess_id):
del self.websockets[sess_id]
if sess_id in self.out_channels:
self.out_channels[sess_id].close()
print("Time: %s, In: %s Out: %s" % (int(time.time() - self.start_time),
self.received_message_counter,
self.sent_message_counter) )
def create_out_channel(self, sess_id):
def on_output_channel_creation(channel):
def on_output_queue_decleration(queue):
channel.basic_consume(self.on_message, queue=sess_id)
self.out_channels[sess_id] = channel
channel.queue_declare(callback=on_output_queue_decleration,
queue=sess_id,
auto_delete=True,
exclusive=True)
self.connection.channel(on_output_channel_creation)
def redirect_incoming_message(self, sess_id, message):
if not self.sent_message_counter:
self.start_time = time.time()
self.received_message_counter += 1
if not self.received_message_counter % 1000:
print("Total Received: %s " % self.received_message_counter)
self.in_channel.basic_publish(exchange='tornado_input',
routing_key=sess_id,
body=message)
def on_message(self, channel, method, header, body):
self.sent_message_counter += 1
sess_id = method.routing_key
if sess_id in self.websockets:
self.websockets[sess_id].write_message(body)
channel.basic_ack(delivery_tag=method.delivery_tag)
else:
channel.basic_reject(delivery_tag=method.delivery_tag)
def start(self):
logger.info("Starting the connection")
self._connection = TornadoConnection(pika.URLParameters(self._url),
self.on_connection_open,
self.on_connection_error,
self.on_connection_error)
def go(self):
self.amqp = TornadoConnection(
parameters=self.parameters,
on_open_callback=self.on_amqp_connection_open
)
class RabbitMQClient(BaseClient):
"""
Base RabbitMQ asynchronous adapter.
"""
def __init__(self,
host=settings.HOST,
port=settings.PORT,
virtual_host=settings.VIRTUAL_HOST,
username=settings.USERNAME,
password=settings.PASSWORD,
exchange_name='direct',
exchange_type='direct'):
"""
Initialize RabbitMQ client with passed configuration or get parameters
from django settings module.
:param host: RabbitMQ host
:type host: :class:`str`
:param port: RabbitMQ port
:type port: :class:`int`
:param virtual_host: RabbitMQ virtual host
:type virtual_host: :class:`str`
:param username: RabbitMQ user
:type username: :class:`str`
:param password: RabbitMQ user's password
:type password: :class:`str`
:param exchange_name: Exchange name, see RabbitMQ docs for more info.
:type exchange_name: :class:`str`
:param exchange_type: Exchange type, see RabbitMQ docs for more info.
:type exchange_type: :class:`str`
"""
self.host = host
self.port = int(port)
self.virtual_host = virtual_host
self.username = username
self.password = password
self.exchange_name = exchange_name
self.exchange_type = exchange_type
self.connected = False
self.connecting = False
self.reconnecting = False
self.closing = False
self.connection = None
self.channel = None
############################################################################
# CONNECTION METHODS
############################################################################
def connect(self):
"""
Asynchronous connection. Connects to RabbitMQ server and establish
non-blocking connection. After connection is established on_connected
callback will be executed which will try to reconnect if this function
failed.
"""
if self.connecting and not self.reconnecting:
return
self.connecting = True
credentials = PlainCredentials(self.username, self.password)
param = ConnectionParameters(host=self.host,
port=self.port,
virtual_host=self.virtual_host,
credentials=credentials)
self.connection = TornadoConnection(param,
on_open_callback=self.on_connected)
self.connection.add_on_close_callback(self.on_close)
def reconnect(self):
"""
Reconnect method.
Basically you don't need to call this method manually.
"""
self.reconnecting = True
self.connect()
def disconnect(self):
"""
Disconnect method.
Call this method after you end with messaging.
"""
self.closing = True
self.connection.close()
############################################################################
# CALLBACKS
############################################################################
def on_connected(self, connection):
"""
Callback on connection. Reconnect if connection dropped. Otherwise it
will open channel and on_channel_open callback will be executed.
:param connection: RabbitMQ connection.
:type connection: :class:`TornadoConnection`
"""
self.connected = True
self.connection = connection
self.connection.channel(self.on_channel_open)
def on_channel_open(self, channel, callback=lambda frame: None):
"""
Callback on channel open. It will declare exchanges for messaging. See
RabbitMQ docs for more information.
:param channel: Opened channel.
:type channel: :class:`Channel`
:param callback: Callback that will be executed after channel open.
:type callback: callable object
"""
self.channel = channel
self.channel.exchange_declare(exchange=self.exchange_name,
type=self.exchange_type,
auto_delete=False,
durable=True,
callback=callback)
def on_close(self, connection, *args, **kwargs):
"""
On close callback.
You don't need to manually call this method.
:param connection: Established connection.
:type connection: :class:`TornadoConnection`
:param args: Internal args
:param kwargs: Internal kwargs
"""
self.channel = None
if not self.closing:
self.reconnect()
self.connection.add_timeout(5, self.reconnect)
else:
connection.close()
class QueueManager(object):
"""
Async RabbitMQ & Tornado websocket connector
"""
INPUT_QUEUE_NAME = 'in_queue'
def __init__(self, io_loop=None):
log.info('PikaClient: __init__')
self.io_loop = io_loop
self.connected = False
self.connecting = False
self.connection = None
self.in_channel = None
self.out_channels = {}
self.out_channel = None
self.websockets = {}
# self.connect()
def connect(self):
"""
Creates connection to RabbitMQ server
"""
if self.connecting:
log.info('PikaClient: Already connecting to RabbitMQ')
return
log.info('PikaClient: Connecting to RabbitMQ')
self.connecting = True
self.connection = TornadoConnection(NON_BLOCKING_MQ_PARAMS,
stop_ioloop_on_close=False,
custom_ioloop=self.io_loop,
on_open_callback=self.on_connected)
def on_connected(self, connection):
"""
AMQP connection callback.
Creates input channel.
Args:
connection: AMQP connection
"""
log.info('PikaClient: connected to RabbitMQ')
self.connected = True
self.in_channel = self.connection.channel(self.on_channel_open)
def on_channel_open(self, channel):
"""
Input channel creation callback
Queue declaration done here
Args:
channel: input channel
"""
self.in_channel.exchange_declare(exchange='input_exc',
type='topic',
durable=True)
channel.queue_declare(callback=self.on_input_queue_declare,
queue=self.INPUT_QUEUE_NAME)
def on_input_queue_declare(self, queue):
"""
Input queue declaration callback.
Input Queue/Exchange binding done here
Args:
queue: input queue
"""
self.in_channel.queue_bind(callback=None,
exchange='input_exc',
queue=self.INPUT_QUEUE_NAME,
routing_key="#")
def register_websocket(self, sess_id, ws):
"""
Args:
sess_id:
ws:
"""
self.websockets[sess_id] = ws
self.create_out_channel(sess_id)
def inform_disconnection(self, sess_id):
self.in_channel.basic_publish(exchange='input_exc',
routing_key=sess_id,
body=json_encode(
dict(data={
'view':
'_zops_mark_offline_user',
'sess_id': sess_id,
},
_zops_source='Internal',
_zops_remote_ip='')))
self.websockets[sess_id].write_message(
json.dumps({
"cmd": "status",
"status": "closing"
}))
def unregister_websocket(self, sess_id):
# user_id = sys.sessid_to_userid.get(sess_id, None)
try:
self.inform_disconnection(sess_id)
del self.websockets[sess_id]
except KeyError:
log.exception("Non-existent websocket for %s" % sess_id)
if sess_id in self.out_channels:
try:
self.out_channels[sess_id].close()
except ChannelClosed:
log.exception("Pika client (out) channel already closed")
def create_out_channel(self, sess_id):
def _on_output_channel_creation(channel):
def _on_output_queue_decleration(queue):
# differentiate and identify incoming message with registered consumer
channel.basic_consume(
self.on_message,
queue=sess_id,
consumer_tag=sess_id,
# no_ack=True
)
log.debug("BINDED QUEUE TO WS Q.%s" % sess_id)
self.out_channels[sess_id] = channel
channel.queue_declare(
callback=_on_output_queue_decleration,
queue=sess_id,
arguments={'x-expires': 40000},
# auto_delete=True,
# exclusive=True
)
self.connection.channel(_on_output_channel_creation)
def redirect_incoming_message(self, sess_id, message, request):
message = json_decode(message)
message['_zops_sess_id'] = sess_id
message['_zops_remote_ip'] = request.remote_ip
message['_zops_source'] = 'Remote'
self.in_channel.basic_publish(exchange='input_exc',
routing_key=sess_id,
body=json_encode(message))
def on_message(self, channel, method, header, body):
sess_id = method.consumer_tag
log.debug("WS RPLY for %s" % sess_id)
log.debug("WS BODY for %s" % body)
try:
if sess_id in self.websockets:
log.info("write msg to client")
self.websockets[sess_id].write_message(body)
log.debug("WS OBJ %s" % self.websockets[sess_id])
channel.basic_ack(delivery_tag=method.delivery_tag)
except RuntimeError:
log.exception("CANT WRITE TO HTTP OR WS: %s\n \n%s" %
(sess_id, body))
except KeyError:
self.unregister_websocket(sess_id)
log.exception("CANT FIND WS OR HTTP: %s" % sess_id)
class TAMQPTornadoTransport(TTransportBase):
def __init__(self, channel=None, exchange_name=constant.EXCHANGE_NAME,
routing_key=constant.ROUTING_KEY, properties=None,
method=None, io_loop=None, **kwargs):
self._channel = channel
self._exchange_name = exchange_name
self._routing_key = routing_key
self._wbuf = StringIO()
self._properties = properties
self._method = method
self._url = kwargs.get('url', constant.DEFAULT_URL)
self._reply_to = None
self._lock = _Lock()
self._callback = None
self._connection = None
self._consumer_tag = None
self._consumer_name = kwargs.get('consumer_tag')
self._message_expiration = kwargs.get('message_expiration')
self._callback_queue = Queue()
self.io_loop = io_loop or ioloop.IOLoop.instance()
@gen.coroutine
def assign_queue(self):
logger.info("Openning callback queue")
result = yield gen.Task(self._channel.queue_declare,
exclusive=True)
logger.info("Callback queue openned")
self._reply_to = result.method.queue
self._lock.release()
self._consumer_tag = self._channel.basic_consume(
self.on_reply_message, self._reply_to,
consumer_tag=self._consumer_name)
if self._callback:
self._callback()
def on_reply_message(self, _channel, method, properties, body):
if method:
self._channel.basic_ack(delivery_tag=method.delivery_tag)
self._callback_queue.put(body)
def on_connection_open(self, _connection):
logger.info("Openning channel")
self._connection.channel(on_open_callback=self.on_channel_open)
def open(self, callback=None):
logger.info("Openning AMQP transport")
if self._channel is not None:
logger.info("Already set")
callback()
else:
logger.info("Openning connection")
self._callback = callback
self._connection = TornadoConnection(pika.URLParameters(self._url),
self.on_connection_open,
self.on_connection_error,
self.on_connection_error)
self._lock.acquire()
@gen.coroutine
def on_connection_error(self, *args, **kwargs):
logger.info("Connection failed")
yield gen.sleep(constant.TIMEOUT_RECONNECT)
self.start()
@gen.coroutine
def readFrame(self):
result = yield self._callback_queue.get()
raise gen.Return(result)
def on_channel_open(self, channel):
logger.info("Channel openned")
self._channel = channel
self.assign_queue()
def close(self):
if self._channel:
if self._consumer_tag:
self._channel.basic_cancel(consumer_tag=self._consumer_tag,
nowait=True)
self._channel.close()
def isOpen(self):
return self._channem is not None
def read(self, _):
assert False, "wrong stuff"
def write(self, buf):
self._wbuf.write(buf)
@gen.coroutine
def flush(self, recovered=False):
try:
yield self.flush_once()
except Exception as e:
self._connection.connect()
raise thrift.transport.TTransport.TTransportException(
message=str(e))
@gen.coroutine
def flush_once(self):
if self._properties is not None:
props = pika.BasicProperties(
correlation_id=self._properties.correlation_id)
self._channel.basic_publish(exchange='',
routing_key=self._properties.reply_to,
properties=props,
body=self._wbuf.getvalue())
if self._method is not None:
self._channel.basic_ack(
delivery_tag=self._method.delivery_tag)
else:
with (yield self._lock.acquire()):
props = pika.BasicProperties(correlation_id=str(uuid.uuid4()),
reply_to=self._reply_to)
if self._message_expiration:
props.expiration = str(self._message_expiration * 1000)
self._channel.basic_publish(exchange=self._exchange_name,
routing_key=self._routing_key,
properties=props,
body=str(self._wbuf.getvalue()))
self._wbuf = StringIO()
class Bus(Component):
"""
Apollo's message bus system.
"""
def __init__(self, core):
super(Bus, self).__init__(core)
self.ready = False
self.busName = uuid.uuid4().hex
self.parameters = ConnectionParameters(
credentials=PlainCredentials(
options.amqp_username,
options.amqp_password
),
host=options.amqp_host,
port=options.amqp_port,
virtual_host=options.amqp_vhost
)
def go(self):
self.amqp = TornadoConnection(
parameters=self.parameters,
on_open_callback=self.on_amqp_connection_open
)
def on_amqp_connection_open(self, conn):
self.amqp.channel(self.on_amqp_channel_open)
def on_amqp_channel_open(self, channel):
self.channel = channel
self.declareQueue("inter", self.on_queue_declared)
def on_queue_declared(self, something):
self.channel.basic_consume(
consumer_callback=self.on_inter_message,
queue="inter",
no_ack=False
)
self.channel.queue_bind(
exchange="amq.topic",
queue="inter",
routing_key="inter.#"
)
self.ready = True
logging.info("Message bus ready.")
def send(self, dest, packet):
"""
Send a packet to a specific destination.
:Parameters:
* ``dest``
Destination. Prefixed with ``cross.`` for cross-server
communication and ``user.`` for direct user messaging.
* ``packet``
Packet to send.
"""
packet_dump = packet.dump()
logging.debug("Sending to %s: %s" % (dest, packet_dump))
self.channel.basic_publish(
exchange="amq.topic",
routing_key=dest,
body=packet_dump,
properties=BasicProperties(
delivery_mode=2
)
)
def declareQueue(self, queue, callback=None):
self.channel.queue_declare(
queue=queue,
auto_delete=False,
callback=callback
)
def bindQueue(self, queue, dest, callback=None):
logging.debug("Binding %s to %s" % (queue, dest))
self.channel.queue_bind(
exchange="amq.topic",
queue=queue,
routing_key=dest,
callback=callback
)
def unbindQueue(self, queue, dest, callback=None):
logging.debug("Unbinding %s from %s" % (queue, dest))
self.channel.queue_unbind(
exchange="amq.topic",
queue=queue,
routing_key=dest,
callback=callback
)
def deleteQueue(self, queue):
logging.debug("Deleting queue %s" % queue)
self.channel.queue_delete(queue=queue)
def on_inter_message(self, channel, method, header, body):
"""
Process an "inter" message.
"""
sess = meta.Session()
channel.basic_ack(delivery_tag=method.delivery_tag)
prefixparts = method.routing_key.split(".")
packet = deserializePacket(body)
packet._origin = ORIGIN_INTER
if prefixparts[1] == "global":
for user in sess.query(User):
packet.dispatch(self.core, FakeSession(user.id))
return
ident = uuid.UUID(hex=prefixparts[2])
if prefixparts[1] == "User":
user = sess.query(User).get(ident)
packet.dispatch(self.core, FakeSession(user.id))
elif prefixparts[1] == "Tile":
for user in sess.query(User).filter(User.location_id == ident):
packet.dispatch(self.core, FakeSession(user.id))
elif prefixparts[1] == "Group":
for user in sess.query(User).filter(User.group_id == ident):
packet.dispatch(self.core, FakeSession(user.id))
elif prefixparts[1] == "Realm":
for user in sess.query(User).filter(User.location_id == Tile.id).filter(Tile.chunk_id == Chunk.id).filter(Chunk.realm_id == Realm.id).filter(Realm.id == ident):
packet.dispatch(self.core, FakeSession(user.id))
def broadcastEx(self, packet):
self.send("ex.global", packet)
def broadcastInter(self, packet):
self.send("inter.global", packet)
def globalBind(self, session, callback=None):
self.bindQueue("ex:%s" % session.id, "ex.global", callback)
def globalUnbind(self, session, callback=None):
self.unbindQueue("ex:%s" % session.id, "ex.global", callback)
class TAMQPTornadoTransport(TTransportBase):
def __init__(self,
channel=None,
exchange_name=constant.EXCHANGE_NAME,
routing_key=constant.ROUTING_KEY,
properties=None,
method=None,
io_loop=None,
**kwargs):
self._channel = channel
self._exchange_name = exchange_name
self._routing_key = routing_key
self._wbuf = StringIO()
self._properties = properties
self._method = method
self._url = kwargs.get('url', constant.DEFAULT_URL)
self._reply_to = None
self._lock = _Lock()
self._callback = None
self._connection = None
self._consumer_tag = None
self._consumer_name = kwargs.get('consumer_tag')
self._message_expiration = kwargs.get('message_expiration')
self._reply_queue_name = kwargs.get('reply_queue_name', '')
self._error_logger = kwargs.get('error_logger')
self._callback_queue = Queue()
self.io_loop = io_loop or ioloop.IOLoop.instance()
self._closing = False
self._starting = False
@gen.coroutine
def assign_queue(self):
logger.info("Openning callback queue")
result = yield gen.Task(self._channel.queue_declare,
queue=self._reply_queue_name,
exclusive=True,
auto_delete=True)
logger.info("Callback queue openned")
if self._reply_queue_name == '':
self._reply_to = result.method.queue
else:
self._reply_to = self._reply_queue_name
if self._lock.acquired():
self._lock.release()
self._consumer_tag = self._channel.basic_consume(
self.on_reply_message,
self._reply_to,
consumer_tag=self._consumer_name)
self._starting = False
if self._callback:
self._callback()
def on_reply_message(self, _channel, method, properties, body):
if method:
self._channel.basic_ack(delivery_tag=method.delivery_tag)
self._callback_queue.put(body)
def on_connection_open(self, _connection):
logger.info("Openning channel")
self._connection.channel(on_open_callback=self.on_channel_open)
def open(self, callback=None):
if self._starting:
if callback:
callback()
return
self._starting = True
logger.info("Openning AMQP transport")
if self._channel is not None and self._channel.is_open:
logger.info("Already set")
if callback:
callback()
else:
logger.info("Openning connection")
self._callback = callback
self._lock.acquire()
self._connection = TornadoConnection(pika.URLParameters(self._url),
self.on_connection_open,
self.on_connection_error,
self.on_connection_error)
@gen.coroutine
def on_connection_error(self, *args, **kwargs):
self._starting = False
logger.info("Connection failed")
yield gen.sleep(constant.TIMEOUT_RECONNECT)
self.open(self._callback)
@gen.coroutine
def readFrame(self):
result = yield self._callback_queue.get()
if issubclass(result.__class__, Exception):
raise result
raise gen.Return(result)
@gen.coroutine
def on_channel_close(self, *args):
logger.info("Channel closed")
yield gen.sleep(constant.TIMEOUT_RECONNECT)
self._callback_queue.put(
thrift.transport.TTransport.TTransportException(
message='channel closed'))
if not self._closing and not self._starting:
if self._connection and self._connection.is_open:
self._lock.acquire()
self._connection.channel(on_open_callback=self.on_channel_open)
else:
self.open()
def on_channel_open(self, channel):
logger.info("Channel openned")
self._channel = channel
self._channel.add_on_close_callback(self.on_channel_close)
self.assign_queue()
def close(self):
self._closing = True
if self._channel:
if self._consumer_tag:
self._channel.basic_cancel(consumer_tag=self._consumer_tag,
nowait=True)
self._channel.close()
def isOpen(self):
return self._channel is not None
def read(self, _):
assert False, "wrong stuff"
@gen.coroutine
def write(self, buf):
with (yield self._lock.acquire()):
if type(buf) is unicode:
self._wbuf.write(buf.encode("utf-8"))
else:
self._wbuf.write(buf)
@gen.coroutine
def flush(self, recovered=False):
try:
yield self.flush_once()
self._wbuf = StringIO()
except Exception as e:
yield gen.sleep(constant.TIMEOUT_RECONNECT)
if not recovered:
yield self.flush(True)
else:
if self._error_logger:
self._error_logger.capture_exception()
logger.info(e, exc_info=True)
self._wbuf = StringIO()
raise thrift.transport.TTransport.TTransportException(
message=str(e))
@gen.coroutine
def flush_once(self):
if self._properties is not None:
props = pika.BasicProperties(
correlation_id=self._properties.correlation_id)
result = self._wbuf.getvalue()
if type(result) is unicode:
result = result.encode("utf-8")
self._channel.basic_publish(exchange='',
routing_key=self._properties.reply_to,
properties=props,
body=result)
if self._method is not None:
self._channel.basic_ack(delivery_tag=self._method.delivery_tag)
else:
with (yield self._lock.acquire()):
props = pika.BasicProperties(correlation_id=str(uuid.uuid4()),
reply_to=self._reply_to)
if self._message_expiration:
props.expiration = str(self._message_expiration * 1000)
result = self._wbuf.getvalue()
if type(result) is unicode:
result = result.encode("utf-8")
self._channel.basic_publish(exchange=self._exchange_name,
routing_key=self._routing_key,
properties=props,
body=result)
self._wbuf = StringIO()
