class PikaClient(object):
#all the following functions precede in order starting with connect
def connect(self):
try:
logger = logging.getLogger('rmq_tornado')
credentials = pika.PlainCredentials(RMQ_USER, RMQ_PWD)
param = pika.ConnectionParameters(host=RMQ_HOST, port=RMQ_PORT, credentials=credentials)
self.connection = TornadoConnection(param, on_open_callback=self.on_connected)
except Exception as e:
logger.error('Something went wrong... %s', e)
def on_connected(self, connection):
"""When we are completely connected to rabbitmq this is called"""
logger.info('Succesfully connected to rabbitmq')
#open a channel
self.connection.channel(self.on_channel_open)
def on_channel_open(self, new_channel):
"""When the channel is open this is called"""
logging.info('Opening channel to rabbitmq')
global channel
channel = new_channel
class PikaClient(object):
# All the following functions precede in order starting with connect
def connect(self):
try:
logger = logging.getLogger('rmq_tornado')
credentials = pika.PlainCredentials(RMQ_USER, RMQ_PWD)
param = pika.ConnectionParameters(host=RMQ_HOST, port=RMQ_PORT, credentials=credentials)
self.connection = TornadoConnection(param, on_open_callback=self.on_connected)
except Exception as e:
logger.error('Something went wrong... %s', e)
def on_connected(self, connection):
"""When we are completely connected to rabbitmq this is called"""
logger.info('Succesfully connected to rabbitmq')
# Open a channel
self.connection.channel(self.on_channel_open)
def on_channel_open(self, new_channel):
"""When the channel is open this is called"""
logging.info('Opening channel to rabbitmq')
global channel
channel = new_channel
class PikaClient(object):
def __init__(self, username='******', exchange_name='ws', password='******', host='localhost', port=5672, virtual_host='/'):
self.exchange_name = exchange_name
# Options
self.username = username
self.password = password
self.host = host
self.port = port
self.virtual_host = virtual_host
# Default values
self.connected = False
self.connecting = False
self.connection = None
self.channel = None
def connect(self):
if self.connecting:
pika.log.info('PikaClient: Already connecting to RabbitMQ')
return
pika.log.info('PikaClient: Connecting to RabbitMQ on localhost:5672')
self.connecting = True
credentials = pika.PlainCredentials(self.username, self.password)
param = pika.ConnectionParameters(host=self.host,
port=self.port,
virtual_host=self.virtual_host,
credentials=credentials)
self.connection = TornadoConnection(param,
on_open_callback=self.on_connected)
self.connection.add_on_close_callback(self.on_closed)
def on_connected(self, connection):
pika.log.info('PikaClient: Connected to RabbitMQ on localhost:5672')
self.connected = True
self.connection = connection
self.connection.channel(self.on_channel_open)
def on_channel_open(self, channel):
pika.log.info('PikaClient: Channel Open, Declaring Exchange')
self.channel = channel
self.channel.exchange_declare(exchange=self.exchange_name,
type="direct",
callback=self.on_exchange_declared)
def on_exchange_declared(self, frame):
pika.log.info('PikaClient: Exchange Declared, Ready for declaring Queues')
def on_basic_cancel(self, frame):
pika.log.info('PikaClient: Basic Cancel Ok')
# If we don't have any more consumer processes running close
self.connection.close()
def on_closed(self, connection):
# We've closed our pika connection so stop the demo
tornado.ioloop.IOLoop.instance().stop()
class PikaClient(object):
callbacks = {}
def __init__(self, callback_ins, port):
self.callback_ins=callback_ins
self.queue_name = "q%d" % uuid.uuid4().int
if tornado is None:
raise Exception('You must add tornado to your requirements!')
if pika is None:
raise Exception('You must add pika to your requirements!')
self.ioloop = tornado.ioloop.IOLoop.instance()
self.connection = None
self.channel = None
self._delivery_tag = 0
self.parameters = pika.URLParameters("amqp://*****:*****@hostname:" + str(port) + "/%2F")
def connect(self):
self.connection = TornadoConnection(self.parameters, on_open_callback=self.on_connected, stop_ioloop_on_close=False)
self.connection.add_on_close_callback(self.on_closed)
def on_connected(self, connection):
logger.info('PikaClient: connected to RabbitMQ')
self.connection.channel(self.on_exchange_declare)
def on_exchange_declare(self, channel):
logger.info('PikaClient: Channel %s open, Declaring exchange' % channel)
self.channel = channel
self.channel.exchange_declare(self.on_queue_declare, exchange='compute', type='fanout')
def on_queue_declare(self, method_frame):
logger.info('PikaClient: Channel open, Declaring queue')
self.result = self.channel.queue_declare(self.on_queue_bind, queue=self.queue_name, durable=True)
def on_queue_bind(self, method_frame):
logger.info('Queue bound')
self.channel.queue_bind(self.on_consume_bind, queue=self.queue_name, exchange="compute")
def on_consume_bind(self, frame):
self.channel.basic_qos(prefetch_count=1)
self.channel.basic_consume(self.on_response, queue=self.queue_name, no_ack=False)
def on_response(self, channel, method, properties, body):
message = body
self.callback_ins.handle_rabbitmq_message(message)
channel.basic_ack(delivery_tag = method.delivery_tag)
logger.info('Recieve a new Message: %r' % message)
def on_closed(self, connection):
logger.info('PikaClient: rabbit connection closed')
self.connection.close()
self.channel.close()
self.ioloop.stop()
class PikaClient(object):
def __init__(self, io_loop):
self.io_loop = io_loop
self.connected = False
self.connecting = False
self.connection = None
self.channel = None
self.event_listeners = set([])
def connect(self):
if self.connecting:
return
self.connecting = True
self.connection = TornadoConnection(on_open_callback=self.on_connected)
self.connection.add_on_close_callback(self.on_closed)
def on_connected(self, connection):
self.connected = True
self.connection = connection
self.connection.channel(self.on_channel_open)
def on_channel_open(self, channel):
self.channel = channel
channel.queue_declare(queue="plase",
durable=True,
callback=self.on_queue_declared)
def on_queue_declared(self, frame):
self.channel.basic_consume(self.on_message, queue='plase')
def on_closed(self, connection):
self.io_loop.stop()
def on_message(self, channel, method, header, body):
self.notify_listeners(body)
def notify_listeners(self, event_obj):
event_json = json.dumps(event_obj)
for listener in self.event_listeners:
listener.write_message(event_json)
def add_event_listener(self, listener):
self.event_listeners.add(listener)
def remove_event_listener(self, listener):
try:
self.event_listeners.remove(listener)
except KeyError:
pass
class PikaClient(object):
def __init__(self, io_loop):
self.io_loop = io_loop
self.connected = False
self.connecting = False
self.connection = None
self.channel = None
self.event_listeners = set([])
def connect(self):
if self.connecting:
return
self.connecting = True
self.connection = TornadoConnection(on_open_callback=self.on_connected)
self.connection.add_on_close_callback(self.on_closed)
def on_connected(self, connection):
self.connected = True
self.connection = connection
self.connection.channel(self.on_channel_open)
def on_channel_open(self, channel):
self.channel = channel
channel.queue_declare(queue="plase", durable=True, callback=self.on_queue_declared)
def on_queue_declared(self, frame):
self.channel.basic_consume(self.on_message, queue="plase")
def on_closed(self, connection):
self.io_loop.stop()
def on_message(self, channel, method, header, body):
self.notify_listeners(body)
def notify_listeners(self, event_obj):
event_json = json.dumps(event_obj)
for listener in self.event_listeners:
listener.write_message(event_json)
def add_event_listener(self, listener):
self.event_listeners.add(listener)
def remove_event_listener(self, listener):
try:
self.event_listeners.remove(listener)
except KeyError:
pass
class PikaConsumer(object):
def __init__(self):
if tornado is None:
raise Exception('You must add tornado to your requirements!')
if pika is None:
raise Exception('You must add pika to your requirements!')
self.ioloop = tornado.ioloop.IOLoop.instance()
self.connection = None
self.channel = None
self._delivery_tag = 0
self.parameters = None
self.queue_name = None
self.on_task_end = None
def connect(self):
self.connection = TornadoConnection(self.parameters,
on_open_callback=self.on_connected,
stop_ioloop_on_close=False)
self.connection.add_on_close_callback(self.on_closed)
def on_connected(self, connection):
logger.info('PikaConsumer: connected to RabbitMQ')
self.connection.channel(on_open_callback=self.on_channel_declared)
def on_channel_declared(self, channel):
self.channel = channel
self.channel.queue_declare(self.on_queue_declared,
queue=self.queue_name)
def on_queue_declared(self, method_frame):
self.channel.basic_qos(prefetch_count=1)
self.channel.basic_consume(self.on_consume,
queue=self.queue_name,
no_ack=False)
def on_closed(self, connection):
logger.info('PikaConsumer: rabbit connection closed')
self.connection.close()
self.channel.close()
self.ioloop.stop()
def on_consume(self, channel, method, properties, body):
try:
if self.on_task_end:
self.on_task_end(body)
except Exception as e:
logger.info('PikaConsumer: Error %r' % e)
channel.basic_ack(delivery_tag=method.delivery_tag)
class PikaClient:
def __init__(self, io_loop):
print(f'{self.__str__()}: __init__()')
self.io_loop = io_loop
self.connection = None
self.connected = False
self.channel = None
# 打开链接
def connect(self):
if self.connected:
return
cred = PlainCredentials(USERNAME, PASSWORD)
params = ConnectionParameters(host=HOST, port=PORT, virtual_host=VHOST, credentials=cred)
self.connection = TornadoConnection(params, on_open_callback=self.on_connected)
self.connection.add_on_close_callback(callback=self.on_closed)
# 连接成功 callback
def on_connected(self, connection):
self.connected = True
self.connection = connection
print(f'{self.__str__()}: connected() succeed')
self.connection.channel(on_open_callback=self.on_channel_open)
# 关闭连接 callback
def on_closed(self, connection):
print(f'{self.__str__()}: on_closed()')
connection.close()
self.connection = None
self.connected = False
self.io_loop.stop()
# 打开通道 callback
def on_channel_open(self, channel):
self.channel = channel
self.channel.exchange_declare(exchange=EXCHANGE, exchange_type="direct", durable=True)
# queue如果通过其他方式已经创建的话可能出错, 如果出错先删除再重新创建
try:
self.channel.queue_declare(queue=QUEUE, durable=True)
except Exception as e:
print(f'{self.__str__()} channel.queue_declare Error: ', e)
channel.queue_delete(queue=QUEUE)
channel.queue_declare(queue=QUEUE, durable=True)
self.channel.queue_bind(queue=QUEUE, exchange=EXCHANGE, routing_key=ROUTING_KEY)
# 如果是消费者, 绑定消息处理 callback
if isinstance(self, Consumer):
self.channel.basic_consume(queue=QUEUE, on_message_callback=self.handle_message, auto_ack=True)
class AMQPClient(object):
"Connect to RabbitMQ and create a channel"
def __init__(self, on_msg_callback=None, oid=None, io_loop=None,
on_channel_created=None):
self.oid = oid or options.oid
self.io_loop = io_loop or IOLoop.instance()
self.on_msg_callback = on_msg_callback
self.connection = None
self.channel = None
self._on_channel_created = on_channel_created
self.checker = PeriodicCallback(self._check_connection, 1000)
def connect(self):
"Connect to RabbitMQ"
log.debug("Connecting to RabbitMQ")
if self.connection:
return
self.connection = TornadoConnection(
get_conn_params(),
self.on_connected,
custom_ioloop=self.io_loop)
def on_connected(self, connection):
"Create a channel just after connected"
log.debug("%s is established" % connection)
self.connection.channel(self.on_channel_created)
self.checker.start()
def on_channel_created(self, channel):
"Implement in subclasses"
log.debug("%s is established" % channel)
self.channel = channel
if self._on_channel_created:
self._on_channel_created(channel)
def _check_connection(self):
"Restablish connection to server if we lost it"
if self.connection:
try:
self.connection.socket.fileno()
except socket.error, exc:
log.debug("lost connection to RabbitMQ, %s" % str(exc))
self.checker.stop()
self.connection = None
self.connect()
class PikaClient(object):
def __init__(self, host, queue):
self.queue = queue
self.host = host
self.connected = False
self.connecting = False
self.connection = None
self.channel = None
self.callbacks = {}
def connect(self):
if self.connecting:
return
self.connecting = True
self.connection = TornadoConnection(
pika.ConnectionParameters(host=self.host),
on_open_callback=self.on_connected
)
self.connection.add_on_close_callback(self.on_closed)
def on_connected(self, connection):
self.connected = True
self.connection = connection
self.connection.channel(self.on_channel_open)
def on_channel_open(self, chanel):
self.channel = chanel
self.channel.queue_declare(
queue=self.queue,
durable=True,
)
def on_basic_cancel(self, frame):
self.connection.close()
def on_closed(self, connection):
tornado.ioloop.IOLoop.instance().stop()
def send(self, body):
self.channel.basic_publish(
exchange='',
routing_key=self.queue,
body=json.dumps(body),
)
class PikaClient(object):
def __init__(self):
self.connected = False
self.connection = None
self.channel = None
self.messages = list()
def connect(self):
self.connection = TornadoConnection(on_open_callback=self.on_connected)
def on_connected(self, connection):
self.connected = True
self.connection = connection
self.connection.channel(self.on_channel_open)
def on_channel_open(self, channel):
pika.log.info("channel open")
self.channel = channel
class PikaClient(object):
def __init__(self, host, queue):
self.queue = queue
self.host = host
self.connected = False
self.connecting = False
self.connection = None
self.channel = None
self.callbacks = {}
def connect(self):
if self.connecting:
return
self.connecting = True
self.connection = TornadoConnection(
pika.ConnectionParameters(host=self.host),
on_open_callback=self.on_connected)
self.connection.add_on_close_callback(self.on_closed)
def on_connected(self, connection):
self.connected = True
self.connection = connection
self.connection.channel(self.on_channel_open)
def on_channel_open(self, chanel):
self.channel = chanel
self.channel.queue_declare(
queue=self.queue,
durable=True,
)
def on_basic_cancel(self, frame):
self.connection.close()
def on_closed(self, connection):
tornado.ioloop.IOLoop.instance().stop()
def send(self, body):
self.channel.basic_publish(
exchange='',
routing_key=self.queue,
body=json.dumps(body),
)
class PikaProducer(object):
def __init__(self):
if tornado is None:
raise Exception('You must add tornado to your requirements!')
if pika is None:
raise Exception('You must add pika to your requirements!')
self.ioloop = tornado.ioloop.IOLoop.instance()
self.connection = None
self.channel = None
self._delivery_tag = 0
self.parameters = None
self.queue_name = None
def connect(self):
self.connection = TornadoConnection(self.parameters,
on_open_callback=self.on_connected,
stop_ioloop_on_close=False)
self.connection.add_on_close_callback(self.on_closed)
def on_connected(self, connection):
logger.info('PikaProducer: connected to RabbitMQ')
self.connection.channel(on_open_callback=self.on_channel_declared)
def on_channel_declared(self, channel):
self.channel = channel
self.channel.queue_declare(self.on_queue_declared,
queue=self.queue_name)
def on_queue_declared(self, method_frame):
pass
def publish(self, body):
self.channel.basic_publish(exchange='',
routing_key=self.queue_name,
body=body)
def on_closed(self, connection):
logger.info('PikaProducer: rabbit connection closed')
self.connection.close()
self.channel.close()
class PikaClient(object):
def __init__(self, io_loop):
self.connected = False
self.connecting = False
self.connection = None
self.channel = None
self.ioloop = io_loop
def connect(self):
if self.connecting:
logger.info('PikaClient: Already connecting to RabbitMQ')
return
logger.info('PikaClient: Connecting to RabbitMQ on port 5672, Object: %s', self)
self.connecting = True
credentials = pika.PlainCredentials(RABBIT_USERNAME, RABBIT_PASS)
param = pika.ConnectionParameters(host=RABBIT_URL,
port=5672,
virtual_host='/',
credentials=credentials)
self.connection = TornadoConnection(param,
on_open_callback=self.on_connected)
global pika_connected
pika_connected = True
def on_connected(self, connection):
logger.info('PikaClient: Connected to RabbitMQ on :5672')
self.connected = True
self.connection = connection
self.connection.channel(self.on_channel_open)
def on_channel_open(self, channel):
logger.info('PikaClient: Channel Open, Declaring Exchange, Channel ID: %s', channel)
self.channel = channel
self.channel.exchange_declare(exchange='tornado',
type='direct',
durable=False,
auto_delete=True)
def declare_queue(self, token):
logger.info('PikaClient: Exchange Declared, Declaring Queue')
self.queue_name = token
self.channel.queue_declare(queue=self.queue_name,
durable=False,
auto_delete=True,
callback=self.on_queue_declared)
def on_queue_declared(self, frame):
self.channel.queue_bind(exchange='tornado',
queue=self.queue_name,
routing_key=self.queue_name,
callback=self.on_queue_bound)
def on_queue_bound(self, frame):
logger.info('PikaClient: Queue Bound, Issuing Basic Consume')
self.channel.basic_consume(consumer_callback=self.on_pika_message,
queue=self.queue_name,
no_ack=True)
def on_pika_message(self, channel, method, header, body):
logger.info('PikaCient: Message receive, delivery tag #%i', method.delivery_tag)
message = json.loads(body)
for i in websockets[message['token']]:
try:
i.write_message(body)
except:
logger.exception('exception while writing message to client')
def on_basic_cancel(self, frame):
logger.info('PikaClient: Basic Cancel Ok')
# If we don't have any more consumer processes running close
self.connection.close()
def on_closed(self, connection):
# We've closed our pika connection so stop the demo
self.ioloop.IOLoop.instance().stop()
def sample_message(self, ws_msg):
token = json.loads(ws_msg)['token']
properties = pika.BasicProperties(
content_type='text/plain', delivery_mode=1)
self.channel.basic_publish(exchange='tornado',
routing_key=token,
body=ws_msg,
properties=properties)
class PikaClient(object):
def __init__(self, io_loop):
pika.log.info("PikaClient: __init__")
self.io_loop = io_loop
self.connected = False
self.connecting = False
self.connection = None
self.channel = None
self.event_listeners = set([])
self.queue_name = "tornado-test-%i" % os.getpid()
def connect(self):
if self.connecting:
pika.log.info("PikaClient: Already connecting to RabbitMQ")
return
pika.log.info("PikaClient: Connecting to RabbitMQ")
self.connecting = True
# cred = pika.PlainCredentials('guest', 'guest')
param = pika.ConnectionParameters(
host="115.146.94.68",
# port=5672,
# virtual_host='/',
# credentials=cred
)
self.connection = TornadoConnection(param, on_open_callback=self.on_connected)
self.connection.add_on_close_callback(self.on_closed)
def on_connected(self, connection):
pika.log.info("PikaClient: connected to RabbitMQ")
self.connected = True
self.connection = connection
self.connection.channel(self.on_channel_open)
def on_channel_open(self, channel):
pika.log.info("PikaClient: Channel open, Declaring exchange")
self.channel = channel
self.channel.exchange_declare(exchange="www", type="direct", callback=self.on_exchange_declared)
return
def on_exchange_declared(self, frame):
pika.log.info("PikaClient: Exchange Declared, Declaring Queue")
self.channel.queue_declare(
queue=self.queue_name, auto_delete=True, durable=False, exclusive=False, callback=self.on_queue_declared
)
return
def on_queue_declared(self, frame):
pika.log.info("PikaClient: Queue Declared, Binding Queue")
self.channel.queue_bind(
exchange="www", queue=self.queue_name, routing_key="client", callback=self.on_queue_bound
)
def on_queue_bound(self, frame):
pika.log.info("PikaClient: Queue Bound, Issuing Basic Consume")
self.channel.basic_consume(consumer_callback=self.on_message, queue=self.queue_name, no_ack=True)
def on_closed(self, connection):
pika.log.info("PikaClient: rabbit connection closed")
self.io_loop.stop()
def on_message(self, channel, method, header, body):
pika.log.info("PikaClient: message received: %s" % body)
self.notify_listeners(body)
def send_message(self, body):
self.channel.basic_publish(exchange="www", routing_key="server", body=body)
def notify_listeners(self, body):
for listener in self.event_listeners:
listener.write_message(body)
pika.log.info("PikaClient: notified %s" % repr(listener))
def add_event_listener(self, listener):
self.event_listeners.add(listener)
pika.log.info("PikaClient: listener %s added" % repr(listener))
def remove_event_listener(self, listener):
try:
self.event_listeners.remove(listener)
pika.log.info("PikaClient: listener %s removed" % repr(listener))
except KeyError:
pass
class PikaClient(object):
def __init__(self, io_loop):
self.connected = False
self.connecting = False
self.connection = None
self.channel = None
self.ioloop = io_loop
def connect(self):
if self.connecting:
logger.info('PikaClient: Already connecting to RabbitMQ')
return
logger.info(
'PikaClient: Connecting to RabbitMQ on port 5672, Object: %s',
self)
self.connecting = True
credentials = pika.PlainCredentials(RABBIT_USERNAME, RABBIT_PASS)
param = pika.ConnectionParameters(host=RABBIT_URL,
port=5672,
virtual_host="/",
credentials=credentials)
self.connection = TornadoConnection(param,
on_open_callback=self.on_connected)
global pika_connected
pika_connected = True
def on_connected(self, connection):
logger.info('PikaClient: Connected to RabbitMQ on :5672')
self.connected = True
self.connection = connection
self.connection.channel(self.on_channel_open)
def on_channel_open(self, channel):
logger.info(
'PikaClient: Channel Open, Declaring Exchange, Channel ID: %s',
channel)
self.channel = channel
self.channel.exchange_declare(exchange='tornado',
type="direct",
durable=False,
auto_delete=True)
def declare_queue(self, token):
logger.info('PikaClient: Exchange Declared, Declaring Queue')
self.queue_name = token
self.channel.queue_declare(queue=self.queue_name,
durable=False,
auto_delete=True,
callback=self.on_queue_declared)
def on_queue_declared(self, frame):
self.channel.queue_bind(exchange='tornado',
queue=self.queue_name,
routing_key=self.queue_name,
callback=self.on_queue_bound)
def on_queue_bound(self, frame):
logger.info('PikaClient: Queue Bound, Issuing Basic Consume')
self.channel.basic_consume(consumer_callback=self.on_pika_message,
queue=self.queue_name,
no_ack=True)
def on_pika_message(self, channel, method, header, body):
logger.info('PikaCient: Message receive, delivery tag #%i',
method.delivery_tag)
message = json.loads(body)
for i in websockets[message['token']]:
try:
i.write_message(body)
except:
logger.exception("exception while writing message to client")
def on_basic_cancel(self, frame):
logger.info('PikaClient: Basic Cancel Ok')
# If we don't have any more consumer processes running close
self.connection.close()
def on_closed(self, connection):
# We've closed our pika connection so stop the demo
self.ioloop.IOLoop.instance().stop()
def sample_message(self, ws_msg):
token = json.loads(ws_msg)['token']
properties = pika.BasicProperties(content_type="text/plain",
delivery_mode=1)
self.channel.basic_publish(exchange='tornado',
routing_key=token,
body=ws_msg,
properties=properties)
class PikaClient(object):
def __init__(self):
# Construct a queue name we'll use for this instance only
#Giving unique queue for each consumer under a channel.
self.queue_name = "/4EAqhJ1FaJuuGc07qS3KwZZamZypZDMVfpx/TIO/CaqNjhuV7+36tEUCIdzborOFqtcxSBvgjW0Ywe+s0Sj9Q6+t+TfRk44kxZpw8Eyg8H6LcKbiUzWmZqMIfdsTUfqRWLaAj2UN2ZVHf6hmDyyOEbEGTl6t0qUXXNJDMaSlBLVAqsDAQAB"#"queue-%s" % (id(self),)
# Default values
self.connected = False
self.connecting = False
self.connection = None
self.channel = None
#Webscoket object.
self.websocket = None
def connect(self):
if self.connecting:
pika.log.info('PikaClient: Already connecting to RabbitMQ')
return
pika.log.info('PikaClient: Connecting to RabbitMQ on localhost:5672, Object: %s' % (self,))
self.connecting = True
credentials = pika.PlainCredentials('guest', 'guest')
param = pika.ConnectionParameters(host='pepperjack.stanford.edu',
port=5672,
virtual_host="/",
credentials=credentials)
self.connection = TornadoConnection(param,
on_open_callback=self.on_connected)
#Currently this will close tornado ioloop.
#self.connection.add_on_close_callback(self.on_closed)
def on_connected(self, connection):
pika.log.info('PikaClient: Connected to RabbitMQ on localhost:5672')
self.connected = True
self.connection = connection
self.connection.channel(self.on_channel_open)
def on_channel_open(self, channel):
pika.log.info('PikaClient: Channel Open, Declaring Exchange, Channel ID: %s' % (channel,))
self.channel = channel
self.channel.exchange_declare(exchange='tornado',
type="direct",
auto_delete=True,
durable=False,
callback=self.on_exchange_declared)
def on_exchange_declared(self, frame):
pika.log.info('PikaClient: Exchange Declared, Declaring Queue')
pika.log.info(self.queue_name)
self.channel.queue_declare(auto_delete=False,
queue = self.queue_name,
durable=True,
exclusive=False,
callback=self.on_queue_declared)
def on_queue_declared(self, frame):
pika.log.info('PikaClient: Queue Declared, Binding Queue')
self.channel.queue_bind(exchange='tornado',
queue=self.queue_name,
routing_key='tornado.*',
callback=self.on_queue_bound)
def on_queue_bound(self, frame):
pika.log.info('PikaClient: Queue Bound, Issuing Basic Consume')
self.channel.basic_consume(consumer_callback=self.on_pika_message,
queue=self.queue_name,
no_ack=True)
def on_pika_message(self, channel, method, header, body):
pika.log.info('PikaCient: Message receive, delivery tag #%i' % \
method.delivery_tag)
#Send the Cosumed message via Websocket to browser.
self.websocket.write_message(base64.encodestring(body))
def on_basic_cancel(self, frame):
pika.log.info('PikaClient: Basic Cancel Ok')
# If we don't have any more consumer processes running close
self.connection.close()
def on_closed(self, connection):
# We've closed our pika connection so stop the demo
tornado.ioloop.IOLoop.instance().stop()
def sample_message(self, ws_msg):
#Publish the message from Websocket to RabbitMQ
properties = pika.BasicProperties(content_type="text/plain",delivery_mode=1)
self.channel.basic_publish(exchange='tornado',
routing_key='tornado.*',
body = ws_msg,
properties=properties)
class PikaClient(object):
def __init__(self, config, app=None):
self.classifier = Classifier()
# Connection params
self.host = config['host'] or 'localhost'
self.port = config['port'] or '5672'
self.vhost = config['vhost'] or '/'
self.user = config['user'] or 'guest'
self.passwd = config['passwd'] or 'guest'
self.exchange = config['exchange'] or 'twitter2'
self.queue_name = config['queue_name'] or 'twitter_topic_feed'
self.routing_key = config['routing_key'] or 'twitter_topic_feed'
# Default values
self.connected = False
self.connecting = False
self.connection = None
self.channel = None
self.app = app
self.event_listeners = set([])
# A place for us to keep messages sent to us by Rabbitmq
self.messages = list()
# A place for us to put pending messages while we're waiting to connect
self.pending = list()
self.connect()
def connect(self):
if self.connecting:
print('PikaClient: Already connecting to RabbitMQ')
return
print('PikaClient: Connecting to RabbitMQ on %s:%i' \
% (self.host, self.port))
self.connecting = True
credentials = pika.PlainCredentials(self.user, self.passwd)
param = pika.ConnectionParameters(host=self.host, port=self.port,
virtual_host=self.vhost, credentials=credentials)
logging.debug('Events: Connecting to AMQP Broker: %s:%i' % (self.host,
self.port))
# from pika.adapters import SelectConnection
# connection = SelectConnection(parameters, on_connected)
self.connection = TornadoConnection(param,
on_open_callback=self.on_connected)
self.connection.add_on_close_callback(self.on_closed)
def on_connected(self, connection):
print('PikaClient: Connected to RabbitMQ on %s:%i' \
% (self.host, self.port))
self.connected = True
self.connection = connection
self.connection.channel(self.on_channel_open)
def on_channel_open(self, channel):
print('PikaClient: Channel Open, Declaring Exchange %s' \
% self.exchange)
self.channel = channel
self.channel.exchange_declare(exchange=self.exchange,
type="direct",
durable=False,
callback=self.on_exchange_declared)
def on_exchange_declared(self, frame):
print('PikaClient: Exchange Declared, Declaring Queue %s' \
% self.queue_name)
self.channel.queue_declare(queue=self.queue_name,
durable=False,
exclusive=False,
callback=self.on_queue_declared)
def on_queue_declared(self, frame):
print('PikaClient: Queue Declared, Binding Queue')
self.channel.queue_bind(exchange=self.exchange,
queue=self.queue_name,
routing_key=self.routing_key,
callback=self.on_queue_bound)
def on_queue_bound(self, frame):
print('PikaClient: Queue Bound, Issuing Basic Consume')
self.channel.basic_consume(consumer_callback=self.on_message,
queue=self.queue_name,
no_ack=True)
# Send any messages pending
for properties, body in self.pending:
self.channel.basic_publish(exchange=self.exchange,
routing_key=self.routing_key,
body=body,
properties=properties)
def on_basic_cancel(self, frame):
print('PikaClient: Basic Cancel Ok')
# If we don't have any more consumer processes running close
self.connection.close()
def on_closed(self, connection):
# We've closed our pika connection so stop the demo
tornado.ioloop.IOLoop.instance().stop()
def on_message(self, channel, method, header, body):
print('PikaCient: Message received: %s delivery tag #%i: %s' \
% (header.content_type, method.delivery_tag, body))
# Append it to our messages list
self.messages.append(body)
self.app.dispatcher.notifyCallbacks(body)
self.notify_listeners(body)
def notify_listeners(self, event_obj):
# here we assume the message the sourcing app
# post to the message queue is in JSON format
event_json = self.classify_text(event_obj)
for listener in self.event_listeners:
listener.write_message(event_json)
print('PikaClient: notified %s' % repr(listener))
def add_event_listener(self, listener):
self.event_listeners.add(listener)
print('PikaClient: listener %s added' % repr(listener))
def remove_event_listener(self, listener):
try:
self.event_listeners.remove(listener)
print('PikaClient: listener %s removed' % repr(listener))
except KeyError:
pass
def get_messages(self):
# Get the messages to return, then empty the list
output = self.messages
self.messages = list()
return output
def publish(self, msg):
# Build a message to publish to RabbitMQ
#body = '%.8f: Request from %s [%s]' % \
# (tornado_request._start_time,
# tornado_request.remote_ip,
# tornado_request.headers.get("User-Agent"))
# Send the message
properties = pika.BasicProperties(content_type="text/plain",
delivery_mode=1)
self.channel.basic_publish(exchange=self.exchange,
routing_key=self.routing_key,
#body='Message: %s - %s' % (msg, body),
body='Message: %s' % msg,
properties=properties)
def classify_text(self,event_json):
#Temporaray: Processing should happen in intermediary handler
#Parse JSON, return JSON with text geo and polarity
print("CLASSIFIER MESSAGE " + event_json)
parsedJson =json.loads(event_json)
tweettext = parsedJson['text']
tweetgeo = parsedJson['geo']
#tweetplace = parsedJson['place']
#tweetcoords = parsedJson['coordinates']
tweet_polarity = self.classifier.classify(tweettext)
return {'text':tweettext, 'geo': tweetgeo, 'polarity':tweet_polarity}#, 'place':tweetplace,'coordinates':tweetcoords}
class PikaClient(object):
def __init__(self, io_loop):
pika.log.info('PikaClient: __init__')
self.io_loop = io_loop
self.connected = False
self.connecting = False
self.connection = None
self.channel = None
self.event_listeners = set([])
def connect(self):
if self.connecting:
pika.log.info('PikaClient: Already connecting to RabbitMQ')
return
pika.log.info('PikaClient: Connecting to RabbitMQ')
self.connecting = True
cred = pika.PlainCredentials('guest', 'guest')
param = pika.ConnectionParameters(
host='localhost',
port=5672,
virtual_host='/',
credentials=cred
)
self.connection = TornadoConnection(param,
on_open_callback=self.on_connected)
self.connection.add_on_close_callback(self.on_closed)
def on_connected(self, connection):
pika.log.info('PikaClient: connected to RabbitMQ')
self.connected = True
self.connection = connection
self.connection.channel(self.on_channel_open)
def on_channel_open(self, channel):
pika.log.info('PikaClient: Channel open, Declaring exchange')
self.channel = channel
# declare exchanges, which in turn, declare
# queues, and bind exchange to queues
def on_closed(self, connection):
pika.log.info('PikaClient: rabbit connection closed')
self.io_loop.stop()
def on_message(self, channel, method, header, body):
pika.log.info('PikaClient: message received: %s' % body)
self.notify_listeners(event_factory(body))
def notify_listeners(self, event_obj):
# here we assume the message the sourcing app
# post to the message queue is in JSON format
event_json = json.dumps(event_obj)
for listener in self.event_listeners:
listener.write_message(event_json)
pika.log.info('PikaClient: notified %s' % repr(listener))
def add_event_listener(self, listener):
self.event_listeners.add(listener)
pika.log.info('PikaClient: listener %s added' % repr(listener))
def remove_event_listener(self, listener):
try:
self.event_listeners.remove(listener)
pika.log.info('PikaClient: listener %s removed' % repr(listener))
except KeyError:
pass
class PikaClient(object):
def __init__(self, io_loop):
pika.log.info('PikaClient: __init__')
self.io_loop = io_loop
self.connected = False
self.connecting = False
self.connection = None
self.channel = None
self.event_listeners = set([])
def connect(self):
if self.connecting:
pika.log.info('PikaClient: Already connecting to RabbitMQ')
return
pika.log.info('PikaClient: Connecting to RabbitMQ')
self.connecting = True
#cred = pika.PlainCredentials('guest', 'guest')
param = pika.ConnectionParameters(
host='115.146.93.175',
#port=5672,
#virtual_host='/',
#credentials=cred
)
self.connection = TornadoConnection(param,
on_open_callback=self.on_connected)
self.connection.add_on_close_callback(self.on_closed)
def on_connected(self, connection):
pika.log.info('PikaClient: connected to RabbitMQ')
self.connected = True
self.connection = connection
self.connection.channel(self.on_channel_open)
def on_channel_open(self, channel):
pika.log.info('PikaClient: Channel open, Declaring exchange')
self.channel = channel
self.channel.queue_declare(queue='hello')
self.channel.basic_consume(self.on_message, queue='hello', no_ack=True)
# declare exchanges, which in turn, declare
# queues, and bind exchange to queues
def on_closed(self, connection):
pika.log.info('PikaClient: rabbit connection closed')
self.io_loop.stop()
def on_message(self, channel, method, header, body):
pika.log.info('PikaClient: message received: %s' % body)
self.notify_listeners(body)
def send_message(self, body):
self.channel.basic_publish(exchange='', routing_key='hello', body=body)
def notify_listeners(self, body):
for listener in self.event_listeners:
listener.write_message(body)
pika.log.info('PikaClient: notified %s' % repr(listener))
def add_event_listener(self, listener):
print "added listener"
self.event_listeners.add(listener)
pika.log.info('PikaClient: listener %s added' % repr(listener))
def remove_event_listener(self, listener):
try:
self.event_listeners.remove(listener)
pika.log.info('PikaClient: listener %s removed' % repr(listener))
except KeyError:
pass
class PikaClient(object):
def __init__(self, settings):
self.settings = settings
# Construct a queue name we'll use for this instance only
self.queue_name = "tornado-test-%i" % os.getpid()
# Default values
self.connected = False
self.connecting = False
self.connection = None
self.channel = None
# A place for us to keep messages sent to us by Rabbitmq
self.messages = list()
# A place for us to put pending messages while we're waiting to connect
self.pending = list()
def connect(self):
if self.connecting:
pika.log.info("PikaClient: Already connecting to RabbitMQ")
return
pika.log.info("PikaClient: Connecting to RabbitMQ on localhost:5672")
self.connecting = True
credentials = pika.PlainCredentials(self.settings.user, self.settings.password)
param = pika.ConnectionParameters(
host=self.settings.host,
port=self.settings.port,
virtual_host=self.settings.virtual_host,
credentials=credentials,
)
self.connection = TornadoConnection(param, on_open_callback=self.on_connected)
self.connection.add_on_close_callback(self.on_closed)
def on_connected(self, connection):
pika.log.info("PikaClient: Connected to RabbitMQ on localhost:5672")
self.connected = True
self.connection = connection
self.connection.channel(self.on_channel_open)
def on_channel_open(self, channel):
pika.log.info("PikaClient: Channel Open, Declaring Exchange")
self.channel = channel
self.channel.exchange_declare(
exchange="tornado", type="direct", auto_delete=True, durable=False, callback=self.on_exchange_declared
)
def on_exchange_declared(self, frame):
pika.log.info("PikaClient: Exchange Declared, Declaring Queue")
self.channel.queue_declare(
queue=self.queue_name, auto_delete=True, durable=False, exclusive=False, callback=self.on_queue_declared
)
def on_queue_declared(self, frame):
pika.log.info("PikaClient: Queue Declared, Binding Queue")
self.channel.queue_bind(
exchange="tornado", queue=self.queue_name, routing_key="tornado.*", callback=self.on_queue_bound
)
def on_queue_bound(self, frame):
pika.log.info("PikaClient: Queue Bound, Issuing Basic Consume")
self.channel.basic_consume(consumer_callback=self.on_pika_message, queue=self.queue_name, no_ack=True)
# Send any messages pending
for properties, body in self.pending:
self.channel.basic_publish(exchange="tornado", routing_key="tornado.*", body=body, properties=properties)
def on_pika_message(self, channel, method, header, body):
pika.log.info("PikaCient: Message receive, delivery tag #%i" % method.delivery_tag)
# Append it to our messages list
self.messages.append(body)
# ==============================================================================
self.callback(body) # 3.use external callback(self.send) on here
def cb(self, callback): # 1. use app.pika.cb for setting external callback function
self.callback = callback # 2.assign external callback to self.callback
# ==============================================================================
def on_basic_cancel(self, frame):
pika.log.info("PikaClient: Basic Cancel Ok")
# If we don't have any more consumer processes running close
self.connection.close()
def on_closed(self, connection):
# TODO:We've closed our pika connection so stop the demo
tornado.ioloop.IOLoop.instance().stop()
def sample_message(self, msg):
# Build a message to publish to RabbitMQ
body = msg
# Send the message
properties = pika.BasicProperties(content_type="text/plain", delivery_mode=1)
self.channel.basic_publish(exchange="tornado", routing_key="tornado.*", body=body, properties=properties)
def get_messages(self):
# Get the messages to return, then empty the list
output = self.messages
self.messages = list()
return output
class SockjsServer(object):
def __init__(self, io_loop):
self.logger = logging.getLogger(__name__)
self.logger.info('SockjsServer: __init__')
self.io_loop = io_loop
self.connected = False
self.connecting = False
self.connection = None
self.channel = None
self.redis = redis_client
self.event_listeners_count = 0
self.event_listeners = set()
self.connection_dict = dict()
self.subscription_dict = defaultdict(set)
self.last_reconnect = now()
self.uptime_start = now()
self.config = SockJSServerSettings()
def connect(self):
if self.connecting:
self.logger.info('django-sockjs-server(SockjsServer): Already connecting to RabbitMQ')
return
self.logger.info('django-sockjs-server(SockjsServer): Connecting to RabbitMQ')
self.connecting = True
cred = pika.PlainCredentials(self.config.rabbitmq_user, self.config.rabbitmq_password)
param = pika.ConnectionParameters(
host=self.config.rabbitmq_host,
port=self.config.rabbitmq_port,
virtual_host=self.config.rabbitmq_vhost,
credentials=cred
)
try:
self.connection = TornadoConnection(param,
on_open_callback=self.on_connected)
self.connection.add_on_close_callback(self.on_closed)
except AMQPConnectionError:
self.logger.info('django-sockjs-server(SockjsServer): error connect, wait 5 sec')
time.sleep(5)
self.reconnect()
self.last_reconnect = now()
def on_connected(self, connection):
self.logger.info('django-sockjs-server(SockjsServer): connected to RabbitMQ')
self.connected = True
self.connection = connection
self.connection.channel(self.on_channel_open)
def on_channel_open(self, channel):
self.logger.info('django-sockjs-server(SockjsServer): Channel open, Declaring exchange')
self.channel = channel
self.channel.exchange_declare(exchange=self.config.rabbitmq_exchange_name,
exchange_type=self.config.rabbitmq_exchange_type)
self.channel.queue_declare(
queue=self.config.rabbitmq_queue_name,
exclusive=False,
auto_delete=True,
callback=self.on_queue_declared
)
def on_queue_declared(self, frame):
self.logger.info('django-sockjs-server(SockjsServer): queue bind')
self.queue = frame.method.queue
self.channel.queue_bind(callback=None, exchange=self.config.rabbitmq_exchange_name, queue=frame.method.queue)
self.channel.basic_consume(self.handle_delivery, queue=frame.method.queue, no_ack=True)
def handle_delivery(self, channel, method, header, body):
"""Called when we receive a message from RabbitMQ"""
self.notify_listeners(body)
def on_closed(self, connection, error_code, error_message):
self.logger.info('django-sockjs-server(SockjsServer): rabbit connection closed, wait 5 seconds')
connection.add_timeout(5, self.reconnect)
def reconnect(self):
self.connecting = False
self.logger.info('django-sockjs-server(SockjsServer): reconnect')
self.connect()
def notify_listeners(self, event_json):
event_obj = json.loads(event_json)
self.logger.debug('django-sockjs-server(SockjsServer): send message %s ' % event_obj)
try:
client = self.connection_dict[event_obj['uid']]
except KeyError:
self.redis.lrem(event_obj['room'], 0, json.dumps({'id': event_obj['uid'], 'host': event_obj['host']}))
else:
new_event_json = json.dumps({'data': event_obj['data']})
client.broadcast([client], new_event_json)
def add_event_listener(self, listener):
self.event_listeners_count += 1
self.event_listeners.add(listener)
self.logger.debug('django-sockjs-server(SockjsServer): listener %s added' % repr(listener))
def remove_event_listener(self, listener):
try:
self.event_listeners_count -= 1
self.event_listeners.remove(listener)
self.logger.debug('django-sockjs-server(SockjsServer): listener %s removed' % repr(listener))
except KeyError:
pass
def add_subscriber_room(self, room, conn):
try:
conn_id = conn.id
self.connection_dict.setdefault(conn_id, conn)
client = self.connection_dict[conn_id]
self.subscription_dict[conn_id].add(room)
self.logger.debug('django-sockjs-server(SockjsServer): listener %s add to room %s' % (repr(client), room))
except KeyError, exc:
pass
class PikaClient(object):
def __init__(self):
# Construct a queue name we'll use for this instance only
#Giving unique queue for each consumer under a channel.
self.queue_name = "queue-%s" % (id(self), )
# Default values
self.connected = False
self.connecting = False
self.connection = None
self.channel = None
#Webscoket object.
self.websocket = None
def connect(self):
if self.connecting:
print('PikaClient: Already connecting to RabbitMQ')
return
print(
'PikaClient: Connecting to RabbitMQ on localhost:5672, Object: %s'
% (self, ))
self.connecting = True
credentials = pika.PlainCredentials('guest', 'guest')
param = pika.ConnectionParameters(host='localhost',
port=5672,
virtual_host="/",
credentials=credentials)
self.connection = TornadoConnection(param,
on_open_callback=self.on_connected)
def on_connected(self, connection):
print('PikaClient: Connected to RabbitMQ on localhost:5672')
self.connected = True
self.connection = connection
self.connection.channel(self.on_channel_open)
def on_channel_open(self, channel):
print('PikaClient: Channel Open, Declaring Exchange, Channel ID: %s' %
(channel, ))
self.channel = channel
self.channel.exchange_declare(exchange='tornado',
exchange_type="direct",
auto_delete=True,
durable=False,
callback=self.on_exchange_declared)
def on_exchange_declared(self, frame):
print('PikaClient: Exchange Declared, Declaring Queue')
self.channel.queue_declare(auto_delete=True,
queue=self.queue_name,
durable=False,
exclusive=True,
callback=self.on_queue_declared)
def on_queue_declared(self, frame):
print('PikaClient: Queue Declared, Binding Queue')
self.channel.queue_bind(exchange='tornado',
queue=self.queue_name,
routing_key='tornado.*',
callback=self.on_queue_bound)
def on_queue_bound(self, frame):
print('PikaClient: Queue Bound, Issuing Basic Consume')
self.channel.basic_consume(consumer_callback=self.on_pika_message,
queue=self.queue_name,
no_ack=True)
def on_pika_message(self, channel, method, header, body):
print('PikaCient: Message receive, delivery tag #%i' %
method.delivery_tag)
#Send the Cosumed message via Websocket to browser.
self.websocket.write_message(body)
def on_basic_cancel(self, frame):
print('PikaClient: Basic Cancel Ok')
# If we don't have any more consumer processes running close
self.connection.close()
def on_closed(self, connection):
# We've closed our pika connection so stop the web chat
tornado.ioloop.IOLoop.instance().stop()
def chat_message(self, ws_msg):
#Publish the message from Websocket to RabbitMQ
properties = pika.BasicProperties(content_type="text/plain",
delivery_mode=1)
self.channel.basic_publish(exchange='tornado',
routing_key='tornado.*',
body=ws_msg,
properties=properties)
class PikaClient(object):
def __init__(self, ):
# Default values
self.connected = False
self.connecting = False
self.connection = None
self.channel = None
self.main_future = Future()
# A place for us to keep messages sent to us by Rabbitmq
self.messages = list()
def _make_response(code, message):
# TODO:
pass
def _get_callback_with_future(self, callback, future):
def _callback(*args, **kwargs):
res = callback(*args, **kwargs)
future.set_result(res)
return _callback
# -----------------------------------------------------
# Connection functions
# -----------------------------------------------------
# TODO: make this async
def connect(self):
if self.connecting:
logging.info('Already connecting to RabbitMQ')
return
logging.info('Connecting to RabbitMQ on localhost:5672')
self.connecting = True
credentials = pika.PlainCredentials('guest', 'guest')
param = pika.ConnectionParameters(host='localhost',
port=5672,
virtual_host="/",
credentials=credentials)
self.connection = TornadoConnection(param,
on_open_callback=self.on_connected)
self.connection.add_on_close_callback(self.on_closed)
def on_connected(self, connection):
logging.info('Connected to RabbitMQ on localhost:5672')
self.connected = True
self.connection = connection
self._init_channel()
def on_closed(self, connection):
# We've closed our pika connection so stop the demo
tornado.ioloop.IOLoop.instance().stop()
# -----------------------------------------------------
# Channel functions
# -----------------------------------------------------
@property
def channel(self):
if self._channel is None or not self._channel.is_open:
self._init_channel()
return self._channel
def _init_channel(self):
self.connection.channel(self.on_channel_open)
def on_channel_close(self, channel, reply_code, reply_text):
logging.info('Channel closed, reply_code=%s, reply_text=%s', reply_code,
reply_text)
self.main_future.set_result(reply_code)
def on_channel_open(self, channel):
logging.info('Channel Open')
self._channel = channel
self._channel.add_on_close_callback(self.on_channel_close)
# -----------------------------------------------------
# Queue functions
# -----------------------------------------------------
@async_func
def queue_declare(self, future, queue):
logging.info('Declaring Queue')
cb = self._get_callback_with_future(self._on_queue_declare_ok, future)
self.channel.queue_declare(queue=queue,
durable=True,
callback=cb)
def _on_queue_declare_ok(self, frame):
logging.info('Queue Declared')
return frame
@async_func
def queue_bind(self, future, exchange, queue, routing_key):
logging.info('Binding Queue %s to Exchange %s', queue, exchange)
cb = self._get_callback_with_future(self.on_queue_bind_ok, future)
self.channel.queue_bind(exchange=exchange, queue=queue,
routing_key=routing_key,
callback=cb)
def on_queue_bind_ok(self, frame):
logging.info('Queue Bound')
return frame
def get_messages():
self.channel.basic_consume(consumer_callback=self.on_pika_message,
queue=self.queue_name,
no_ack=True)
def on_pika_message(self, channel, method, header, body):
logging.info('Message receive, delivery tag #%i' % \
method.delivery_tag)
# Append it to our messages list
self.messages.append(body)
def on_basic_cancel(self, frame):
logging.info('Basic Cancel Ok')
# If we don't have any more consumer processes running close
self.connection.close()
def publish_message(self, exchange, queue):
# Build a message to publish to RabbitMQ
body = '%.8f: Request from %s [%s]' % \
(tornado_request._start_time,
tornado_request.remote_ip,
tornado_request.headers.get("User-Agent"))
# Send the message
properties = pika.BasicProperties(content_type="text/plain",
delivery_mode=1)
self.channel.basic_publish(exchange='tornado',
routing_key='tornado.*',
body=body,
properties=properties)
class QueueManager(object):
def __init__(self):
self.connection = None
self.channel = None
self._connected = False
self.event_listeners = {}
self.connections = {}
self.cache = None
self.run()
def run(self):
t1 = Thread(target=sd_redis.watch_redis)
t1.start()
while not getattr(sd_redis, 'redis_master'):
time.sleep(0.01)
self.cache = ZRedis(host=sd_redis.redis_master,
password=redis_db_pw,
db=os.getenv('REDIS_DB'))
self.connect()
def connect(self, host=None):
"""
Connection with Rabbit.
"""
if self._connected:
zlogger.info('PikaClient: Already connecting to RabbitMQ')
return
if not host:
_, host = list(json.loads(sd_rabbit.rabbit_nodes).items())[0]
zlogger.info('PikaClient: Connecting to RabbitMQ in Queue Manager')
param = ConnectionParameters(host=host,
port=5672,
virtual_host=VIRTUAL_HOST,
credentials=sd_rabbit.rabbit_credential)
self.connection = TornadoConnection(param,
on_open_callback=self.on_connected)
self.connection.add_on_close_callback(self.on_closed)
self._connected = True
zlogger.info("Connection is successful: host:{}".format(host))
def on_connected(self, connection):
"""
AMQP connection callback.
Creates input channel.
Args:
connection: AMQP connection
"""
zlogger.info('PikaClient: connected to RabbitMQ')
self.connection.channel(self.on_channel_open)
def on_channel_open(self, channel):
"""
Input channel creation callback, exchange declaring.
"""
zlogger.info('PikaClient: Channel open, Declaring exchange')
self.channel = channel
self.channel.exchange_declare(exchange='messages',
type='topic',
durable=True)
def on_message(self, channel, method, header, body):
"""
When message is received, prepare notifier list.
"""
user_id = method.consumer_tag
notify_list = self.event_listeners[user_id]
self.notify_listeners(body, notify_list)
def listen_messages(self, queue_name, user_id):
"""
Listen rabbit messages.
"""
self.channel.basic_consume(consumer_callback=self.on_message,
queue=queue_name,
consumer_tag=user_id,
no_ack=True)
def notify_listeners(self, message, notify_list):
"""
Write message to notifier list.
"""
for listener in notify_list:
listener.write_message(message)
def add_event_listener(self, listener, user_info):
"""
Add listener to user set. If queue creation is new,
"""
user_id = user_info.get("user")
if not self.event_listeners.get(user_id):
queue_name = self.get_queue_name(user_id)
self.channel.queue_declare(queue=queue_name,
auto_delete=True,
callback=None)
self.event_listeners.setdefault(user_id, []).append(listener)
self.input_queue_bind(queue_name, user_info)
self.listen_messages(queue_name, user_id)
self.cache.sadd('QueueList:{}'.format(user_id), queue_name)
else:
self.event_listeners[user_id].append(listener)
zlogger.info(
"New websocket connection is added for user which has user_id:{}".
format(user_id))
def remove_event_listener(self, listener, user_id):
"""
Remove listener from listener list.
"""
try:
if self.event_listeners.get(user_id):
self.event_listeners[user_id].remove(listener)
if not self.event_listeners[user_id]:
self.channel.queue_delete(
queue=self.get_queue_name(user_id))
del self.event_listeners[user_id]
except Exception as exc:
zlogger.error(
"An error occurred on remove_event_listener method inside QueueManager. "
"User Id: {}, Exception: {}".format(user_id, exc))
def input_queue_bind(self, queue, user_info):
"""
Input queue declaration callback.
Input Queue/Exchange binding done here
Args:
queue: input queue
user_info: user information dict include project, service and user ids
"""
bind_list = self.get_bind_list(user_info)
for route_key in bind_list:
self.channel.queue_bind(callback=None,
exchange='messages',
queue=queue,
routing_key=route_key)
def get_bind_list(self, user_info):
"""
Args:
user_info: user information dict include project, service and user ids
"""
user_id = user_info.get("user")
project_id = user_info.get("project")
service = user_info.get("service")
bind_list = [user_id]
# 'CACHE_SUBSCRIBER_CHANNELS' key is Channels list of a user
subs_channels_key = CACHE_SUBSCRIBER_CHANNELS.format(
project_id=project_id, service=service, subscriber_id=user_id)
channels = self.cache.smembers(subs_channels_key)
channels = [channel_id.decode() for channel_id in channels]
bind_list.extend(channels)
return bind_list
def on_closed(self, connection, _, __):
"""
Moves listeners from close node's queue manager to queue's new master node.
"""
self._connected = False
@staticmethod
def get_queue_name(user_id):
"""
Gets queue name according to user id.
"""
return "{}_{}".format(user_id, os.getpid())
class PikaClient(object):
def __init__(self, config, app=None):
# Connection params
self.host = config['host'] or 'localhost'
self.port = config['port'] or '5672'
self.vhost = config['vhost'] or '/'
self.user = config['user'] or 'guest'
self.passwd = config['passwd'] or 'guest'
self.exchange = config['exchange'] or 'myx'
self.queue_name = config['queue_name'] or 'tornado-test-%i' \
% os.getpid()
self.routing_key = config['routing_key'] or 'tornado.*'
# Default values
self.connected = False
self.connecting = False
self.connection = None
self.channel = None
self.app = app
# Set our pika.log options
pika.log.setup(color=True)
# A place for us to keep messages sent to us by Rabbitmq
self.messages = list()
# A place for us to put pending messages while we're waiting to connect
self.pending = list()
self.connect()
def connect(self):
if self.connecting:
pika.log.info('PikaClient: Already connecting to RabbitMQ')
return
pika.log.info('PikaClient: Connecting to RabbitMQ on %s:%i' \
% (self.host, self.port))
self.connecting = True
credentials = pika.PlainCredentials(self.user, self.passwd)
param = pika.ConnectionParameters(host=self.host, port=self.port,
virtual_host=self.vhost, credentials=credentials)
srs = SimpleReconnectionStrategy()
logging.debug('Events: Connecting to AMQP Broker: %s:%i' % (self.host,
self.port))
# from pika.adapters import SelectConnection
# connection = SelectConnection(parameters, on_connected)
self.connection = TornadoConnection(param, reconnection_strategy=srs,
on_open_callback=self.on_connected)
self.connection.add_on_close_callback(self.on_closed)
def on_connected(self, connection):
pika.log.info('PikaClient: Connected to RabbitMQ on %s:%i' \
% (self.host, self.port))
self.connected = True
self.connection = connection
self.connection.channel(self.on_channel_open)
def on_channel_open(self, channel):
pika.log.info('PikaClient: Channel Open, Declaring Exchange %s' \
% self.exchange)
self.channel = channel
self.channel.exchange_declare(exchange=self.exchange,
type="direct",
auto_delete=True,
durable=False,
callback=self.on_exchange_declared)
def on_exchange_declared(self, frame):
pika.log.info('PikaClient: Exchange Declared, Declaring Queue %s' \
% self.queue_name)
self.channel.queue_declare(queue=self.queue_name,
auto_delete=True,
durable=False,
exclusive=False,
callback=self.on_queue_declared)
def on_queue_declared(self, frame):
pika.log.info('PikaClient: Queue Declared, Binding Queue')
self.channel.queue_bind(exchange=self.exchange,
queue=self.queue_name,
routing_key=self.routing_key,
callback=self.on_queue_bound)
def on_queue_bound(self, frame):
pika.log.info('PikaClient: Queue Bound, Issuing Basic Consume')
self.channel.basic_consume(consumer_callback=self.on_message,
queue=self.queue_name,
no_ack=True)
# Send any messages pending
for properties, body in self.pending:
self.channel.basic_publish(exchange=self.exchange,
routing_key=self.routing_key,
body=body,
properties=properties)
def on_basic_cancel(self, frame):
pika.log.info('PikaClient: Basic Cancel Ok')
# If we don't have any more consumer processes running close
self.connection.close()
def on_closed(self, connection):
# We've closed our pika connection so stop the demo
tornado.ioloop.IOLoop.instance().stop()
def on_message(self, channel, method, header, body):
pika.log.info('PikaCient: Message received: %s delivery tag #%i: %s' \
% (header.content_type, method.delivery_tag, body))
# Append it to our messages list
self.messages.append(body)
self.app.dispatcher.notifyCallbacks(body)
def get_messages(self):
# Get the messages to return, then empty the list
output = self.messages
self.messages = list()
return output
def publish(self, msg):
# Build a message to publish to RabbitMQ
#body = '%.8f: Request from %s [%s]' % \
# (tornado_request._start_time,
# tornado_request.remote_ip,
# tornado_request.headers.get("User-Agent"))
# Send the message
properties = pika.BasicProperties(content_type="text/plain",
delivery_mode=1)
self.channel.basic_publish(exchange=self.exchange,
routing_key=self.routing_key,
#body='Message: %s - %s' % (msg, body),
body='Message: %s' % msg,
properties=properties)
class PikaClient(object):
def __init__(self, io_loop):
self.connected = False
self.connecting = False
self.connection = None
self.channel = None
self.ioloop = io_loop
def connect(self):
if self.connecting:
logger.info('PikaClient: Already connecting to RabbitMQ')
return
logger.info('PikaClient: Connecting to RabbitMQ on port 5672, Object: %s', self)
self.connecting = True
credentials = pika.PlainCredentials(RABBIT_USERNAME, RABBIT_PASS)
param = pika.ConnectionParameters(host=RABBIT_URL,
port=5672,
virtual_host=RABBIT_VHOST,
credentials=credentials)
self.connection = TornadoConnection(param,
on_open_callback=self.on_connected)
logger.info("Connection to rabbit")
global pika_connected
pika_connected = True
def on_connected(self, connection):
logger.info('PikaClient: Connected to RabbitMQ on :5672')
self.connected = True
self.connection = connection
self.connection.channel(self.on_channel_open)
def on_channel_open(self, channel):
logger.info('PikaClient: Channel Open, Declaring Exchange, Channel ID: %s', channel)
self.channel = channel
logger.info("setting channel")
logger.info(channel)
self.channel.exchange_declare(exchange='tornado',
type="direct",
durable=False,
auto_delete=True)
def declare_queue(self, token):
logger.info('PikaClient: Exchange Declared, Declaring Queue')
self.queue_name = token
self.channel.queue_declare(queue=self.queue_name,
durable=False,
auto_delete=True,
callback=self.on_queue_declared)
def on_queue_declared(self, frame):
self.channel.queue_bind(exchange='tornado',
queue=self.queue_name,
routing_key=self.queue_name,
callback=self.on_queue_bound)
def on_queue_bound(self, frame):
logger.info('PikaClient: Queue Bound, Issuing Basic Consume')
self.channel.basic_consume(consumer_callback=self.on_pika_message,
queue=self.queue_name,
no_ack=True)
def on_pika_message(self, channel, method, header, body):
logger.info('PikaCient: Message receive, delivery tag #%i', method.delivery_tag)
message = json.loads(body)
# TODO: decrement unread count
for i in websockets[message['token']]:
try:
if 'action' in message and message["action"] == "kick" and 'token_to_kick' in message:
if i.authentication_token == message["token_to_kick"]:
REDIS_CONNECTION.srem('%s-%s' % ('members', message['token']), message["token_to_kick"])
i.close()
continue
del message["token_to_kick"]
del message["action"]
body = json.dumps(message)
i.write_message(body)
except Exception as error:
logger.exception("exception while writing message to client: {}".format(error.message))
def on_basic_cancel(self, frame):
logger.info('PikaClient: Basic Cancel Ok')
# If we don't have any more consumer processes running close
self.connection.close()
def on_closed(self, connection):
# We've closed our pika connection so stop the demo
self.ioloop.IOLoop.instance().stop()
def sample_message(self, ws_msg):
logger.info(ws_msg)
logger.info("zaa")
token = json.loads(ws_msg)['token']
properties = pika.BasicProperties(
content_type="text/plain", delivery_mode=1)
logger.info(self.channel)
self.channel.basic_publish(exchange='tornado',
routing_key=token,
body=ws_msg,
properties=properties)
class PikaClient(object):
def __init__(self, io_loop, ank_accessor, host_address):
#pika.log.info('PikaClient: __init__')
self.io_loop = io_loop
self.connected = False
self.connecting = False
self.connection = None
self.channel = None
self.event_listeners = set([])
self.queue_name = 'webserver-%i' % os.getpid()
self.ank_accessor = ank_accessor
self.host_address = host_address
def connect(self):
if self.connecting:
#pika.log.info('PikaClient: Already connecting to RabbitMQ')
return
#pika.log.info('PikaClient: Connecting to RabbitMQ')
self.connecting = True
#cred = pika.PlainCredentials('guest', 'guest')
param = pika.ConnectionParameters(
host= self.host_address,
#port=5672,
#virtual_host='/',
#credentials=cred
)
try:
self.connection = TornadoConnection(param,
on_open_callback=self.on_connected)
self.connection.add_on_close_callback(self.on_closed)
except pika.exceptions.AMQPConnectionError:
print("Unable to connect to RabbitMQ")
def on_connected(self, connection):
#pika.log.info('PikaClient: connected to RabbitMQ')
self.connected = True
self.connection = connection
self.connection.channel(self.on_channel_open)
def on_channel_open(self, channel):
#pika.log.info('PikaClient: Channel open, Declaring exchange')
self.channel = channel
self.channel.exchange_declare(exchange='www',
type="direct",
callback=self.on_exchange_declared)
return
def on_exchange_declared(self, frame):
#pika.log.info('PikaClient: Exchange Declared, Declaring Queue')
self.channel.queue_declare(queue=self.queue_name,
auto_delete=True,
durable=False,
exclusive=False,
callback=self.on_queue_declared)
return
def on_queue_declared(self, frame):
#pika.log.info('PikaClient: Queue Declared, Binding Queue')
self.channel.queue_bind(exchange='www',
queue=self.queue_name,
routing_key='client',
callback=self.on_queue_bound)
def on_queue_bound(self, frame):
#pika.log.info('PikaClient: Queue Bound, Issuing Basic Consume')
self.channel.basic_consume(consumer_callback=self.on_message,
queue=self.queue_name,
no_ack=True)
def on_closed(self, connection):
#pika.log.info('PikaClient: rabbit connection closed')
self.io_loop.stop()
def on_message(self, channel, method, header, body):
#pika.log.info('PikaClient: message received: %s' % body)
import zlib
try:
body = zlib.decompress(body)
except zlib.error:
pass # likely not compressed body
body_parsed = json.loads(body)
if body_parsed.has_key("anm"):
print "Received updated network topology"
try:
self.ank_accessor.anm = body_parsed['anm']
#TODO: could process diff and only update client if data has changed -> more efficient client side
self.update_listeners("overlays")
# TODO: find better way to replace object not just local reference, as need to replace for RequestHandler too
except Exception, e:
print "Exception is", e
elif body_parsed.has_key("ip_allocations"):
alloc = json.loads(body_parsed['ip_allocations'])
self.ank_accessor.ip_allocation = alloc
self.update_listeners("ip_allocations")
class PikaPublisher(object):
def __init__(self, amqp_parameters, exchange, exchange_type, queue, routing_key):
self._exchange = exchange
self._exchange_type = exchange_type
self._queue = queue
self._routing_key = routing_key
self._connection = None
self._channel = None
self._deliveries = []
self._acked = 0
self._nacked = 0
self._message_number = 0
self._stopping = False
self._params = amqp_parameters
self._closing = False
logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
def connect(self):
self._connection = TornadoConnection(self._params, self.on_connection_open, stop_ioloop_on_close=False)
LOGGER.info('Adding connection close callback')
self._connection.add_on_close_callback(self.on_connection_closed)
return self._connection
def close_connection(self):
LOGGER.info('Closing connection')
self._closing = True
self._connection.close()
def on_connection_closed(self, connection, reply_code, reply_text):
self._channel = None
if self._closing:
self._connection.ioloop.stop()
else:
LOGGER.warning('Connection closed, reopening in 5 seconds: (%s) %s',
reply_code, reply_text)
self._connection.add_timeout(5, self.reconnect)
def on_connection_open(self, unused_connection):
LOGGER.info('Connection opened')
self.open_channel()
def reconnect(self):
# This is the old connection IOLoop instance, stop its ioloop
self._connection.ioloop.stop()
# Create a new connection
self._connection = self.connect()
# There is now a new connection, needs a new ioloop to run
self._connection.ioloop.start()
def add_on_channel_close_callback(self):
LOGGER.info('Adding channel close callback')
self._channel.add_on_close_callback(self.on_channel_closed)
def on_channel_closed(self, channel, reply_code, reply_text):
LOGGER.warning('Channel was closed: (%s) %s', reply_code, reply_text)
if not self._closing:
self._connection.close()
def on_channel_open(self, channel):
LOGGER.info('Channel opened')
self._channel = channel
self.add_on_channel_close_callback()
self.setup_exchange(self._exchange)
def setup_exchange(self, exchange_name):
LOGGER.info('Declaring exchange %s', exchange_name)
self._channel.exchange_declare(self.on_exchange_declareok,
exchange_name,
self._exchange_type, durable=True)
def on_exchange_declareok(self, unused_frame):
LOGGER.info('Exchange declared')
self.setup_queue(self._queue)
def setup_queue(self, queue_name):
LOGGER.info('Declaring queue %s', queue_name)
self._channel.queue_declare(self.on_queue_declareok, queue_name, durable = True)
def on_queue_declareok(self, method_frame):
LOGGER.info('Binding %s to %s with %s',
self._exchange, self._queue, self._routing_key)
self._channel.queue_bind(self.on_bindok, self._queue,
self._exchange, self._routing_key)
def on_delivery_confirmation(self, method_frame):
confirmation_type = method_frame.method.NAME.split('.')[1].lower()
LOGGER.info('Received %s for delivery tag: %i',
confirmation_type,
method_frame.method.delivery_tag)
if confirmation_type == 'ack':
self._acked += 1
elif confirmation_type == 'nack':
self._nacked += 1
self._deliveries.remove(method_frame.method.delivery_tag)
LOGGER.info('Published %i messages, %i have yet to be confirmed, '
'%i were acked and %i were nacked',
self._message_number, len(self._deliveries),
self._acked, self._nacked)
def enable_delivery_confirmations(self):
LOGGER.info('Issuing Confirm.Select RPC command')
self._channel.confirm_delivery(self.on_delivery_confirmation)
def publish_message(self, message):
if self._stopping:
return
properties = pika.BasicProperties(app_id='example-publisher',
content_type='application/json',
delivery_mode = 2,
headers=message)
self._channel.basic_publish(self._exchange, self._routing_key,
json.dumps(message, ensure_ascii=False),
properties)
self._message_number += 1
self._deliveries.append(self._message_number)
LOGGER.info('Published message # %i', self._message_number)
def on_bindok(self, unused_frame):
LOGGER.info('Queue bound')
def close_channel(self):
LOGGER.info('Closing the channel')
if self._channel:
self._channel.close()
def open_channel(self):
LOGGER.info('Creating a new channel')
self._connection.channel(on_open_callback=self.on_channel_open)
def stop(self):
LOGGER.info('Stopping')
self._stopping = True
self.close_channel()
self.close_connection()
self._connection.ioloop.start()
LOGGER.info('Stopped')
class TornadoQueueConnection(object):
def __init__(self, host, user='******', password='******', vhost='/'):
if tornado is None:
raise Exception('You must add tornado to your requirements!')
if pika is None:
raise Exception('You must add pika to your requirements!')
self._parameters = pika.ConnectionParameters(
host=host,
credentials=pika.PlainCredentials(user, password),
virtual_host=vhost
)
self._connection = None
self._channel = None
self.ioloop = tornado.ioloop.IOLoop.instance()
self.ioloop.add_timeout(0, self._connect)
self._delivery_tag = 0
self._confirmation_callbacks = {}
self.client_map = {}
def publish(self, exchange, routing_key, headers, body, callback):
properties = pika.BasicProperties(content_type='text/plain')
if self._connection is None or self._connection.is_closed:
self._connect()
callback(False)
if self._channel is None or self._channel.is_closed:
self._open_channel()
callback(False)
self._channel.basic_publish(exchange, routing_key, body, properties)
self._delivery_tag += 1
self._confirmation_callbacks[self._delivery_tag] = callback
def publish_json(self, exchange, routing_key, headers, body, callback):
data = ujson.dumps(body)
self.publish(exchange, routing_key, headers, data, callback)
def _on_delivery_confirmation(self, method_frame):
confirmation_type = method_frame.method.NAME.split('.')[1].lower()
tag = method_frame.method.delivery_tag
if confirmation_type == 'ack':
success = True
else:
success = False
callback = self._confirmation_callbacks[tag]
del self._confirmation_callbacks[tag]
callback(success)
def close(self):
self._connection.close()
def _connect(self):
self.connection = TornadoConnection(
self._parameters,
on_open_callback=self._on_connected,
stop_ioloop_on_close=False,
)
def _on_connected(self, connection):
self._connection = connection
self._connection.add_on_close_callback(self._on_connection_closed)
self._open_channel()
def _on_connection_closed(self, method_frame):
self._connection = None
self._connect()
def _open_channel(self):
self.connection.channel(self._on_channel_open)
def _on_channel_open(self, channel):
self._channel = channel
#self._channel.confirm_delivery(self._on_delivery_confirmation)
self._channel.queue_declare(self.callback_queue_declare, exclusive=True)
def on_response(self, ch, method, props, body):
print 'on_response called'
client = self.client_map[props.correlation_id]
if client != None:
#print 'on_response ' + props.type + ' ' + body
client.write_message(body)
del self.client_map[props.correlation_id]
def call(self, client, cmd_type, cmd_body):
corr_id = str(uuid.uuid4())
self.client_map[corr_id] = client
self._channel.basic_publish(exchange='easy_net_q_rpc',
routing_key=cmd_type, #'Packet.LoginCommand:Packet',
properties=pika.BasicProperties(
type = cmd_type, #'Packet.LoginCommand:Packet',
reply_to = self.callback_queue,
correlation_id = corr_id,
),
body=cmd_body.encode('utf-8')) #'{"id":"gb", "pw":"pw"}')
def callback_queue_declare(self, result):
print 'callback_queue_declare called'
self.callback_queue = result.method.queue
self._channel.basic_consume(self.on_response, no_ack=True,
queue=self.callback_queue)
class PikaClient(object):
tornado_callback = None
_closing = False
_connect_index = 0
_connect_pull = None
_one_respond_made = None
_waiting_to_reconnect = 3
_expire_reconnect = 15
_last_reconnect_fail = None
def __init__(self, logger,
queue_answer, queue_read, queue_create,
node_list):
# Construct a queue name we'll use for this instance only
self.queue_answer = queue_answer
# Create queue for sending
self.queue_read = queue_read
self.queue_create = queue_create
self.logger = logger
self.connected = False
self.connecting = False
self.connection = None
self.channel = None
# A place for us to keep messages sent to us by Rabbitmq
self.messages = list()
# A place for us to put pending messages while we're waiting to connect
self.pending = list()
self._connect_pull = []
for node in node_list:
self._connect_pull.append(
pika.ConnectionParameters(
host=node[0], port=int(node[1])))
def connect(self):
if self.connecting:
self.logger.warning('Already connecting to RabbitMQ')
return
param = self._connect_pull[self._connect_index]
self.logger.debug('Connecting to RabbitMQ on '
'{host}:{port}'.format(host=param.host,
port=param.port))
self.connecting = True
# TODO: add on_connection_error
try:
self.connection = TornadoConnection(
param,
on_open_callback=self.on_connected
)
self.connection.add_on_close_callback(self.on_closed)
except AMQPConnectionError:
self.reconnect()
def on_connected(self, connection):
self.logger.debug('Connected to RabbitMQ on: '
'{connection}'.format(connection=connection))
self.connected = True
self.connection = connection
self.connection.channel(self.on_channel_open)
def on_channel_open(self, channel):
self.logger.debug('Channel Open, Declaring Exchange')
self.channel = channel
self.channel.exchange_declare(exchange='tornado',
type='topic',
durable=True,
callback=self.on_exchange_declared)
def on_exchange_declared(self, frame):
self.logger.debug('Exchange Declared, Declaring Queue')
self.channel.queue_declare(queue=self.queue_answer,
durable=True,
callback=self.on_queue_declared)
self.channel.queue_declare(queue=self.queue_create,
durable=True,
callback=lambda frame:
self.channel.queue_bind(
exchange='tornado',
queue=self.queue_create,
routing_key=self.queue_create,
callback=None))
self.channel.queue_declare(queue=self.queue_read,
durable=True,
callback=lambda frame:
self.channel.queue_bind(
exchange='tornado',
queue=self.queue_read,
routing_key=self.queue_read,
callback=None))
def on_queue_declared(self, frame):
self.logger.debug('Queue Declared, Binding Queue')
self.channel.queue_bind(exchange='tornado',
queue=self.queue_answer,
callback=self.on_queue_bound)
def on_queue_bound(self, frame):
self.logger.debug('Queue Bound, Issuing Basic Consume')
self.channel.basic_consume(consumer_callback=self.on_pika_message,
queue=self.queue_answer,
no_ack=True)
# TODO: still not implemented
# for properties, body in self.pending:
# self.logger.debug('Pending Message:'
# ' %s | %s' % (properties, body))
# self.channel.basic_publish(exchange='tornado',
# # TODO: save routing_key or
# # it already in properties
# routing_key='reading',
# body=body,
# properties=properties)
def on_pika_message(self, channel, method, header, body):
self.logger.debug('Message receive: '
'body: {body}'.format(method=method,
header=header,
body=body))
if self.tornado_callback and not self._one_respond_made:
self.messages.append(body)
self.tornado_callback(self.get_messages())
self._one_respond_made = True
def on_basic_cancel(self, frame):
self.logger.debug('Basic Cancel Ok')
self.connection.close()
def on_closed(self, *args):
self.logger.warning('On closed. Try to reconnect...')
self.reconnect()
def reconnect(self):
self.logger.warning('Fail to connect')
if self._last_reconnect_fail:
current_fail_time = datetime.datetime.now()
fail_timedelta = current_fail_time - self._last_reconnect_fail
self.logger.debug('Check reconnect expires, '
'timedelta: %s seconds' % fail_timedelta.seconds)
if fail_timedelta.seconds >= self._expire_reconnect:
self._closing = False
self._connect_index = 0
self._last_reconnect_fail = datetime.datetime.now()
self.logger.debug('Reconnect time is expired, '
'reset reconnect parameters')
else:
self.logger.warning('Set first fail reconnect time')
self._last_reconnect_fail = datetime.datetime.now()
self.connecting = False
self.connected = False
self.logger.warning('%s sec waiting...' % self._waiting_to_reconnect)
time.sleep(self._waiting_to_reconnect)
if not self._closing:
self._connect_index += 1
self.logger.warning('Try to reconnect to %s, try number %s' %
(self._connect_pull[self._connect_index],
self._connect_index))
if self._connect_index == len(self._connect_pull) - 1:
self._closing = True
self.connect()
else:
self.logger.warning('Closing. Stop trying')
self.stop()
def stop(self):
self.logger.warning('STOP')
self._closing = True
tornado.ioloop.IOLoop.instance().stop()
def sample_message(self, msg, routing_key, tornado_callback):
self.logger.debug('Sample Message to %s' % routing_key)
self.tornado_callback = tornado_callback
properties = pika.BasicProperties(delivery_mode=1)
self._one_respond_made = False
self.channel.basic_publish(exchange='tornado',
routing_key=routing_key,
body=msg,
properties=properties)
def get_messages(self):
output = self.messages
self.messages = list()
return output
class PikaConsumer(object):
EXCHANGE_TYPE = "topic"
def __init__(self,
io_loop="",
amqp_url="",
exchange="",
queue_name="",
routing_key=""):
self._io_loop = io_loop
self._connection = None
self._closing = False
self._channel = None
self._url = amqp_url
self._consumer_tag = None
self.QUEUE = queue_name or "default_queue"
self.ROUTING_KEY = routing_key or "default.get_routing_key"
self.EXCHANGE = exchange or "default_exchange"
LOG.info("消费者绑定: 交换机{},类型{},队列名字{},路由前缀{}".format(
self.EXCHANGE, self.EXCHANGE_TYPE, self.QUEUE, self.ROUTING_KEY))
def connect(self):
"""连接到rabbitmq, 赋值给self._connection, 并调用self._on_connection_open方法"""
self._connection = TornadoConnection(
parameters=pika.URLParameters(self._url),
on_open_callback=self._on_connection_open)
def close_connection(self):
self._connection.close()
def _on_connection_open(self, unused_connection):
"""当rabbitmq被打开时, 设置基本信息"""
self._add_on_connection_close_callback()
self._open_channel()
def _add_on_connection_close_callback(self):
"""当rabbitmq意外关闭与发布者的连接的时候, 调用关闭连接"""
self._connection.add_on_close_callback(self._on_connection_closed)
def _on_connection_closed(self, connection, reason):
"""当rabbitmq意外关闭的时候尝试重新连接"""
self._channel = None
if self._closing:
self._io_loop.stop()
else:
LOG.warning("connection close, reopening in 5 seconds: %s", reason)
self._connection.ioloop.call_later(5, self._reconnect)
def _reconnect(self):
"""重新连接"""
if not self._closing:
self._connection = self.connect()
def _open_channel(self):
"""发布channel, 打开一个新的rabbitmq渠道...成功的时候将调用self._on_channel_open"""
self._connection.channel(on_open_callback=self._on_channel_open)
def _on_channel_open(self, channel):
"""当通道打开的时候, 我们申明使用的交换信息"""
self._channel = channel
self._add_on_channel_close_callback()
self._set_exchange(self.EXCHANGE)
def _add_on_channel_close_callback(self):
"""设置通道关闭回调"""
self._channel.add_on_close_callback(self._on_channel_closed)
def _on_channel_closed(self, channel, reason):
self._connection.close()
def _set_exchange(self, exchange_name):
LOG.info('Declaring exchange %s', exchange_name)
self._channel.exchange_declare(callback=self._on_exchange_declareok,
exchange=exchange_name,
exchange_type=self.EXCHANGE_TYPE)
def _on_exchange_declareok(self, unused_frame):
self._setup_queue(self.QUEUE)
def _setup_queue(self, queue_name):
self._channel.queue_declare(callback=self._on_queue_declareok,
queue=queue_name)
def _on_queue_declareok(self, method_frame):
self._channel.queue_bind(callback=self._on_bindok,
queue=self.QUEUE,
exchange=self.EXCHANGE,
routing_key=self.ROUTING_KEY)
def _on_bindok(self, unused_frame):
self._start_consuming()
def _start_consuming(self):
self._add_on_cancel_callback()
self._consumer_tag = self._channel.basic_consume(
on_message_callback=self._on_message, queue=self.QUEUE)
def _add_on_cancel_callback(self):
self._channel.add_on_cancel_callback(self._on_consumer_cancelled)
def _on_consumer_cancelled(self, method_frame):
if self._channel:
self._channel.close()
def _on_message(self, channel, basic_deliver, properties, body):
"""
作为消费者接受消息,并做处理
"""
# LOG.info("rabbitmq get body: %s", str(body))
code, return_data = self.handler_body(body)
return_exchange = properties.message_id
return_routing_key = properties.reply_to
# return_correlation_id = properties.correlation_id
if (return_exchange and return_routing_key and return_data and code):
# LOG.info(
# "send msg to rabbitmq: exchange: %s, routing_key: %s, body: %s",
# return_exchange, return_routing_key, return_data)
# props = pika.BasicProperties(correlation_id=return_correlation_id)
channel.basic_publish(exchange=return_exchange,
body=json.dumps(return_data),
routing_key=return_routing_key)
elif code:
LOG.warning("request properties info: %s",
[return_exchange, return_routing_key])
if code:
self._acknowledge_message(basic_deliver.delivery_tag)
def _acknowledge_message(self, delivery_tag):
self._channel.basic_ack(delivery_tag)
def handler_body(self, body):
return True, "ok"
class PikaClient(object):
def __init__(self):
# Construct a queue name we'll use for this instance only
#Giving unique queue for each consumer under a channel.
self.queue_name = "/4EAqhJ1FaJuuGc07qS3KwZZamZypZDMVfpx/TIO/CaqNjhuV7+36tEUCIdzborOFqtcxSBvgjW0Ywe+s0Sj9Q6+t+TfRk44kxZpw8Eyg8H6LcKbiUzWmZqMIfdsTUfqRWLaAj2UN2ZVHf6hmDyyOEbEGTl6t0qUXXNJDMaSlBLVAqsDAQAB" #"queue-%s" % (id(self),)
# Default values
self.connected = False
self.connecting = False
self.connection = None
self.channel = None
#Webscoket object.
self.websocket = None
def connect(self):
if self.connecting:
pika.log.info('PikaClient: Already connecting to RabbitMQ')
return
pika.log.info(
'PikaClient: Connecting to RabbitMQ on localhost:5672, Object: %s'
% (self, ))
self.connecting = True
credentials = pika.PlainCredentials('guest', 'guest')
param = pika.ConnectionParameters(host='pepperjack.stanford.edu',
port=5672,
virtual_host="/",
credentials=credentials)
self.connection = TornadoConnection(param,
on_open_callback=self.on_connected)
#Currently this will close tornado ioloop.
#self.connection.add_on_close_callback(self.on_closed)
def on_connected(self, connection):
pika.log.info('PikaClient: Connected to RabbitMQ on localhost:5672')
self.connected = True
self.connection = connection
self.connection.channel(self.on_channel_open)
def on_channel_open(self, channel):
pika.log.info(
'PikaClient: Channel Open, Declaring Exchange, Channel ID: %s' %
(channel, ))
self.channel = channel
self.channel.exchange_declare(exchange='tornado',
type="direct",
auto_delete=True,
durable=False,
callback=self.on_exchange_declared)
def on_exchange_declared(self, frame):
pika.log.info('PikaClient: Exchange Declared, Declaring Queue')
pika.log.info(self.queue_name)
self.channel.queue_declare(auto_delete=False,
queue=self.queue_name,
durable=True,
exclusive=False,
callback=self.on_queue_declared)
def on_queue_declared(self, frame):
pika.log.info('PikaClient: Queue Declared, Binding Queue')
self.channel.queue_bind(exchange='tornado',
queue=self.queue_name,
routing_key='tornado.*',
callback=self.on_queue_bound)
def on_queue_bound(self, frame):
pika.log.info('PikaClient: Queue Bound, Issuing Basic Consume')
self.channel.basic_consume(consumer_callback=self.on_pika_message,
queue=self.queue_name,
no_ack=True)
def on_pika_message(self, channel, method, header, body):
pika.log.info('PikaCient: Message receive, delivery tag #%i' % \
method.delivery_tag)
#Send the Cosumed message via Websocket to browser.
self.websocket.write_message(base64.encodestring(body))
def on_basic_cancel(self, frame):
pika.log.info('PikaClient: Basic Cancel Ok')
# If we don't have any more consumer processes running close
self.connection.close()
def on_closed(self, connection):
# We've closed our pika connection so stop the demo
tornado.ioloop.IOLoop.instance().stop()
def sample_message(self, ws_msg):
#Publish the message from Websocket to RabbitMQ
properties = pika.BasicProperties(content_type="text/plain",
delivery_mode=1)
self.channel.basic_publish(exchange='tornado',
routing_key='tornado.*',
body=ws_msg,
properties=properties)
class BasePikaClient(object):
"""
Базовый класс для работы с очередями
"""
WEBTODATA_QUEUE_NAME = 'web_to_data'
DATATOWEB_QUEUE_NAME = 'data_to_web'
def __init__(self, amqp_url):
self._amqp_url = amqp_url
def connect(self):
self.connection = TornadoConnection(pika.URLParameters(self._amqp_url),
on_open_callback=self.on_connected)
def on_connected(self, connection):
self.connection = connection
self.connection.channel(self.on_channel_open)
def on_channel_open(self, channel):
"""
Действия при создании канала.
Потомки должны расширить этот метод,
добавив создание очередей и/или точек обмена.
"""
self.channel = channel
self.channel.add_on_cancel_callback(self.on_cancelled)
def on_cancelled(self, method_frame):
if self.channel:
self.channel.close()
def declare_queue(self, queue_name, on_queue_declare_ok=None):
on_queue_declare_ok = on_queue_declare_ok or self.on_queue_declare_ok
self.channel.queue_declare(on_queue_declare_ok, queue_name)
def on_queue_declare_ok(self, method_frame):
"""
Действия при создании очереди.
Потомки могут переопределить этот метод,
добавив создание привязок очередей к точкам обмена.
"""
pass
def declare_exchange(self, exhange_name, exhange_type):
self._channel.exchange_declare(self.on_exchange_declare_ok,
exhange_name, exhange_type)
def on_exchange_declare_ok(self, frame):
"""
Действия при создании точки обмена.
Потомки могут переопределить этот метод,
добавив создание очередей.
"""
pass
def send(self, exchange_name, queue_name, message):
self.channel.basic_publish(exchange_name, queue_name, message)
class PikaClient(object):
def __init__(self, io_loop):
logger.info('Initializing')
self.io_loop = io_loop
self.connected = False
self.connecting = False
self.connection = None
self.channel = None
self.event_listeners = set([])
def connect(self):
if self.connecting:
logger.info('Already connecting to RabbitMQ')
return
logger.info('Connecting to RabbitMQ')
self.connecting = True
cred = pika.PlainCredentials(settings.BAGOU.get('AMQP_BROKER_USER'),
settings.BAGOU.get('AMQP_BROKER_PASS'))
param = pika.ConnectionParameters(
host=settings.BAGOU.get('AMQP_BROKER_ADDR'),
port=settings.BAGOU.get('AMQP_BROKER_PORT'),
virtual_host=settings.BAGOU.get('AMQP_BROKER_PATH'),
credentials=cred)
self.connection = TornadoConnection(param,
on_open_callback=self.on_connected)
self.connection.add_on_close_callback(self.on_closed)
def on_connected(self, connection):
logger.info('Connected to RabbitMQ')
self.connected = True
self.connection = connection
self.connection.channel(self.on_channel_open)
def on_channel_open(self, channel):
logger.info('Channel open, Declaring exchange')
self.channel = channel
self.channel.queue_declare(queue=settings.BAGOU.get('QUEUE_NAME'),
durable=True,
exclusive=False,
auto_delete=True,
callback=self.on_queue_declared)
def on_queue_declared(self, frame):
self.channel.basic_consume(self.on_message,
queue=settings.BAGOU.get('QUEUE_NAME'))
def on_closed(self, connection):
logger.info('RabbitMQ connection closed')
self.io_loop.stop()
def on_message(self, channel, method, header, body):
logger.debug('Message received: %s' % body)
self.notify_listeners(body)
def notify_listeners(self, event_obj):
event_json = json.loads(event_obj)
channels = event_json.get('channel')
for channel in channels:
for listener in self.event_listeners:
if channel:
if channel in listener.channels:
listener.write_message(event_obj)
logger.info('Notified %s (channels: %s)' %
(repr(listener), listener.channels))
else:
listener.write_message(event_obj)
logger.info('Notified %s' % repr(listener))
def add_event_listener(self, listener):
self.event_listeners.add(listener)
logger.info('Listener %s added' % repr(listener))
def remove_event_listener(self, listener):
try:
self.event_listeners.remove(listener)
logger.info('Listener %s removed' % repr(listener))
except KeyError:
pass
class PikaClient(object):
def __init__(self, io_loop):
self.logger = logging.getLogger(__name__)
self.logger.info('PikaClient: __init__')
self.io_loop = io_loop
self.connected = False
self.connecting = False
self.connection = None
self.channel = None
self.redis = redis_client
self.event_listeners_count = 0
self.event_listeners = set()
self.subscrib_channel = dict()
self.last_reconnect = now()
self.uptime_start = now()
self.config = SockJSServerSettings()
def connect(self):
if self.connecting:
self.logger.info(
'django-sockjs-server(PikaClient): Already connecting to RabbitMQ'
)
return
self.logger.info(
'django-sockjs-server(PikaClient): Connecting to RabbitMQ')
self.connecting = True
cred = pika.PlainCredentials(self.config.rabbitmq_user,
self.config.rabbitmq_password)
param = pika.ConnectionParameters(
host=self.config.rabbitmq_host,
port=self.config.rabbitmq_port,
virtual_host=self.config.rabbitmq_vhost,
credentials=cred)
try:
self.connection = TornadoConnection(
param, on_open_callback=self.on_connected)
self.connection.add_on_close_callback(self.on_closed)
except AMQPConnectionError:
self.logger.info(
'django-sockjs-server(PikaClient): error connect, wait 5 sec')
time.sleep(5)
self.reconnect()
self.last_reconnect = now()
def on_connected(self, connection):
self.logger.info(
'django-sockjs-server(PikaClient): connected to RabbitMQ')
self.connected = True
self.connection = connection
self.connection.channel(self.on_channel_open)
def on_channel_open(self, channel):
self.logger.info(
'django-sockjs-server(PikaClient): Channel open, Declaring exchange'
)
self.channel = channel
self.channel.exchange_declare(
exchange=self.config.rabbitmq_exchange_name,
exchange_type=self.config.rabbitmq_exchange_type)
self.channel.queue_declare(queue=self.config.rabbitmq_queue_name,
exclusive=False,
auto_delete=True,
callback=self.on_queue_declared)
def on_queue_declared(self, frame):
self.logger.info('django-sockjs-server(PikaClient): queue bind')
self.queue = frame.method.queue
self.channel.queue_bind(callback=None,
exchange=self.config.rabbitmq_exchange_name,
queue=frame.method.queue)
self.channel.basic_consume(self.handle_delivery,
queue=frame.method.queue,
no_ack=True)
def handle_delivery(self, channel, method, header, body):
"""Called when we receive a message from RabbitMQ"""
self.notify_listeners(body)
def on_closed(self, connection, error_code, error_message):
self.logger.info(
'django-sockjs-server(PikaClient): rabbit connection closed, wait 5 seconds'
)
connection.add_timeout(5, self.reconnect)
def reconnect(self):
self.connecting = False
self.logger.info('django-sockjs-server(PikaClient): reconnect')
self.connect()
def notify_listeners(self, event_json):
event_obj = json.loads(event_json)
self.logger.debug(
'django-sockjs-server(PikaClient): send message %s ' % event_obj)
try:
channel = self.subscrib_channel[event_obj['uid']]
except KeyError:
self.redis.lrem(
event_obj['room'], 0,
json.dumps({
'id': event_obj['uid'],
'host': event_obj['host']
}))
else:
client = channel['conn']
new_event_json = json.dumps({'data': event_obj['data']})
client.broadcast([client], new_event_json)
def add_event_listener(self, listener):
self.event_listeners_count += 1
self.event_listeners.add(listener)
self.logger.debug(
'django-sockjs-server(PikaClient): listener %s added' %
repr(listener))
def remove_event_listener(self, listener):
try:
self.event_listeners_count -= 1
self.event_listeners.remove(listener)
self.logger.debug(
'django-sockjs-server(PikaClient): listener %s removed' %
repr(listener))
except KeyError:
pass
def add_subscriber_channel(self, conn_id, room, client):
self.subscrib_channel[conn_id] = {'room': room, 'conn': client}
self.logger.debug(
'django-sockjs-server(PikaClient): listener %s add to room %s' %
(repr(client), room))
def remove_subscriber_channel(self, conn_id, client):
try:
room = self.subscrib_channel[conn_id].get('room')
del self.subscrib_channel[conn_id]
self.logger.debug(
'django-sockjs-server(PikaClient): listener %s del connection %s from room %s'
% (repr(client), conn_id, room))
except KeyError:
pass
def get_event_listeners_count(self):
return self.event_listeners_count
def get_subscribe_channel_count(self):
return len(self.subscrib_channel.keys())
def get_subscribe_channels(self):
return self.subscrib_channel.keys()
def get_last_reconnect(self):
return self.last_reconnect
def get_uptime(self):
return (now() - self.uptime_start).seconds
class AQMPMultiCastAsyncRxQueueManager(RxMultiDelegateQueueWrapper):
def __init__(self, io_loop, exchange):
super().__init__()
self.logger = logging.getLogger(__name__)
self.logger.info('AQMPMultiCastAsyncRxQueueManager: __init__')
self.io_loop = io_loop
self.exchange = exchange
self.connected = False
self.connecting = False
self.connection = None
self.channel = None
def start(self, routing_key):
if self.connecting:
self.logger.info('AQMPMultiCastAsyncRxQueueManager: Already connecting to RabbitMQ')
return
self.routing_key = routing_key
self.logger.info('AQMPMultiCastAsyncRxQueueManager: Connecting to RabbitMQ')
self.connecting = True
cred = pika.PlainCredentials('guest', 'guest')
param = pika.ConnectionParameters(
host='localhost',
port=5672,
virtual_host='/',
credentials=cred
)
self.connection = TornadoConnection(param, on_open_callback=self.on_connected)
self.connection.add_on_close_callback(self.on_closed)
self.io_loop.start()
# Callback 1
def on_connected(self, connection):
self.logger.info('AQMPMultiCastAsyncRxQueueManager: connected to RabbitMQ')
self.connected = True
self.connection = connection
self.connection.channel(self.on_channel_open)
# Callback 2
def on_channel_open(self, channel):
self.logger.info('AQMPMultiCastAsyncRxQueueManager: Channel open, Declaring exchange')
self.channel = channel
self.channel.queue_declare(exclusive=True, callback=self.on_queue_declared)
# Callback 3
def on_queue_declared(self, frame):
"""Called when RabbitMQ has told us our Queue has been declared, frame is the response from RabbitMQ"""
self.queue_name = frame.method.queue
self.channel.queue_bind(self.on_queue_bind, exchange=self.exchange, queue=self.queue_name, routing_key=self.routing_key)
# Callback 4
def on_queue_bind(self, frame):
self.channel.basic_consume(self.on_message, queue=self.queue_name, no_ack=True)
def on_closed(self, connection):
self.logger.info('AQMPMultiCastAsyncRxQueueManager: rabbit connection closed')
self.io_loop.stop()
def on_message(self, channel, method, header, body):
self.logger.info('AQMPMultiCastAsyncRxQueueManager: message received: %s' % body)
self.notify_delegates(body)
class SockjsServer(object):
def __init__(self, io_loop):
self.logger = logging.getLogger(__name__)
self.logger.info('SockjsServer: __init__')
self.io_loop = io_loop
self.connected = False
self.connecting = False
self.connection = None
self.channel = None
self.redis = redis_client
self.event_listeners_count = 0
self.event_listeners = set()
self.connection_dict = dict()
self.subscription_dict = defaultdict(set)
self.last_reconnect = now()
self.uptime_start = now()
self.config = SockJSServerSettings()
def connect(self):
if self.connecting:
self.logger.info(
'django-sockjs-server(SockjsServer): Already connecting to RabbitMQ'
)
return
self.logger.info(
'django-sockjs-server(SockjsServer): Connecting to RabbitMQ')
self.connecting = True
cred = pika.PlainCredentials(self.config.rabbitmq_user,
self.config.rabbitmq_password)
param = pika.ConnectionParameters(
host=self.config.rabbitmq_host,
port=self.config.rabbitmq_port,
virtual_host=self.config.rabbitmq_vhost,
credentials=cred)
try:
self.connection = TornadoConnection(
param, on_open_callback=self.on_connected)
self.connection.add_on_close_callback(self.on_closed)
except AMQPConnectionError:
self.logger.info(
'django-sockjs-server(SockjsServer): error connect, wait 5 sec'
)
time.sleep(5)
self.reconnect()
self.last_reconnect = now()
def on_connected(self, connection):
self.logger.info(
'django-sockjs-server(SockjsServer): connected to RabbitMQ')
self.connected = True
self.connection = connection
self.connection.channel(self.on_channel_open)
def on_channel_open(self, channel):
self.logger.info(
'django-sockjs-server(SockjsServer): Channel open, Declaring exchange'
)
self.channel = channel
self.channel.exchange_declare(
exchange=self.config.rabbitmq_exchange_name,
exchange_type=self.config.rabbitmq_exchange_type)
self.channel.queue_declare(
queue=self.config.rabbitmq_queue1_name, # declare MQ1
exclusive=False,
auto_delete=True,
callback=self.on_queue1_declared)
self.channel.queue_declare(
queue=self.config.rabbitmq_queue2_name, # declare MQ2
exclusive=False,
auto_delete=True,
callback=self.on_queue2_declared)
# queue binding
def on_queue1_declared(self, frame):
# self.logger.info(frame)
self.logger.info('django-sockjs-server(SockjsServer): queue1 bind')
# self.queue = frame.method.queue
self.channel.queue_bind(callback=None,
exchange=self.config.rabbitmq_exchange_name,
queue=frame.method.queue)
self.channel.basic_consume(self.handle_delivery,
queue=frame.method.queue,
no_ack=True)
def on_queue2_declared(self, frame):
self.logger.info('django-sockjs-server(SockjsServer): queue2 bind')
self.queue = frame.method.queue # declare the host--MQ2
# self.logger.info(self.queue)
self.channel.queue_bind(callback=None,
exchange=self.config.rabbitmq_exchange_name,
queue=frame.method.queue)
self.channel.basic_consume(self.handle_delivery,
queue=frame.method.queue,
no_ack=True)
def handle_delivery(self, channel, method, header, body):
"""Called when we receive a message from RabbitMQ"""
# call zhenfeng's function(body: message) here without caring the return
# to simulate
json_obj = json.loads(body.decode())
# self.logger.info('json_obj >>> :', json_obj)
if json_obj['host'] == 'MQ1':
self.logger.info("host = MQ1!!!")
from django_sockjs_server.lib.client import SockJsServerClient
s = SockJsServerClient()
json_obj['host'] = 'MQ2'
s.publish_message(json_obj, 'MQ2') # put into MQ2
self.logger.info("Has put into MQ2")
else:
self.logger.info("host = MQ2!!!")
self.notify_listeners(body)
def handle_delivery2(self, channel, method, header, body: bytes):
"""
Called when we receive a message from RabbitMQ
body like this: (b'{"data": {"user_name": "Sergey Kravchuk", "user_id": 1},
"host": "MQ2", "uid": "262ef88da87537086e0d93f9fc2c40ba", "room": "user2"}',)
"""
# json_obj = json.loads(body.decode())
# self.logger.info('handle_delivery2 body >>> :', json_obj, json_obj['host'])
# self.logger.info('handle_delivery2 body >>> :', type(channel) , type(method), type(header))
self.notify_listeners(body)
def on_closed(self, connection, error_code, error_message):
self.logger.info(
'django-sockjs-server(SockjsServer): rabbit connection closed, wait 5 seconds'
)
connection.add_timeout(5, self.reconnect)
def reconnect(self):
self.connecting = False
self.logger.info('django-sockjs-server(SockjsServer): reconnect')
self.connect()
def notify_listeners(self, event_json):
event_obj = json.loads(event_json)
self.logger.debug(
'django-sockjs-server(SockjsServer): send message %s ' % event_obj)
try:
client = self.connection_dict[event_obj['uid']]
except KeyError:
self.redis.lrem(
event_obj['room'], 0,
json.dumps({
'id': event_obj['uid'],
'host': event_obj['host']
}))
else:
new_event_json = json.dumps({'data': event_obj['data']})
client.broadcast([client], new_event_json)
def add_event_listener(self, listener):
self.event_listeners_count += 1
self.event_listeners.add(listener)
self.logger.debug(
'django-sockjs-server(SockjsServer): listener %s added' %
repr(listener))
def remove_event_listener(self, listener):
try:
self.event_listeners_count -= 1
self.event_listeners.remove(listener)
self.logger.debug(
'django-sockjs-server(SockjsServer): listener %s removed' %
repr(listener))
except KeyError:
pass
def add_subscriber_room(self, room, conn):
try:
conn_id = conn.id
self.connection_dict.setdefault(conn_id, conn)
client = self.connection_dict[conn_id]
self.subscription_dict[conn_id].add(room)
self.logger.debug(
'django-sockjs-server(SockjsServer): listener %s add to room %s'
% (repr(client), room))
except KeyError as exc:
pass
def remove_subscriber(self, conn_id):
try:
client = self.connection_dict[conn_id]
del self.subscription_dict[conn_id]
del self.connection_dict[conn_id]
self.logger.debug(
'django-sockjs-server(SockjsServer): listener %s del connection %s'
% (repr(client), conn_id))
except KeyError as exc:
pass
def get_event_listeners_count(self):
return self.event_listeners_count
def get_subscribe_connection_count(self):
return len(self.connection_dict.keys())
def get_subscribe_connections(self):
return self.connection_dict.keys()
def get_last_reconnect(self):
return self.last_reconnect
def get_uptime(self):
return (now() - self.uptime_start).seconds
class MQConnection(object):
"""
MQ连接管理类
"""
def __init__(self, url, type='producer', callback=None, *arg, **settings):
"""Create a new instance of the MQConnection class, passing in the AMQP
URL used to connect to RabbitMQ.
:param str amqp_url: The AMQP url to connect with
:param str type: connection type,for excmple,'consumer','producer'
:param str callback: if type is 'consumer',callback is not None
"""
self._connection = None
self._channel = None
self._closing = False
self._consumer_tag = None
self._url = url
self._type = type
self._was_consuming = False
self._was_publishing = False
self._reconnect_delay = 0
self._callback = callback
self.EXCHANGE = settings.get('exchange')
self.QUEUE = settings.get('queue')
self.ROUTING_KEY = settings.get('routing_key')
self.EXCHANGE_TYPE = settings.get('exchange_type')
self.AE_EXCHANGE = settings.get('ae_exchange')
self.AE_QUEUE = settings.get('ae_queue')
self.AE_EXCHANGE_TYPE = settings.get('ae_exchange_type')
self.DL_EXCHANGE = settings.get('dl_exchange')
self.DL_QUEUE = settings.get('dl_queue')
self.DL_EXCHANGE_TYPE = settings.get('dl_exchange_type')
self._passive = settings.get('passive', True)
self._durable = settings.get('durable', True)
self._prefetch_count = settings.get('prefetch_count', 128)
@exception_catch
def connect(self):
"""This method connects to RabbitMQ, returning the connection handle.
When the connection is established, the on_connection_open method
will be invoked by pika.
:rtype: pika.SelectConnection
"""
LOGGER.info('Connecting to %s', self._url)
self._connection = TornadoConnection(
pika.URLParameters(self._url),
on_open_callback=self.on_connection_open,
on_open_error_callback=self.on_connection_open_error)
return self._connection
def on_connection_open_error(self, _unused_connection, err):
"""This method is called by pika if the connection to RabbitMQ
can't be established.
:param pika.SelectConnection _unused_connection: The connection
:param Exception err: The error
"""
reconnect_delay = self._get_reconnect_delay()
LOGGER.error('Connection open failed, reopening in %d seconds: %s',
reconnect_delay, err)
self._connection.ioloop.call_later(reconnect_delay, self.reconnect)
def close_connection(self):
"""This method closes the connection to RabbitMQ."""
LOGGER.info('Closing connection')
self._connection.close()
def add_on_connection_close_callback(self):
"""This method adds an on close callback that will be invoked by pika
when RabbitMQ closes the connection to the publisher unexpectedly.
"""
LOGGER.info('Adding connection close callback')
self._connection.add_on_close_callback(self.on_connection_closed)
def on_connection_closed(self, connection, reason):
"""This method is invoked by pika when the connection to RabbitMQ is
closed unexpectedly. Since it is unexpected, we will reconnect to
RabbitMQ if it disconnects.
:param pika.connection.Connection connection: The closed connection obj
:param Exception reason: exception representing reason for loss of
connection.
"""
self._channel = None
if self._closing:
pass
# self._connection.ioloop.stop()
else:
reconnect_delay = self._get_reconnect_delay()
LOGGER.warning('Connection closed, reopening in %d seconds: %s',
reconnect_delay, reason)
self._connection.ioloop.call_later(reconnect_delay, self.reconnect)
def on_connection_open(self, unused_connection):
"""This method is called by pika once the connection to RabbitMQ has
been established. It passes the handle to the connection object in
case we need it, but in this case, we'll just mark it unused.
:param pika.SelectConnection _unused_connection: The connection
"""
LOGGER.info('Connection opened')
self.add_on_connection_close_callback()
self.open_channel()
def reconnect(self):
"""Will be invoked by the IOLoop timer if the connection is
closed. See the on_connection_closed method.
"""
self._was_consuming = False
self._was_publishing = False
if not self._closing:
# Create a new connection
self._connection = self.connect()
def add_on_channel_close_callback(self):
"""This method tells pika to call the on_channel_closed method if
RabbitMQ unexpectedly closes the channel.
"""
LOGGER.info('Adding channel close callback')
self._channel.add_on_close_callback(self.on_channel_closed)
def on_channel_closed(self, channel, reason):
"""Invoked by pika when RabbitMQ unexpectedly closes the channel.
Channels are usually closed if you attempt to do something that
violates the protocol, such as re-declare an exchange or queue with
different parameters. In this case, we'll close the connection
to shutdown the object.
:param pika.channel.Channel: The closed channel
:param Exception reason: why the channel was closed
"""
LOGGER.warning('Channel %i was closed: %s', channel, reason)
if self._connection.is_open:
self._connection.close()
def on_channel_open(self, channel):
"""This method is invoked by pika when the channel has been opened.
The channel object is passed in so we can make use of it.
Since the channel is now open, we'll declare the exchange to use.
:param pika.channel.Channel channel: The channel object
"""
LOGGER.info('Channel opened')
self._channel = channel
self.add_on_channel_close_callback()
self.setup_exchange(self.EXCHANGE)
if self.AE_EXCHANGE:
self.setup_ae_exchange(self.AE_EXCHANGE)
if self.DL_EXCHANGE:
self.setup_dl_exchange(self.DL_EXCHANGE)
@exception_catch
def setup_exchange(self, exchange_name):
"""Setup the exchange on RabbitMQ by invoking the Exchange.Declare RPC
command. When it is complete, the on_exchange_declareok method will
be invoked by pika.
:param str|unicode exchange_name: The name of the exchange to declare
"""
cb = functools.partial(self.on_exchange_declareok,
userdata=exchange_name)
args = {}
if self.AE_EXCHANGE:
args['alternate-exchange'] = self.AE_EXCHANGE
self._channel.exchange_declare(passive=self._passive,
durable=self._durable,
exchange=exchange_name,
exchange_type=self.EXCHANGE_TYPE,
arguments=args,
callback=cb)
@exception_catch
def setup_ae_exchange(self, exchange_name):
"""Setup the exchange on RabbitMQ by invoking the Exchange.Declare RPC
command. When it is complete, the on_exchange_declareok method will
be invoked by pika.
:param str|unicode exchange_name: The name of the exchange to declare
"""
ae_cb = functools.partial(self.on_ae_exchange_declareok,
userdata=exchange_name)
self._channel.exchange_declare(passive=self._passive,
durable=False,
exchange=exchange_name,
exchange_type=self.AE_EXCHANGE_TYPE,
arguments={},
callback=ae_cb)
@exception_catch
def setup_dl_exchange(self, exchange_name):
"""Setup the exchange on RabbitMQ by invoking the Exchange.Declare RPC
command. When it is complete, the on_exchange_declareok method will
be invoked by pika.
:param str|unicode exchange_name: The name of the exchange to declare
"""
cb = functools.partial(self.on_dl_exchange_declareok,
userdata=exchange_name)
self._channel.exchange_declare(passive=self._passive,
durable=False,
exchange=exchange_name,
exchange_type=self.DL_EXCHANGE_TYPE,
arguments={},
callback=cb)
def on_exchange_declareok(self, _unused_frame, userdata):
"""Invoked by pika when RabbitMQ has finished the Exchange.Declare RPC
command.
:param pika.Frame.Method unused_frame: Exchange.DeclareOk response frame
:param str|unicode userdata: Extra user data (exchange name)
"""
LOGGER.info('Exchange declared: %s', userdata)
self.setup_queue(self.QUEUE)
def on_ae_exchange_declareok(self, _unused_frame, userdata):
"""Invoked by pika when RabbitMQ has finished the Exchange.Declare RPC
command.
:param pika.Frame.Method unused_frame: Exchange.DeclareOk response frame
:param str|unicode userdata: Extra user data (exchange name)
"""
LOGGER.info('Exchange declared: %s', userdata)
self.setup_ae_queue(self.AE_QUEUE)
def on_dl_exchange_declareok(self, _unused_frame, userdata):
"""Invoked by pika when RabbitMQ has finished the Exchange.Declare RPC
command.
:param pika.Frame.Method unused_frame: Exchange.DeclareOk response frame
:param str|unicode userdata: Extra user data (exchange name)
"""
LOGGER.info('Exchange declared: %s', userdata)
self.setup_dl_queue(self.DL_QUEUE)
@exception_catch
def setup_queue(self, queue_name):
"""Setup the queue on RabbitMQ by invoking the Queue.Declare RPC
command. When it is complete, the on_queue_declareok method will
be invoked by pika.
:param str|unicode queue_name: The name of the queue to declare.
"""
LOGGER.info('Declaring queue %s', queue_name)
if self._type == 'consumer' and self.EXCHANGE_TYPE == 'x-modulus-hash':
if not self._was_consuming:
self.start_consuming()
else:
args = {}
if self.DL_EXCHANGE:
args['x-dead-letter-exchange'] = self.DL_EXCHANGE
self._channel.queue_declare(durable=self._durable,
passive=self._passive,
queue=queue_name,
arguments=args,
callback=self.on_queue_declareok)
@exception_catch
def setup_ae_queue(self, queue_name):
"""Setup the queue on RabbitMQ by invoking the Queue.Declare RPC
command. When it is complete, the on_queue_declareok method will
be invoked by pika.
:param str|unicode queue_name: The name of the queue to declare.
"""
LOGGER.info('Declaring queue %s', queue_name)
self._channel.queue_declare(durable=False,
passive=self._passive,
queue=queue_name,
callback=self.on_ae_queue_declareok)
@exception_catch
def setup_dl_queue(self, queue_name):
"""Setup the queue on RabbitMQ by invoking the Queue.Declare RPC
command. When it is complete, the on_queue_declareok method will
be invoked by pika.
:param str|unicode queue_name: The name of the queue to declare.
"""
LOGGER.info('Declaring queue %s', queue_name)
self._channel.queue_declare(durable=False,
passive=self._passive,
queue=queue_name,
callback=self.on_dl_queue_declareok)
@exception_catch
def on_queue_declareok(self, _unused_frame):
"""Method invoked by pika when the Queue.Declare RPC call made in
setup_queue has completed. In this method we will bind the queue
and exchange together with the routing key by issuing the Queue.Bind
RPC command. When this command is complete, the on_bindok method will
be invoked by pika.
:param pika.frame.Method method_frame: The Queue.DeclareOk frame
"""
LOGGER.info('Binding %s to %s with %s', self.EXCHANGE, self.QUEUE,
self.ROUTING_KEY)
self._channel.queue_bind(self.QUEUE,
self.EXCHANGE,
routing_key=self.ROUTING_KEY,
callback=self.on_bindok)
@exception_catch
def on_ae_queue_declareok(self, _unused_frame):
"""Method invoked by pika when the Queue.Declare RPC call made in
setup_queue has completed. In this method we will bind the queue
and exchange together with the routing key by issuing the Queue.Bind
RPC command. When this command is complete, the on_bindok method will
be invoked by pika.
:param pika.frame.Method method_frame: The Queue.DeclareOk frame
"""
LOGGER.info('Binding %s to %s with %s', self.AE_EXCHANGE,
self.AE_QUEUE, self.ROUTING_KEY)
self._channel.queue_bind(self.AE_QUEUE,
self.AE_EXCHANGE,
routing_key=self.ROUTING_KEY,
callback=self.on_bindok)
@exception_catch
def on_dl_queue_declareok(self, _unused_frame):
"""Method invoked by pika when the Queue.Declare RPC call made in
setup_queue has completed. In this method we will bind the queue
and exchange together with the routing key by issuing the Queue.Bind
RPC command. When this command is complete, the on_bindok method will
be invoked by pika.
:param pika.frame.Method method_frame: The Queue.DeclareOk frame
"""
LOGGER.info('Binding %s to %s with %s', self.DL_EXCHANGE,
self.DL_QUEUE, self.ROUTING_KEY)
self._channel.queue_bind(self.DL_QUEUE,
self.DL_EXCHANGE,
routing_key=self.ROUTING_KEY,
callback=self.on_bindok)
def on_bindok(self, unused_frame):
"""Invoked by pika when the Queue.Bind method has completed. At this
point we will start consuming messages by calling start_consuming
which will invoke the needed RPC commands to start the process.
:param pika.frame.Method unused_frame: The Queue.BindOk response frame
"""
LOGGER.info('Queue bound')
if self._type == 'consumer':
if not self._was_consuming:
self.start_consuming()
else:
if not self._was_publishing:
self.start_publishing()
@exception_catch
def stop_consuming(self):
"""Tell RabbitMQ that you would like to stop consuming by sending the
Basic.Cancel RPC command.
"""
if self._channel:
LOGGER.info('Sending a Basic.Cancel RPC command to RabbitMQ')
self._channel.basic_cancel(self.on_cancelok, self._consumer_tag)
@exception_catch
def start_consuming(self):
"""This method sets up the consumer by first calling
add_on_cancel_callback so that the object is notified if RabbitMQ
cancels the consumer. It then issues the Basic.Consume RPC command
which returns the consumer tag that is used to uniquely identify the
consumer with RabbitMQ. We keep the value to use it when we want to
cancel consuming. The on_message method is passed in as a callback pika
will invoke when a message is fully received.
"""
LOGGER.info('start consuming')
self._was_consuming = True
self.add_on_cancel_callback()
self._channel.basic_qos(prefetch_count=self._prefetch_count)
self._consumer_tag = self._channel.basic_consume(
self.QUEUE, self._callback)
def add_on_cancel_callback(self):
"""Add a callback that will be invoked if RabbitMQ cancels the consumer
for some reason. If RabbitMQ does cancel the consumer,
on_consumer_cancelled will be invoked by pika.
"""
LOGGER.info('Adding consumer cancellation callback')
self._channel.add_on_cancel_callback(self.on_consumer_cancelled)
def on_consumer_cancelled(self, method_frame):
"""Invoked by pika when RabbitMQ sends a Basic.Cancel for a consumer
receiving messages.
:param pika.frame.Method method_frame: The Basic.Cancel frame
"""
LOGGER.info('Consumer was cancelled remotely, shutting down: %r',
method_frame)
if self._channel:
self._channel.close()
def start_publishing(self):
"""This method will enable delivery confirmations and schedule the
first message to be sent to RabbitMQ
"""
LOGGER.info('start publishing')
self._was_publishing = True
self.enable_delivery_confirmations()
# self.schedule_next_message()
def enable_delivery_confirmations(self):
"""Send the Confirm.Select RPC method to RabbitMQ to enable delivery
confirmations on the channel. The only way to turn this off is to close
the channel and create a new one.
When the message is confirmed from RabbitMQ, the
on_delivery_confirmation method will be invoked passing in a Basic.Ack
or Basic.Nack method from RabbitMQ that will indicate which messages it
is confirming or rejecting.
"""
LOGGER.info('Issuing Confirm.Select RPC command')
self._channel.confirm_delivery(self._callback)
def close_channel(self):
"""Call to close the channel with RabbitMQ cleanly by issuing the
Channel.Close RPC command.
"""
LOGGER.info('Closing the channel')
self._channel.close()
@exception_catch
def open_channel(self):
"""Open a new channel with RabbitMQ by issuing the Channel.Open RPC
command. When RabbitMQ responds that the channel is open, the
on_channel_open callback will be invoked by pika.
"""
LOGGER.info('Creating a new channel')
self._connection.channel(on_open_callback=self.on_channel_open)
def get_channel(self):
"""return _channel.
"""
return self._channel
def get_connection(self):
"""return _connection.
"""
return self._connection
def stop(self):
"""Cleanly shutdown the connection to RabbitMQ by stopping the consumer
with RabbitMQ. When RabbitMQ confirms the cancellation, on_cancelok
will be invoked by pika, which will then closing the channel and
connection. The IOLoop is started again because this method is invoked
when CTRL-C is pressed raising a KeyboardInterrupt exception. This
exception stops the IOLoop which needs to be running for pika to
communicate with RabbitMQ. All of the commands issued prior to starting
the IOLoop will be buffered but not processed.
"""
self._closing = True
LOGGER.info('Stopped')
def _get_reconnect_delay(self):
if self._was_consuming:
self._reconnect_delay = 0
else:
self._reconnect_delay += 1
if self._reconnect_delay > 30:
self._reconnect_delay = 30
return self._reconnect_delay
class PikaClient(object):
def __init__(self, io_loop):
logger.info('Initializing')
self.io_loop = io_loop
self.connected = False
self.connecting = False
self.connection = None
self.channel = None
self.event_listeners = set([])
def connect(self):
if self.connecting:
logger.info('Already connecting to RabbitMQ')
return
logger.info('Connecting to RabbitMQ')
self.connecting = True
cred = pika.PlainCredentials(
settings.BAGOU.get('AMQP_BROKER_USER'),
settings.BAGOU.get('AMQP_BROKER_PASS'))
param = pika.ConnectionParameters(
host=settings.BAGOU.get('AMQP_BROKER_ADDR'),
port=settings.BAGOU.get('AMQP_BROKER_PORT'),
virtual_host=settings.BAGOU.get('AMQP_BROKER_PATH'),
credentials=cred)
self.connection = TornadoConnection(param, on_open_callback=self.on_connected)
self.connection.add_on_close_callback(self.on_closed)
def on_connected(self, connection):
logger.info('Connected to RabbitMQ')
self.connected = True
self.connection = connection
self.connection.channel(self.on_channel_open)
def on_channel_open(self, channel):
logger.info('Channel open, Declaring exchange')
self.channel = channel
self.channel.queue_declare(
queue=settings.BAGOU.get('QUEUE_NAME'),
durable=True,
exclusive=False,
auto_delete=True,
callback=self.on_queue_declared)
def on_queue_declared(self, frame):
self.channel.basic_consume(self.on_message, queue=settings.BAGOU.get('QUEUE_NAME'))
def on_closed(self, connection):
logger.info('RabbitMQ connection closed')
self.io_loop.stop()
def on_message(self, channel, method, header, body):
logger.debug('Message received: %s' % body)
self.notify_listeners(body)
def notify_listeners(self, event_obj):
event_json = json.loads(event_obj)
channels = event_json.get('channel')
for channel in channels:
for listener in self.event_listeners:
if channel:
if channel in listener.channels:
listener.write_message(event_obj)
logger.info('Notified %s (channels: %s)' % (repr(listener), listener.channels))
else:
listener.write_message(event_obj)
logger.info('Notified %s' % repr(listener))
def add_event_listener(self, listener):
self.event_listeners.add(listener)
logger.info('Listener %s added' % repr(listener))
def remove_event_listener(self, listener):
try:
self.event_listeners.remove(listener)
logger.info('Listener %s removed' % repr(listener))
except KeyError:
pass
class PikaClient(object):
def __init__(self, config: dict, exchanges: dict) -> None:
self.connecting = False
self.connection = None
self.channel = None
self.config = config
self.exchanges = exchanges
def connect(self):
if self.connecting:
logging.info('connecting - so not re-connecting')
return
self.connecting = True
host = self.config.get('host')
port = 5671 if self.config.get('amqps') else 5672
scheme = 'amqps' if self.config.get('amqps') else 'amqp'
virtual_host = urllib.parse.quote(self.config.get('vhost'), safe='')
user = self.config.get('user')
pw = self.config.get('pw')
heartbeat = self.config.get('heartbeat') if self.config.get('heartbeat') else 0
params = pika.URLParameters(
f"{scheme}://{user}:{pw}@{host}:{port}/{virtual_host}?heartbeat={heartbeat}"
)
self.connection = TornadoConnection(params)
self.connection.add_on_open_callback(self.on_connect)
self.connection.add_on_close_callback(self.on_closed)
self.connection.add_on_open_error_callback(self.on_open_error_callback)
self.connecting = False
return
def on_open_error_callback(self, connection: TornadoConnection, exception: Exception) -> None:
logging.error('could not connect')
def on_connect(self, connection: TornadoConnection) -> None:
self.connection = connection
self.channel = self.connection.channel(
on_open_callback=self.on_channel_open
)
return
def on_channel_open(self, channel: pika.channel.Channel) -> None:
for backend, config in self.exchanges.items():
ex_name = config.get('exchange')
channel.exchange_declare(
ex_name,
exchange_type='topic',
durable=True,
)
logging.info(f'rabbitmq exchange: {ex_name} declared')
return
def on_basic_cancel(self, frame: pika.frame.Frame) -> None:
self.connection.close()
def on_closed(self, connection: TornadoConnection, exception: Exception) -> None:
logging.info('rabbitmq connection closed')
logging.info(exception)
def publish_message(
self,
*,
exchange: str,
routing_key: str,
method: str,
uri: str,
version: str,
data: dict,
persistent: bool = True,
timestamp: int = int(time.time()),
) -> None:
"""
Publilsh a message to an exchange.
Parameters
----------
exchange: str, exchange name
routing_key: str, routing key for topic exchange
method: str, HTTP method
uri: str, HTTP request URI
version: str, e.g. v1
data: dict
persistent: bool, default True
tell rabbitmq to persist messages to disk, or not
"""
data = {
'method': method,
'uri': uri,
'version': version,
'data': data,
}
message = json.dumps(data)
delivery_mode = 2 if persistent else 1
self.channel.basic_publish(
exchange=exchange,
routing_key=routing_key,
body=message,
properties=pika.BasicProperties(
content_type='application/json',
delivery_mode=delivery_mode,
timestamp=timestamp,
message_id=str(uuid.uuid4()),
)
)
return
class PikaClient(object):
def __init__(self, chat_id):
# Construct a queue name we'll use for this instance only
# Giving unique queue for each consumer under a channel.
self.queue_name = "queue-%s" % (id(self),)
# Default values
self.connected = False
self.connecting = False
self.connection = None
self.channel = None
self.chat_id = chat_id
# Webscoket object.
self.websocket = None
def connect(self):
if self.connecting:
print('PikaClient: Already connecting to RabbitMQ')
return
print('PikaClient: Connecting to RabbitMQ on localhost:5672, Object: %s' % (self,))
self.connecting = True
credentials = pika.PlainCredentials('guest', 'guest')
param = pika.ConnectionParameters(host='localhost',
port=5672,
virtual_host="/",
credentials=credentials)
self.connection = TornadoConnection(param,
on_open_callback=self.on_connected)
# Currently this will close tornado ioloop.
# self.connection.add_on_close_callback(self.on_closed)
def on_connected(self, connection):
print('PikaClient: Connected to RabbitMQ on localhost:5672')
self.connected = True
self.connection = connection
self.connection.channel(self.on_channel_open)
def on_channel_open(self, channel):
print('PikaClient: Channel Open, Declaring Exchange, Channel ID: %s' %
(channel,))
self.channel = channel
self.channel.exchange_declare(exchange='chat',
type="direct",
callback=self.on_exchange_declared)
def on_exchange_declared(self, frame):
print('PikaClient: Exchange Declared, Declaring Queue')
self.channel.queue_declare(queue=self.queue_name,
exclusive=True,
callback=self.on_queue_declared)
def on_queue_declared(self, frame):
print('PikaClient: Queue Declared, Binding Queue')
self.channel.queue_bind(exchange='chat',
queue=self.queue_name,
routing_key=self.chat_id,
callback=self.on_queue_bound)
def on_queue_bound(self, frame):
print('PikaClient: Queue Bound, Issuing Basic Consume')
self.channel.basic_consume(consumer_callback=self.on_pika_message,
queue=self.queue_name,
no_ack=True)
def on_pika_message(self, channel, method, header, body):
print('PikaCient: Message receive, delivery tag #%i' %
method.delivery_tag)
# Send the Cosumed message via Websocket to browser.
self.websocket.write_message(body.decode('utf8'))
def on_basic_cancel(self, frame):
print('PikaClient: Basic Cancel Ok')
# If we don't have any more consumer processes running close
self.connection.close()
def on_closed(self, connection):
# We've closed our pika connection so stop the demo
tornado.ioloop.IOLoop.instance().stop()
class RabbitClient(object):
EXCHANGE = 'message'
EXCHANGE_TYPE = 'topic'
PUBLISH_INTERVAL = .001
QUEUE = 'text'
ROUTING_KEY = 'example.text'
def __init__(self, app, ioloop):
"""Setup the example publisher object, passing in the URL we will use
to connect to RabbitMQ.
:param str amqp_url: The URL for connecting to RabbitMQ
"""
self._connection = None
self._channel = None
self._deliveries = []
self._acked = 0
self._nacked = 0
self._message_number = 0
self._stopping = False
self.app = app
self.ioloop = ioloop
## Connection Logic ##########################################################
def connect(self):
LOGGER.info('Connecting to RabbitMQ')
# exc_type, exc_value, exc_traceback = sys.exc_info()
# traceback.print_tb(exc_traceback, limit=None, file=sys.stdout)
# self.connecting = True
self._connection = TornadoConnection(rc.connParam,
on_open_callback=self.on_connection_open)
def close_connection(self):
"""This method closes the connection to RabbitMQ."""
LOGGER.info('Closing connection')
self._connection.close()
def add_on_connection_close_callback(self):
LOGGER.info('Adding connection close callback')
self._connection.add_on_close_callback(self.on_connection_closed)
def on_connection_closed(self, method_frame):
# if we loose the connection, try to reconnect
LOGGER.warning('Server closed connection, reopening: (%s) %s',
method_frame.method.reply_code,
method_frame.method.reply_text)
self._channel = None
self._connection = self.connect()
def on_connection_open(self, unused_connection):
LOGGER.info('Connection opened')
self.add_on_connection_close_callback()
self.open_channel()
def add_on_channel_close_callback(self):
LOGGER.info('Adding channel close callback')
self._channel.add_on_close_callback(self.on_channel_closed)
def on_channel_closed(self, method_frame):
# if rabbit closes the channel, quit server
LOGGER.warning('Channel was closed: (%s) %s',
method_frame.method.reply_code,
method_frame.method.reply_text)
self._connection.close()
def on_channel_open(self, channel):
LOGGER.info('Channel opened')
self._channel = channel
self.add_on_channel_close_callback()
self.setupExngsQueues()
def sendNodeCtrlMsg(self, nodeHostname, ctrlCode):
self._channel.basic_publish(
rc.msgExngAttr["exchange"],
"node.{0}.cmd".format(nodeHostname),
str(ctrlCode)
)
## Message Route Init ########################################################
def setupExngsQueues(self):
LOGGER.info('')
self.exngQueCount = 0
self.exngQueNum = len(rc.svrExngs) + len(rc.svrQueues)
# open all exchanges and queues we need asynchronously
for exng in rc.svrExngs:
self.setupExchange(exng)
for queue in rc.svrQueues:
self.setupQueue(queue)
# callback fn counts when everything is declared
def setupExchange(self, exng):
LOGGER.info('Declaring exchange %s', exng["exchange"])
self._channel.exchange_declare(self.onExngQueDeclare, **exng)
def setupQueue(self, queue):
LOGGER.info('Declaring queue %s', queue["queue"])
self._channel.queue_declare(self.onExngQueDeclare, **queue)
def onExngQueDeclare(self, mystery=None): # got unknown return
# check to see how many rabbit entities are declared
self.exngQueCount = self.exngQueCount + 1
LOGGER.info('Declared %d exchanges/queues.', self.exngQueCount)
if(self.exngQueCount == self.exngQueNum):
# all our exchanges and queues are accounted for; setup bindings
self.setupBindings()
def setupBindings(self):
self.bindingsCount = 0
self.bindingsNum = len(rc.svrBindings)
for binding in rc.svrBindings:
LOGGER.info('Binding exchange %s to queue %s', binding["exchange"],
binding["queue"])
self._channel.queue_bind(self.onBind, binding["queue"],
binding["exchange"], binding["routing_key"])
def onBind(self, mystery=None):
# check to see if we made all our bindings
self.bindingsCount = self.bindingsCount + 1
LOGGER.info('Made %d bindings.', self.bindingsCount)
if(self.bindingsCount == self.bindingsNum):
# all of our bindings are accounted for; build message logic
self.setupMsgRouting()
def setupMsgRouting(self):
# map the message queues we defined to our handlers
LOGGER.info('Mapping handlers...')
self._channel.basic_consume(self.valueHandler, rc.valueQueAttr["queue"],
no_ack=True)
self._channel.basic_consume(self.initNodeHandler,
rc.handshakeQueAttr["queue"])
self._channel.basic_consume(self.serverCmdHandler, rc.cmdQueAttr["queue"])
self._channel.basic_consume(self.nErrHandler, rc.nErrQueAttr["queue"])
LOGGER.info('Server ready!')
## Handlers ##################################################################
# [I 130116 01:17:40 wifiWattSrv:203]
# Bad message (TypeError('not all arguments converted during string formatting',)):
# {'threadName': 'MainThread', 'name': '__main__', 'thread': 139807606114048, 'created': 1358317060.982559, 'process': 10225, 'processName': 'MainProcess',
# 'args': (<pika.channel.Channel object at 0x1b77590>,
# <Basic.Deliver(['consumer_tag=ctag1.0', 'redelivered=True', 'routing_key=', 'delivery_tag=1', 'exchange=ww.valueUpdate'])>,
# <BasicProperties(['delivery_mode=1', "headers={'hostname': 'applepi'}"])>, '2.9493'), 'module': 'wifiWattSrv', 'filename': 'wifiWattSrv.py', 'levelno': 20, 'exc_text': None, 'pathname': 'wifiWattSrv.py', 'lineno': 203, 'msg':
# 'Got new channel: %s, basicDeliver: %s, prop: %s, body: ', 'exc_info': None, 'funcName': 'valueHandler', 'relativeCreated': 252.96688079833984, 'levelname': 'INFO', 'msecs': 982.5589656829834}
def valueHandler(self, channel, basicDeliver, prop, body):
"""
:param pika.channel.Channel unused_channel: The channel object
:param pika.Spec.Basic.Deliver: basic_deliver method
:param pika.Spec.BasicProperties: properties
:param str|unicode body: The message body
"""
# attempt to parse message and check fields
try:
msgData = json.loads(body)
except:
LOGGER.error("Couldn't parse message. Malformed.")
return -1
if(("hostname" not in msgData)
or ("current" not in msgData)
or ("relayState" not in msgData)):
LOGGER.error("Couldn't parse message. Mising fields.")
return -1
hostname = msgData["hostname"]
value = float(msgData["current"])
relayState = bool(int(msgData["relayState"]))
# LOGGER.info("Raw: %s Parsed: %d", msgData["relayState"], int(relayState))
# if we've inited this node, call it's append method
if hostname in self.app.nodes:
nodeObj = self.app.nodes[hostname]
newDP = wifiWattNode.wwDataPoint(value, time.time())
nodeObj.appendData(newDP, relayState)
else:
LOGGER.error('No node with hostname <%s>!', hostname)
# attempt to read message header info
# if "hostname" in prop.headers:
# hostname = prop.headers["hostname"]
# else:
# LOGGER.error('Couldn\'t read hostname from message!')
# return -1
# # parse the message
# value = float(body)
# LOGGER.info('Got new value "%f" from hostname <%s>.', value, hostname)
def initNodeHandler(self, channel, basicDeliver, prop, body):
"""
Initialize the data stores for a node on the server. Works as a callback for
a new message to the server handshake queue.
"""
# parse the hostname from handshake message
hostname = body
nodes = self.app.nodes
# check if we already have structures for this node
if(hostname in nodes):
# we have it already
LOGGER.info('Got handshake for node <%s>; already exists.', hostname)
else:
# make a new node instance
LOGGER.info('Initing new node for hostname: %s', hostname)
newNode = wifiWattNode.wifiWattNode(hostname)
nodes[hostname] = newNode
# send back the prefered message rate to start value stream
self._channel.basic_publish(
rc.msgExngAttr["exchange"],
"node.{0}.handshake".format(hostname),
str(prefMsgRate)
)
# notify all our webclients that we have a new node
for webClient in self.app.webClients:
webClient.newNodeCb(newNode)
def serverCmdHandler(self, channel, basicDeliver, prop, body):
LOGGER.info('Got new command: %s', msg)
def nErrHandler(self, channel, basicDeliver, prop, body):
LOGGER.info('Got node error: %s', msg)
def close_channel(self):
"""Invoke this command to close the channel with RabbitMQ by sending
the Channel.Close RPC command.
"""
LOGGER.info('Closing the channel')
self._channel.close()
def open_channel(self):
"""This method will open a new channel with RabbitMQ by issuing the
Channel.Open RPC command. When RabbitMQ confirms the channel is open
by sending the Channel.OpenOK RPC reply, the on_channel_open method
will be invoked.
"""
LOGGER.info('Creating a new channel')
self._connection.channel(on_open_callback=self.on_channel_open)
class NotiPikator(object):
'''
This is a singleton-wannabe class for connecting, listening and
triggering events from RabbitMQ.
It uses the <pika> library with adapter to internal Tornado ioloop
for non-blocking operations on MQ.
'''
def __init__(self, *args, **kwargs):
self._listeners = {} # username: {conn_ts: callback, conn_ts2: callback2}
self._host = kwargs.get('host', 'localhost')
self._port = kwargs.get('port', 5672)
def add_listener(self, rkey, conn_ts, callback):
'Add listener callback for a specific routing key'
rkey_listeners = self._listeners.get(rkey, {})
rkey_listeners[conn_ts] = callback
self._listeners[rkey] = rkey_listeners
self.channel.queue_bind(exchange='mail',
routing_key=str(rkey),
queue='mailq')
logger.debug('new listener for <%s, %s> was set' % (rkey, conn_ts))
def remove_listener(self, rkey, conn_ts):
'Remove callback for a routing key'
try:
rkey_listeners = self._listeners[rkey]
if conn_ts in rkey_listeners:
del rkey_listeners[conn_ts]
logger.debug('listener for <%s, %s> was removed' % (rkey, conn_ts))
if len(rkey_listeners) == 0:
del self._listeners[rkey]
self.channel.queue_unbind(exchange='mail',
routing_key=str(rkey),
queue='mailq')
logger.debug('not listening for <%s> events anymore' % (rkey, ))
except KeyError:
pass # disconnected another time?
def connect(self):
'Establish RabbitMQ connection.'
self.connection = TornadoConnection(
pika.ConnectionParameters(host=self._host, port=self._port),
on_open_callback=self.on_connected)
logger.info('connecting to RabbitMQ...')
def on_connected(self, connection):
'Callback for successfully established connecction'
logger.info('connection with RabbitMQ is established')
self.connection.channel(self.on_channel_open)
def on_channel_open(self, channel):
'Callback for successfully opened connection channel'
logger.debug('RabbitMQ channel is opened')
self.channel = channel
self.channel.exchange_declare(exchange='mail', type='direct')
self.channel.queue_declare(queue='mailq',
callback=self.on_queue_declared,
arguments={'x-message-ttl':30000})
def on_queue_declared(self, frame):
'Callback for successfull exchange hub and a queue on it declaration'
logger.debug('queue_declared event fired')
self.channel.basic_consume(self.on_message, queue='mailq', no_ack=True)
def on_message(self, ch, method, properties, body):
'Callback on incoming event for some binded routing_key'
logger.debug('message received: %s %s' % (method.routing_key, body,))
rkey_listeners = self._listeners.get(method.routing_key, {})
for ts, cb in rkey_listeners.items():
cb(body)
class PikaClient(object):
def __init__(self):
self.ioloop = tornado.ioloop.IOLoop.instance()
self.connection = None
self.channel = None
self._delivery_tag = 0
self.parameters = pika.ConnectionParameters(rabbitmq_server)
def connect(self):
self.connection = TornadoConnection(
self.parameters,
on_open_callback=self.on_connected,
stop_ioloop_on_close=False,
on_open_error_callback=self.on_open_error)
self.connection.add_on_close_callback(self.on_closed)
def on_open_error(self, unused_connection, err):
sys.exit(1)
def on_connected(self, connection):
logging.info('PikaClient: connected to RabbitMQ')
self.connection.channel(self.on_exchange_declare)
def on_exchange_declare(self, channel):
logging.info('PikaClient: Channel %s open, Declaring exchange' %
channel)
self.channel = channel
self.channel.exchange_declare(self.on_queue_declare,
exchange='notification',
exchange_type='direct')
def on_queue_declare(self, method_frame):
logging.info('PikaClient: Channel open, Declaring queue')
self.channel.queue_declare(self.on_queue_bind,
queue='notification') #, durable=True)
def on_queue_bind(self, method_frame):
logging.info('Queue bound')
self.channel.queue_bind(self.on_consume_bind,
queue="notification",
exchange="notification",
routing_key="notification")
def on_consume_bind(self, frame):
logging.info("Consume bind")
self.channel.basic_qos(prefetch_count=1)
self.channel.basic_consume(self.on_response,
queue='notification',
no_ack=False)
def on_response(self, channel, method, properties, body):
logging.info('on_response')
message = pickle.loads(body)
logging.info(message)
ChatSocketHandler.on_caculate_success(message['id'], message['index'])
channel.basic_ack(delivery_tag=method.delivery_tag)
def on_closed(self, connection):
logging.info('PikaClient: rabbit connection closed')
self.connection.close()
self.channel.close()
self.ioloop.stop()
class PikaClient(object):
def __init__(self):
# Construct a queue name we'll use for this instance only
self.queue_name = 'tornado-test-%i' % os.getpid()
# Default values
self.connected = False
self.connecting = False
self.connection = None
self.channel = None
# A place for us to keep messages sent to us by Rabbitmq
self.messages = list()
# A place for us to put pending messages while we're waiting to connect
self.pending = list()
def connect(self):
if self.connecting:
pika.log.info('PikaClient: Already connecting to RabbitMQ')
return
pika.log.info('PikaClient: Connecting to RabbitMQ on localhost:5672')
self.connecting = True
credentials = pika.PlainCredentials('guest', 'guest')
param = pika.ConnectionParameters(host='localhost',
port=5672,
virtual_host="/",
credentials=credentials)
self.connection = TornadoConnection(param,
on_open_callback=self.on_connected)
self.connection.add_on_close_callback(self.on_closed)
def on_connected(self, connection):
pika.log.info('PikaClient: Connected to RabbitMQ on localhost:5672')
self.connected = True
self.connection = connection
self.connection.channel(self.on_channel_open)
def on_channel_open(self, channel):
pika.log.info('PikaClient: Channel Open, Declaring Exchange')
self.channel = channel
self.channel.exchange_declare(exchange='tornado',
type="direct",
auto_delete=True,
durable=False,
callback=self.on_exchange_declared)
def on_exchange_declared(self, frame):
pika.log.info('PikaClient: Exchange Declared, Declaring Queue')
self.channel.queue_declare(queue=self.queue_name,
auto_delete=True,
durable=False,
exclusive=False,
callback=self.on_queue_declared)
def on_queue_declared(self, frame):
pika.log.info('PikaClient: Queue Declared, Binding Queue')
self.channel.queue_bind(exchange='tornado',
queue=self.queue_name,
routing_key='tornado.*',
callback=self.on_queue_bound)
def on_queue_bound(self, frame):
pika.log.info('PikaClient: Queue Bound, Issuing Basic Consume')
self.channel.basic_consume(consumer_callback=self.on_pika_message,
queue=self.queue_name,
no_ack=True)
# Send any messages pending
for properties, body in self.pending:
self.channel.basic_publish(exchange='tornado',
routing_key='tornado.*',
body=body,
properties=properties)
def on_pika_message(self, channel, method, header, body):
pika.log.info('PikaCient: Message receive, delivery tag #%i' % \
method.delivery_tag)
# Append it to our messages list
self.messages.append(body)
def on_basic_cancel(self, frame):
pika.log.info('PikaClient: Basic Cancel Ok')
# If we don't have any more consumer processes running close
self.connection.close()
def on_closed(self, connection):
# We've closed our pika connection so stop the demo
tornado.ioloop.IOLoop.instance().stop()
def sample_message(self, tornado_request):
# Build a message to publish to RabbitMQ
body = '%.8f: Request from %s [%s]' % \
(tornado_request._start_time,
tornado_request.remote_ip,
tornado_request.headers.get("User-Agent"))
# Send the message
properties = pika.BasicProperties(content_type="text/plain",
delivery_mode=1)
self.channel.basic_publish(exchange='tornado',
routing_key='tornado.*',
body=body,
properties=properties)
def get_messages(self):
# Get the messages to return, then empty the list
output = self.messages
self.messages = list()
return output
class PikaClient(object):
def __init__(self):
self.connected = False
self.connecting = False
self.connection = None
self.channel = None
self.messages = list()
self.pending = list()
def connect(self):
if self.connecting:
logging.error("PikaClient already connected")
return
logging.info("connecting to RabbitMQ")
self.connecting = True
param = pika.ConnectionParameters(host='10.212.66.144', port=5672)
self.connection = TornadoConnection(param,
on_open_callback=self.on_connected)
self.connection.add_on_close_callback(self.on_closed)
def on_connected(self, connection):
pika.log.info(
'PikaClient: Connected to RabbitMQ on 10.212.66.144:5672')
self.connected = True
self.connection = connection
self.connection.channel(self.on_channel_open)
def on_channel_open(self, channel):
pika.log.info('PikaClient: Channel Open, Declaring Exchange')
self.channel = channel
self.channel.exchange_declare(exchange='market',
type="fanout",
callback=self.on_exchange_declared)
def on_exchange_declared(self, frame):
logging.info("Exchange declared. Declaring Queue")
self.channel.queue_bind(exchange='market',
queue='market',
callback=self.on_queue_bound)
def on_queue_bound(self, frame):
logging.info('Queue Bound, Issuing Basic Consume')
self.channel.basic_consume(consumer_callback=self.on_pika_message,
queue='market',
no_ack=True)
def on_pika_message(self, channel, method, header, body):
logging.info('Message receive, %s' % body)
# Append it to our messages list
# self.messages.append(body)
symbol, uuid = body.split("|")
trend = db.get("""SELECT * FROM trends.trends WHERE uuid = %s""", uuid)
for listener in LISTENERS:
date = datetime.fromtimestamp(int(trend['created']))
date = date - timedelta(hours=5)
date = date.strftime("%Y-%m-%d %I:%M:%S")
html = uuid + "|<p id='%s'><img src='/static/images/remove-button.gif' width='10' height='10' style='margin-left:1px; margin-right:3px' class='remove_trend'/><span class='label labelcolor1'>" + date + " " + symbol + " </span><span class='trend'>%s trend discovered</span><a href='/full/?uuid=%s' target='_blank'><img src='/static/images/enlarge-button.gif' width='10' height='10' style='margin-left:1px'></a></p>" % (
uuid, trend['type'], uuid)
listener.write_message(html)
def on_closed(self, connection):
tornado.ioloop.IOLoop.instance().stop()
def send_message(self, body):
logging.info("Sending message")
self.channel.basic_publish(exchange='market',
routing_key='',
body=body)
def get_messages(self):
# Get the messages to return, then empty the list
output = self.messages
self.messages = list()
return output
class PikaClient(object):
def __init__(self, config, exchanges):
self.connecting = False
self.connection = None
self.channel = None
self.config = config
self.exchanges = exchanges
def connect(self):
if self.connecting:
return
self.connecting = True
host = self.config.get('host')
port = 5671 if self.config.get('amqps') else 5672
scheme = 'amqps' if self.config.get('amqps') else 'amqp'
virtual_host = urllib.parse.quote(self.config.get('vhost'), safe='')
user = self.config.get('user')
pw = self.config.get('pw')
heartbeat = self.config.get('heartbeat') if self.config.get(
'heartbeat') else 0
params = pika.URLParameters(
f"{scheme}://{user}:{pw}@{host}:{port}/{virtual_host}?heartbeat={heartbeat}"
)
self.connection = TornadoConnection(params)
self.connection.add_on_open_callback(self.on_connect)
self.connection.add_on_close_callback(self.on_closed)
return
def on_connect(self, connection):
self.connection = connection
self.channel = self.connection.channel(
on_open_callback=self.on_channel_open)
return
def on_channel_open(self, channel):
for backend, config in self.exchanges.items():
ex_name = config.get('exchange')
channel.exchange_declare(ex_name,
exchange_type='topic',
durable=True)
logging.info(f'rabbitmq exchange: {ex_name} declared')
return
def on_basic_cancel(self, frame):
self.connection.close()
def on_closed(self, connection):
tornado.ioloop.IOLoop.instance().stop()
def publish_message(self,
exchange=None,
routing_key=None,
method=None,
uri=None,
version=None,
data=None,
persistent=True):
"""
Publilsh a message to an exchange.
Parameters
----------
exchange: str, exchange name
routing_key: str, routing key for topic exchange
method: str, HTTP method
uri: str, HTTP request URI
version: str, e.g. v1
data: dict
persistent: bool, default True
tell rabbitmq to persist messages to disk, or not
"""
data = {'method': method, 'uri': uri, 'version': version, 'data': data}
message = json.dumps(data)
delivery_mode = 2 if persistent else 1
self.channel.basic_publish(exchange=exchange,
routing_key=routing_key,
body=message,
properties=pika.BasicProperties(
content_type='application/json',
delivery_mode=delivery_mode))
return
class PikaClient(object):
def __init__(self, io_loop):
self.logger = logging.getLogger(__name__)
self.logger.info('PikaClient: __init__')
self.io_loop = io_loop
self.connected = False
self.connecting = False
self.connection = None
self.channel = None
self.event_listeners_count = 0
self.event_listeners = set()
self.subscrib_channel = defaultdict(set)
self.last_reconnect = now()
self.uptime_start = now()
self.config = SockJSSereverSettings()
def connect(self):
if self.connecting:
self.logger.info('django-sockjs-server(PikaClient): Already connecting to RabbitMQ')
return
self.logger.info('django-sockjs-server(PikaClient): Connecting to RabbitMQ')
self.connecting = True
cred = pika.PlainCredentials(self.config.rabbitmq_user, self.config.rabbitmq_password)
param = pika.ConnectionParameters(
host=self.config.rabbitmq_host,
port=self.config.rabbitmq_port,
virtual_host=self.config.rabbitmq_vhost,
credentials=cred
)
try:
self.connection = TornadoConnection(param,
on_open_callback=self.on_connected)
self.connection.add_on_close_callback(self.on_closed)
except AMQPConnectionError:
self.logger.info('django-sockjs-server(PikaClient): error connect, wait 5 sec')
time.sleep(5)
self.reconnect()
self.last_reconnect = now()
def on_connected(self, connection):
self.logger.info('django-sockjs-server(PikaClient): connected to RabbitMQ')
self.connected = True
self.connection = connection
self.connection.channel(self.on_channel_open)
def on_channel_open(self, channel):
self.logger.info('django-sockjs-server(PikaClient): Channel open, Declaring exchange')
self.channel = channel
self.channel.exchange_declare(exchange=self.config.rabbitmq_exhange_name,
exchange_type=self.config.rabbitmq_exchange_type)
self.channel.queue_declare(exclusive=False, auto_delete=True, callback=self.on_queue_declared)
def on_queue_declared(self, frame):
self.logger.info('django-sockjs-server(PikaClient): queue bind')
self.channel.queue_bind(callback=None, exchange=self.config.rabbitmq_exhange_name, queue=frame.method.queue)
self.channel.basic_consume(self.handle_delivery, queue=frame.method.queue, no_ack=True)
def handle_delivery(self, channel, method, header, body):
"""Called when we receive a message from RabbitMQ"""
self.notify_listeners(body)
def on_closed(self, connection, error_code, error_message):
self.logger.info('django-sockjs-server(PikaClient): rabbit connection closed, wait 5 seconds')
connection.add_timeout(5, self.reconnect)
def reconnect(self):
self.connecting = False
self.logger.info('django-sockjs-server(PikaClient): reconnect')
self.connect()
def notify_listeners(self, event_json):
event_obj = json.loads(event_json)
self.logger.debug('django-sockjs-server(PikaClient): get new data = %s ' % event_obj)
try:
if len(self.subscrib_channel[event_obj['channel']]) > 0:
for client in self.subscrib_channel[event_obj['channel']]:
self.logger.debug('django-sockjs-server(PikaClient): send message channel = %s ' % event_obj['channel'])
client.broadcast(self.subscrib_channel[event_obj['channel']], event_json)
break
except KeyError:
pass
def add_event_listener(self, listener):
self.event_listeners.add(listener)
self.event_listeners_count += 1
self.logger.debug('django-sockjs-server(PikaClient): listener %s added' % repr(listener))
def remove_event_listener(self, listener):
try:
self.event_listeners.remove(listener)
self.event_listeners_count -= 1
self.logger.debug('django-sockjs-server(PikaClient): listener %s removed' % repr(listener))
except KeyError:
pass
def add_subscriber_channel(self, chanel, client):
self.subscrib_channel[chanel].add(client)
self.logger.debug('django-sockjs-server(PikaClient): listener %s add to channel %s' % (repr(client), chanel))
def remove_subscriber_channel(self, chanel, client):
try:
self.subscrib_channel[chanel].remove(client)
self.logger.debug('django-sockjs-server(PikaClient): listener %s remove from channel %s' % (repr(client),
chanel))
except KeyError:
pass
def get_event_listeners_count(self):
return self.event_listeners_count
def get_subscribe_channel_count(self):
return len(self.subscrib_channel.keys())
def get_subscribe_channels(self):
return self.subscrib_channel.keys()
def get_last_reconnect(self):
return self.last_reconnect
def get_uptime(self):
return (now() - self.uptime_start).seconds
class PikaClient:
'''
Pika client class for working with rabbitMQ, in that case I will provide a db client for writing messages
'''
def __init__(self, dbclient):
self.connected = False
self.connecting = False
self.websocket = None
self.dbclient = dbclient
self.queue_name = 'input_queue'
self.io_loop = None
print('running PikaClient')
def connect(self):
if self.connecting:
print('PikaClient: Already connecting to RabbitMQ')
return
print(
'PikaClient: Connecting to RabbitMQ on localhost:5672, Object: %s'
% (self, ))
self.connecting = True
credentials = pika.PlainCredentials('guest', 'guest')
param = pika.ConnectionParameters(host='localhost',
port=5672,
virtual_host="/",
credentials=credentials)
self.connection = TornadoConnection(param,
on_open_callback=self.on_connected)
def on_connected(self, connection):
print('PikaClient: Connected to RabbitMQ on localhost:5672')
self.connected = True
self.connection = connection
self.connection.channel(self.on_channel_open)
def on_channel_open(self, channel):
print('PikaClient: Channel Open, Declaring Exchange, Channel ID: %s' %
(channel, ))
self.channel = channel
self.channel.exchange_declare(exchange=options.exchange,
exchange_type='fanout',
auto_delete=True)
#I'm not sure if I should place an auto_delete option here, since when closing task.py exchange also will be closed after queue
self.channel.queue_declare(queue=self.queue_name,
exclusive=True,
callback=self.on_queue_declared)
def on_queue_declared(self, frame):
print('PikaClient: Queue Declared, Binding Queue')
self.channel.queue_bind(exchange=options.exchange,
queue=self.queue_name,
callback=self.on_queue_bound)
def on_queue_bound(self, frame):
print('PikaClient: Queue Bound, Issuing Basic Consume')
self.channel.basic_consume(consumer_callback=self.on_pika_message,
queue=self.queue_name,
no_ack=True)
def on_pika_message(self, channel, method, header, body):
collection, key = method.routing_key.split('.')
self.dbclient.write_message(collection, key, body)
msg = 'collection: {}, key: {}, message: {}'.format(
collection, key, body)
WebSocketBroadcast.broadcast_message(msg)
def on_closed(self, connection):
tornado.ioloop.IOLoop.instance().stop()
class PikaClient(object):
def __init__(self, io_loop):
pika.log.info('PikaClient: __init__')
self.io_loop = io_loop
self.connected = False
self.connecting = False
self.connection = None
self.channel = None
self.event_listeners = set([])
def connect(self):
if self.connecting:
pika.log.info('PikaClient: Already connecting to RabbitMQ')
return
pika.log.info('PikaClient: Connecting to RabbitMQ')
self.connecting = True
#cred = pika.PlainCredentials('guest', 'guest')
param = pika.ConnectionParameters(
host='115.146.93.175',
#port=5672,
#virtual_host='/',
#credentials=cred
)
self.connection = TornadoConnection(param,
on_open_callback=self.on_connected)
self.connection.add_on_close_callback(self.on_closed)
def on_connected(self, connection):
pika.log.info('PikaClient: connected to RabbitMQ')
self.connected = True
self.connection = connection
self.connection.channel(self.on_channel_open)
def on_channel_open(self, channel):
pika.log.info('PikaClient: Channel open, Declaring exchange')
self.channel = channel
self.channel.queue_declare(queue='hello')
self.channel.basic_consume(self.on_message,
queue='hello',
no_ack=True)
# declare exchanges, which in turn, declare
# queues, and bind exchange to queues
def on_closed(self, connection):
pika.log.info('PikaClient: rabbit connection closed')
self.io_loop.stop()
def on_message(self, channel, method, header, body):
pika.log.info('PikaClient: message received: %s' % body)
self.notify_listeners(body)
def send_message(self, body):
self.channel.basic_publish(exchange='',
routing_key='hello',
body=body)
def notify_listeners(self, body):
for listener in self.event_listeners:
listener.write_message(body)
pika.log.info('PikaClient: notified %s' % repr(listener))
def add_event_listener(self, listener):
print "added listener"
self.event_listeners.add(listener)
pika.log.info('PikaClient: listener %s added' % repr(listener))
def remove_event_listener(self, listener):
try:
self.event_listeners.remove(listener)
pika.log.info('PikaClient: listener %s removed' % repr(listener))
except KeyError:
pass
class CaculateWorker(object):
REDIS_HASHMAP_KEY = os.environ.get("REDIS_HASHMAP_KEY")
def __init__(self):
rabbitmq_server = os.environ.get("RABBITMQ_SERVER")
redis_server = os.environ.get("REDIS_SERVER")
self.redis_client = redis.Redis(host=redis_server, port=6379, db=0)
self.ioloop = tornado.ioloop.IOLoop.instance()
self.connection = None
self.channel = None
self._delivery_tag = 0
self.parameters = pika.ConnectionParameters(rabbitmq_server)
@staticmethod
def fibonacci(index):
if index <= 1:
return 1
return CaculateWorker.fibonacci(index -
1) + CaculateWorker.fibonacci(index -
2)
def cache_cacalate(self, index, value):
self.redis_client.hset(CaculateWorker.REDIS_HASHMAP_KEY, index, value)
def connect(self):
try:
self.connection = TornadoConnection(
self.parameters,
on_open_callback=self.on_connected,
stop_ioloop_on_close=False,
on_open_error_callback=self.on_open_error)
self.connection.add_on_close_callback(self.on_closed)
except:
logging.info("connect faield")
def on_open_error(self, unused_connection, err):
sys.exit(1)
def on_connected(self, connection):
logging.info('PikaClient: connected to RabbitMQ')
self.connection.channel(self.on_exchange_declare)
def on_exchange_declare(self, channel):
logging.info('PikaClient: Channel %s open, Declaring exchange' %
channel)
self.channel = channel
self.channel.exchange_declare(self.on_queue_declare,
exchange='calc_fibonacci',
exchange_type='direct')
self.channel.exchange_declare(self.on_queue_declare,
exchange='notification',
exchange_type='direct')
def on_queue_declare(self, method_frame):
logging.info('PikaClient: Channel open, Declaring queue')
self.channel.queue_declare(self.on_queue_bind,
queue='calc_fibonacci') #, durable=True)
self.channel.queue_declare(self.on_queue_bind,
queue='notification') #, durable=True)
def on_queue_bind(self, method_frame):
logging.info('Queue bound')
self.channel.queue_bind(self.on_consume_bind,
queue="calc_fibonacci",
exchange="calc_fibonacci",
routing_key="calc_fibonacci")
self.channel.queue_bind(self.on_consume_bind,
queue="notification",
exchange="notification",
routing_key="notification")
def on_consume_bind(self, frame):
logging.info("Consume bind")
self.channel.basic_qos(prefetch_count=1)
self.channel.basic_consume(self.on_response,
queue='calc_fibonacci',
no_ack=False)
def on_response(self, channel, method, properties, body):
logging.info('on_response')
message = pickle.loads(body)
logging.info(message)
uuid, index = message['uuid'], message['index']
value = CaculateWorker.fibonacci(index)
self.cache_cacalate(index, value)
# self.channel.exchange_declare(exchange="notification", exchange_type="direct")
# self.channel.queue_declare(queue='notification')
# self.channel.queue_bind(exchange='notification', queue=notification_queue, routing_key='notification')
result = {
'id': uuid,
'index': index,
'value': value,
}
logging.info(result)
channel.basic_publish(exchange='notification',
routing_key='notification',
body=pickle.dumps(result))
logging.info("publish done")
channel.basic_ack(delivery_tag=method.delivery_tag)
def on_closed(self, connection):
logging.info('PikaClient: rabbit connection closed')
self.connection.close()
self.channel.close()
self.ioloop.stop()
class AsyncTransport(BaseTransport):
def _connect(self):
self._connection = TornadoConnection(
self._parameters,
on_open_callback=self.on_connection_open,
stop_ioloop_on_close=False
)
def on_connection_open(self, unused_connection):
"""This method is called by pika once the connection to RabbitMQ has
been established. It passes the handle to the connection object in
case we need it, but in this case, we'll just mark it unused.
:type unused_connection: pika.SelectConnection
"""
self._connection.add_on_close_callback(self.on_connection_closed)
self._open_channel()
def on_connection_closed(self, method_frame):
self._connection = None
self._connect()
def close(self):
self._connection.close()
def _open_channel(self):
"""Open a new channel with RabbitMQ by issuing the Channel.Open RPC
command. When RabbitMQ responds that the channel is open, the
on_channel_open callback will be invoked by pika.
"""
self._connection.channel(self._on_channel_open)
def _on_channel_open(self, channel):
self._channel = channel
self._channel.basic_qos(prefetch_count=1)
self.add_on_channel_close_callback()
self.setup_exchange()
def add_on_channel_close_callback(self):
"""This method tells pika to call the on_channel_closed method if
RabbitMQ unexpectedly closes the channel.
"""
self._channel.add_on_close_callback(self.on_channel_closed)
def on_channel_closed(self, channel, reply_code, reply_text):
"""Invoked by pika when RabbitMQ unexpectedly closes the channel.
Channels are usually closed if you attempt to do something that
violates the protocol, such as re-declare an exchange or queue with
different parameters. In this case, we'll close the connection
to shutdown the object.
:param pika.channel.Channel: The closed channel
:param int reply_code: The numeric reason the channel was closed
:param str reply_text: The text reason the channel was closed
"""
LOGGER.warning('Channel %i was closed: (%s) %s',
channel, reply_code, reply_text)
self._connection.close()
def setup_exchange(self):
"""Setup the exchange on RabbitMQ by invoking the Exchange.Declare RPC
command. When it is complete, the on_exchange_declareok method will
be invoked by pika.
:param str|unicode exchange_name: The name of the exchange to declare
"""
self._channel.exchange_declare(self.on_exchange_declareok,
self.EXCHANGE_NAME,
self.EXCHANGE_TYPE)
def on_exchange_declareok(self, unused_frame):
"""Invoked by pika when RabbitMQ has finished the Exchange.Declare RPC
command.
:param pika.Frame.Method unused_frame: Exchange.DeclareOk response frame
"""
LOGGER.info('Exchange declared')
self.setup_queue(self.QUEUE)
def setup_queue(self, queue_name):
"""Setup the queue on RabbitMQ by invoking the Queue.Declare RPC
command. When it is complete, the on_queue_declareok method will
be invoked by pika.
:param str|unicode queue_name: The name of the queue to declare.
"""
LOGGER.info('Declaring queue %s', queue_name)
self._channel.queue_declare(self.on_queue_declareok, queue_name)
def on_queue_declareok(self, method_frame):
"""Method invoked by pika when the Queue.Declare RPC call made in
setup_queue has completed. In this method we will bind the queue
and exchange together with the routing key by issuing the Queue.Bind
RPC command. When this command is complete, the on_bindok method will
be invoked by pika.
:param pika.frame.Method method_frame: The Queue.DeclareOk frame
"""
LOGGER.info('Binding %s to %s with %s',
self.EXCHANGE_NAME, self.QUEUE, self.ROUTING_KEY)
self._channel.queue_bind(self.on_bindok, self.QUEUE,
self.EXCHANGE_NAME, self.ROUTING_KEY)
def add_on_cancel_callback(self):
"""Add a callback that will be invoked if RabbitMQ cancels the consumer
for some reason. If RabbitMQ does cancel the consumer,
on_consumer_cancelled will be invoked by pika.
"""
LOGGER.info('Adding consumer cancellation callback')
self._channel.add_on_cancel_callback(self.on_consumer_cancelled)
def on_consumer_cancelled(self, method_frame):
"""Invoked by pika when RabbitMQ sends a Basic.Cancel for a consumer
receiving messages.
:param pika.frame.Method method_frame: The Basic.Cancel frame
"""
LOGGER.info('Consumer was cancelled remotely, shutting down: %r',
method_frame)
if self._channel:
self._channel.close()
def acknowledge_message(self, delivery_tag):
"""Acknowledge the message delivery from RabbitMQ by sending a
Basic.Ack RPC method for the delivery tag.
:param int delivery_tag: The delivery tag from the Basic.Deliver frame
"""
LOGGER.info('Acknowledging message %s', delivery_tag)
self._channel.basic_ack(delivery_tag)
def on_message(self, unused_channel, basic_deliver, properties, body):
"""Invoked by pika when a message is delivered from RabbitMQ. The
channel is passed for your convenience. The basic_deliver object that
is passed in carries the exchange, routing key, delivery tag and
a redelivered flag for the message. The properties passed in is an
instance of BasicProperties with the message properties and the body
is the message that was sent.
:param pika.channel.Channel unused_channel: The channel object
:param pika.Spec.Basic.Deliver: basic_deliver method
:param pika.Spec.BasicProperties: properties
:param str|unicode body: The message body
"""
LOGGER.info('Received message # %s from %s: %s',
basic_deliver.delivery_tag, properties.app_id, body)
self.work_q.put(json.loads(body))
self.acknowledge_message(basic_deliver.delivery_tag)
def on_cancelok(self, unused_frame):
"""This method is invoked by pika when RabbitMQ acknowledges the
cancellation of a consumer. At this point we will close the channel.
This will invoke the on_channel_closed method once the channel has been
closed, which will in-turn close the connection.
:param pika.frame.Method unused_frame: The Basic.CancelOk frame
"""
LOGGER.info('RabbitMQ acknowledged the cancellation of the consumer')
self.close_channel()
def stop_consuming(self):
"""Tell RabbitMQ that you would like to stop consuming by sending the
Basic.Cancel RPC command.
"""
if self._channel:
LOGGER.info('Sending a Basic.Cancel RPC command to RabbitMQ')
self._channel.basic_cancel(self.on_cancelok, self._consumer_tag)
def start_consuming(self):
"""This method sets up the consumer by first calling
add_on_cancel_callback so that the object is notified if RabbitMQ
cancels the consumer. It then issues the Basic.Consume RPC command
which returns the consumer tag that is used to uniquely identify the
consumer with RabbitMQ. We keep the value to use it when we want to
cancel consuming. The on_message method is passed in as a callback pika
will invoke when a message is fully received.
"""
LOGGER.info('Issuing consumer related RPC commands')
self.add_on_cancel_callback()
self._consumer_tag = self._channel.basic_consume(self.on_message,
self.QUEUE)
def on_bindok(self, unused_frame):
"""Invoked by pika when the Queue.Bind method has completed. At this
point we will start consuming messages by calling start_consuming
which will invoke the needed RPC commands to start the process.
:param pika.frame.Method unused_frame: The Queue.BindOk response frame
"""
LOGGER.info('Queue bound')
self.start_consuming()
class LoginPublish(object):
def __init__(self):
self.channel = None
self.connection = None
self.messages = []
def initialcon(self):
_logger.debug("pikapublish begin initialcon")
credentials = pika.PlainCredentials('mxpub', 'mhearts2015')
parameters = pika.ConnectionParameters('localhost', 6500, credentials=credentials)
self.connection = TornadoConnection(parameters, on_open_callback = self.on_connected)
def on_connected(self, connection):
_logger.debug("pikapublish begin on_connected")
self.connection = connection
self.connection.channel(self.on_channel_open)
self.connection.add_on_close_callback(self.on_connection_closed)
def on_connection_closed(self, connection, reply_code, reply_text):
_logger.info("connection closed")
self.connection.add_timeout(5, self.initialcon)
def on_channel_open(self, channel):
_logger.debug("pikapublish begin on_channel_open")
self.channel = channel
self.publishmsg()
def releasecon(self):
if self.connection:
self.connection.close()
def pushmsg(self, loginkey, body):
_logger.debug("begin pushmsg %r %s" % (body, loginkey))
if not loginkey or not body:
return
self.messages.append({"key":loginkey, "body":body})
self.publishmsg()
def publishmsg(self):
_logger.debug("begin to publishmsg")
if not self.channel:
_logger.debug("begin to open channel")
self.initialcon()
return
for item in self.messages:
key = item.get("key", "")
body = item.get("body", "")
_logger.info("begin to publish %s body = %r" % (key, body))
if not isinstance(key, str) and not isinstance(key, bytes):
_logger.error("invalid key")
continue
noti_body = json.dumps(body)
try:
self.channel.basic_publish(exchange = 'pclogin',
routing_key = key,
body = noti_body)
except Exception as e:
_logger.error("pikapublish catch exception {0}".format(e))
self.messages[:] = []
class RabbitMQConnection(object):
__io_loop = None
__connected = None
__connecting = None
__connection = None
__channel = None
__callback_queue = None
def __init__(
self,
io_loop,
):
logger.debug('PikaClient: __init__')
self.__io_loop = io_loop
self.__connected = False
self.__connecting = False
self.__connection = None
self.__channel = None
self.connect = self.__connect
self.call = self.__call
def __connect(self):
if self.__connecting:
logger.debug('PikaClient: Already connecting to RabbitMQ')
return
logger.debug('PikaClient: Connecting to RabbitMQ')
self.__connecting = True
cred = pika.PlainCredentials(Config.RabbitMQ.username,
Config.RabbitMQ.password)
param = pika.ConnectionParameters(
host=Config.RabbitMQ.host,
port=Config.RabbitMQ.port,
virtual_host=Config.RabbitMQ.virtual_host,
credentials=cred)
self.__connection = TornadoConnection(
param,
on_open_callback=self.__on_connected,
stop_ioloop_on_close=False)
self.__connection.add_on_close_callback(self.__on_close)
def __on_connected(self, connection):
logger.debug('PikaClient: connected to RabbitMQ')
self.__connected = True
self.__connection = connection
self.__connection.channel(self.__on_channel_open)
def __call(self, message, correlation_id):
self.__channel.basic_publish(exchange='',
routing_key='rpc_queue',
properties=pika.BasicProperties(
reply_to=self.__callback_queue,
correlation_id=correlation_id,
),
body=str(message))
def __on_response(self, ch, method, props, body):
print("I got", props.correlation_id)
ClientWebSocket.on_response(message=body,
correlation_id=props.correlation_id)
def __on_close(self, connection):
logger.debug('PikaClient: rabbit connection closed')
self.__io_loop.stop()
def __on_queue_open(self, a):
self.__callback_queue = a.method.queue
self.__channel.basic_consume(self.__on_response,
no_ack=True,
queue=self.__callback_queue)
def __on_channel_open(self, channel):
# print("channel: ", channel)
logger.debug('PikaClient: Channel %s open, Declaring exchange' %
channel)
self.__channel = channel
self.__channel.queue_declare(callback=self.__on_queue_open,
exclusive=True)
class PikaClient(object):
def __init__(self):
# Construct a queue name we'll use for this instance only
self.queue_name = "hello"
# Default values
self.connected = False
self.connecting = False
self.connection = None
self.channel = None
# A place for us to keep messages sent to us by Rabbitmq
self.messages = list()
# A place for us to put pending messages while we're waiting to connect
self.pending = list()
def connect(self):
if self.connecting:
pika.log.info('PikaClient: Already connecting to RabbitMQ')
return
pika.log.info('PikaClient: Connecting to RabbitMQ on localhost:5672')
self.connecting = True
param = pika.ConnectionParameters(host='115.146.93.175')
self.connection = TornadoConnection(param,
on_open_callback=self.on_connected)
self.connection.add_on_close_callback(self.on_closed)
def on_connected(self, connection):
pika.log.info('PikaClient: Connected to RabbitMQ on localhost:5672')
self.connected = True
self.connection = connection
self.connection.channel(self.on_channel_open)
def on_channel_open(self, channel):
pika.log.info('PikaClient: Channel Open, Declaring Exchange')
self.channel = channel
self.channel.exchange_declare(exchange='',
#type="direct",
#auto_delete=True,
#durable=False,
callback=self.on_exchange_declared)
def on_exchange_declared(self, frame):
pika.log.info('PikaClient: Exchange Declared, Declaring Queue')
self.channel.queue_declare(queue=self.queue_name,
#auto_delete=True,
#durable=False,
#exclusive=False,
callback=self.on_queue_declared)
def on_queue_declared(self, frame):
pika.log.info('PikaClient: Queue Declared, Binding Queue')
self.channel.queue_bind(exchange='',
queue=self.queue_name,
routing_key='hello',
callback=self.on_queue_bound)
def on_queue_bound(self, frame):
pika.log.info('PikaClient: Queue Bound, Issuing Basic Consume')
self.channel.basic_consume(consumer_callback=self.on_pika_message,
queue=self.queue_name,
no_ack=True)
# Send any messages pending
for properties, body in self.pending:
self.channel.basic_publish(exchange='',
routing_key='hello',
body=body,
properties=properties)
def on_pika_message(self, channel, method, header, body):
pika.log.info('PikaCient: Message receive, delivery tag #%i' % \
method.delivery_tag)
# Append it to our messages list
self.messages.append(body)
def on_basic_cancel(self, frame):
pika.log.info('PikaClient: Basic Cancel Ok')
# If we don't have any more consumer processes running close
self.connection.close()
def on_closed(self, connection):
# We've closed our pika connection so stop the demo
tornado.ioloop.IOLoop.instance().stop()
def sample_message(self, tornado_request):
# Build a message to publish to RabbitMQ
body = '%.8f: Request from %s [%s]' % \
(tornado_request._start_time,
tornado_request.remote_ip,
tornado_request.headers.get("User-Agent"))
# Send the message
properties = pika.BasicProperties(content_type="text/plain",
delivery_mode=1)
self.channel.basic_publish(exchange='',
routing_key='hello',
body=body,
properties=properties)
def get_messages(self):
# Get the messages to return, then empty the list
output = self.messages
self.messages = list()
return output
class ExamplePublisher(object):
"""This is an example publisher that will handle unexpected interactions
with RabbitMQ such as channel and connection closures.
If RabbitMQ closes the connection, it will reopen it. You should
look at the output, as there are limited reasons why the connection may
be closed, which usually are tied to permission related issues or
socket timeouts.
It uses delivery confirmations and illustrates one way to keep track of
messages that have been sent and if they've been confirmed by RabbitMQ.
"""
EXCHANGE = 'message'
EXCHANGE_TYPE = 'topic'
PUBLISH_INTERVAL = .001
QUEUE = 'text'
ROUTING_KEY = 'example.text'
def __init__(self, app, ioloop):
"""Setup the example publisher object, passing in the URL we will use
to connect to RabbitMQ.
:param str amqp_url: The URL for connecting to RabbitMQ
"""
self._connection = None
self._channel = None
self._deliveries = []
self._acked = 0
self._nacked = 0
self._message_number = 0
self._stopping = False
self.app = app
self.ioloop = ioloop
def connect(self):
LOGGER.info('Connecting to RabbitMQ')
# exc_type, exc_value, exc_traceback = sys.exc_info()
# traceback.print_tb(exc_traceback, limit=None, file=sys.stdout)
# self.connecting = True
self._connection = TornadoConnection(rc.connParam,
on_open_callback=self.on_connection_open)
def close_connection(self):
"""This method closes the connection to RabbitMQ."""
LOGGER.info('Closing connection')
self._connection.close()
def add_on_connection_close_callback(self):
"""This method adds an on close callback that will be invoked by pika
when RabbitMQ closes the connection to the publisher unexpectedly.
"""
LOGGER.info('Adding connection close callback')
self._connection.add_on_close_callback(self.on_connection_closed)
def on_connection_closed(self, method_frame):
"""This method is invoked by pika when the connection to RabbitMQ is
closed unexpectedly. Since it is unexpected, we will reconnect to
RabbitMQ if it disconnects.
:param pika.frame.Method method_frame: The method frame from RabbitMQ
"""
LOGGER.warning('Server closed connection, reopening: (%s) %s',
method_frame.method.reply_code,
method_frame.method.reply_text)
self._channel = None
self._connection = self.connect()
def on_connection_open(self, unused_connection):
"""This method is called by pika once the connection to RabbitMQ has
been established. It passes the handle to the connection object in
case we need it, but in this case, we'll just mark it unused.
:type unused_connection: pika.SelectConnection
"""
LOGGER.info('Connection opened')
self.add_on_connection_close_callback()
self.open_channel()
def add_on_channel_close_callback(self):
"""This method tells pika to call the on_channel_closed method if
RabbitMQ unexpectedly closes the channel.
"""
LOGGER.info('Adding channel close callback')
self._channel.add_on_close_callback(self.on_channel_closed)
def on_channel_closed(self, method_frame):
"""Invoked by pika when RabbitMQ unexpectedly closes the channel.
Channels are usually closed if you attempt to do something that
violates the protocol, such as redeclare an exchange or queue with
different paramters. In this case, we'll close the connection
to shutdown the object.
:param pika.frame.Method method_frame: The Channel.Close method frame
"""
LOGGER.warning('Channel was closed: (%s) %s',
method_frame.method.reply_code,
method_frame.method.reply_text)
self._connection.close()
def on_channel_open(self, channel):
"""This method is invoked by pika when the channel has been opened.
The channel object is passed in so we can make use of it.
Since the channel is now open, we'll declare the exchange to use.
:param pika.channel.Channel channel: The channel object
"""
LOGGER.info('Channel opened')
self._channel = channel
self.add_on_channel_close_callback()
self.setup_exchange()
def setup_exchange(self):
"""Setup the exchange on RabbitMQ by invoking the Exchange.Declare RPC
command. When it is complete, the on_exchange_declareok method will
be invoked by pika.
:param str|unicode exchange_name: The name of the exchange to declare
"""
LOGGER.info('Declaring exchange %s', rc.valueExngAttr["exchange"])
self._channel.exchange_declare(self.on_exchange_declareok,
**rc.valueExngAttr)
def on_exchange_declareok(self, unused_frame):
"""Invoked by pika when RabbitMQ has finished the Exchange.Declare RPC
command.
:param pika.Frame.Method unused_frame: Exchange.DeclareOk response frame
"""
LOGGER.info('Exchange declared')
self.setup_queue(self.QUEUE)
def setup_queue(self, queue_name):
"""Setup the queue on RabbitMQ by invoking the Queue.Declare RPC
command. When it is complete, the on_queue_declareok method will
be invoked by pika.
:param str|unicode queue_name: The name of the queue to declare.
"""
LOGGER.info('Declaring queue %s', queue_name)
self._channel.queue_declare(self.on_queue_declareok, queue_name)
def on_queue_declareok(self, method_frame):
"""Method invoked by pika when the Queue.Declare RPC call made in
setup_queue has completed. In this method we will bind the queue
and exchange together with the routing key by issuing the Queue.Bind
RPC command. When this command is complete, the on_bindok method will
be invoked by pika.
:param pika.frame.Method method_frame: The Queue.DeclareOk frame
"""
LOGGER.info('Binding %s to %s with %s',
rc.valueExngAttr["exchange"], self.QUEUE, self.ROUTING_KEY)
self._channel.queue_bind(self.on_bindok, self.QUEUE,
rc.valueExngAttr["exchange"], self.ROUTING_KEY)
def on_delivery_confirmation(self, method_frame):
"""Invoked by pika when RabbitMQ responds to a Basic.Publish RPC
command, passing in either a Basic.Ack or Basic.Nack frame with
the delivery tag of the message that was published. The delivery tag
is an integer counter indicating the message number that was sent
on the channel via Basic.Publish. Here we're just doing house keeping
to keep track of stats and remove message numbers that we expect
a delivery confirmation of from the list used to keep track of messages
that are pending confirmation.
:param pika.frame.Method method_frame: Basic.Ack or Basic.Nack frame
"""
confirmation_type = method_frame.method.NAME.split('.')[1].lower()
LOGGER.info('Received %s for delivery tag: %i',
confirmation_type,
method_frame.method.delivery_tag)
if confirmation_type == 'ack':
self._acked += 1
elif confirmation_type == 'nack':
self._nacked += 1
self._deliveries.remove(method_frame.method.delivery_tag)
LOGGER.info('Published %i messages, %i have yet to be confirmed, '
'%i were acked and %i were nacked',
self._message_number, len(self._deliveries),
self._acked, self._nacked)
def enable_delivery_confirmations(self):
"""Send the Confirm.Select RPC method to RabbitMQ to enable delivery
confirmations on the channel. The only way to turn this off is to close
the channel and create a new one.
When the message is confirmed from RabbitMQ, the
on_delivery_confirmation method will be invoked passing in a Basic.Ack
or Basic.Nack method from RabbitMQ that will indicate which messages it
is confirming or rejecting.
"""
LOGGER.info('Issuing Confirm.Select RPC command')
self._channel.confirm_delivery(self.on_delivery_confirmation)
def publish_message(self):
"""If the class is not stopping, publish a message to RabbitMQ,
appending a list of deliveries with the message number that was sent.
This list will be used to check for delivery confirmations in the
on_delivery_confirmations method.
Once the message has been sent, schedule another message to be sent.
The main reason I put scheduling in was just so you can get a good idea
of how the process is flowing by slowing down and speeding up the
delivery intervals by changing the PUBLISH_INTERVAL constant in the
class.
"""
if self._stopping:
return
message = 'The current epoch value is %i' % time.time()
properties = pika.BasicProperties(app_id='example-publisher',
content_type='text/plain')
self._channel.basic_publish(rc.valueExngAttr["exchange"], self.ROUTING_KEY,
message, properties)
self._message_number += 1
self._deliveries.append(self._message_number)
LOGGER.info('Published message # %i', self._message_number)
self.schedule_next_message()
def schedule_next_message(self):
"""If we are not closing our connection to RabbitMQ, schedule another
message to be delivered in PUBLISH_INTERVAL seconds.
"""
if self._stopping:
return
LOGGER.info('Scheduling next message for %0.1f ms',
self.PUBLISH_INTERVAL)
self._connection.add_timeout(self.PUBLISH_INTERVAL,
self.publish_message)
def start_publishing(self):
"""This method will enable delivery confirmations and schedule the
first message to be sent to RabbitMQ
"""
LOGGER.info('Issuing consumer related RPC commands')
self.enable_delivery_confirmations()
self.schedule_next_message()
def on_bindok(self, unused_frame):
"""This method is invoked by pika when it receives the Queue.BindOk
response from RabbitMQ. Since we know we're now setup and bound, it's
time to start publishing."""
LOGGER.info('Queue bound')
self.start_publishing()
def close_channel(self):
"""Invoke this command to close the channel with RabbitMQ by sending
the Channel.Close RPC command.
"""
LOGGER.info('Closing the channel')
self._channel.close()
def open_channel(self):
"""This method will open a new channel with RabbitMQ by issuing the
Channel.Open RPC command. When RabbitMQ confirms the channel is open
by sending the Channel.OpenOK RPC reply, the on_channel_open method
will be invoked.
"""
LOGGER.info('Creating a new channel')
self._connection.channel(on_open_callback=self.on_channel_open)
