class Sender(object): def __init__(self, settings, io_loop): self.io_loop = io_loop self.channel = None self.exchange = "poly" credentials = None if settings.get("username", None): credentials = pika.PlainCredentials( settings["username"], settings["password"] ) self.connection_parameters = None if credentials or settings.get("host", None) or settings.get("vhost"): self.connection_parameters = pika.ConnectionParameters( credentials=credentials, host=settings.get("host", None), port=settings.get("port", None), virtual_host=settings.get("vhost", None) ) else: raise Exception("NO self.connection_parameters") self.settings = settings def connect(self): logging.info("MQ connect...") try: self.connection = TornadoConnection( self.connection_parameters, self.on_connected ) self.connection.add_on_close_callback(self.on_close) except socket.error, e: logging.warn("connection failed, trying again in 5 seconds") self.io_loop.add_timeout(time.time() + 5, self.connect)
class PikaClient(object): callbacks = {} def __init__(self, callback_ins, port): self.callback_ins=callback_ins self.queue_name = "q%d" % uuid.uuid4().int if tornado is None: raise Exception('You must add tornado to your requirements!') if pika is None: raise Exception('You must add pika to your requirements!') self.ioloop = tornado.ioloop.IOLoop.instance() self.connection = None self.channel = None self._delivery_tag = 0 self.parameters = pika.URLParameters("amqp://*****:*****@hostname:" + str(port) + "/%2F") def connect(self): self.connection = TornadoConnection(self.parameters, on_open_callback=self.on_connected, stop_ioloop_on_close=False) self.connection.add_on_close_callback(self.on_closed) def on_connected(self, connection): logger.info('PikaClient: connected to RabbitMQ') self.connection.channel(self.on_exchange_declare) def on_exchange_declare(self, channel): logger.info('PikaClient: Channel %s open, Declaring exchange' % channel) self.channel = channel self.channel.exchange_declare(self.on_queue_declare, exchange='compute', type='fanout') def on_queue_declare(self, method_frame): logger.info('PikaClient: Channel open, Declaring queue') self.result = self.channel.queue_declare(self.on_queue_bind, queue=self.queue_name, durable=True) def on_queue_bind(self, method_frame): logger.info('Queue bound') self.channel.queue_bind(self.on_consume_bind, queue=self.queue_name, exchange="compute") def on_consume_bind(self, frame): self.channel.basic_qos(prefetch_count=1) self.channel.basic_consume(self.on_response, queue=self.queue_name, no_ack=False) def on_response(self, channel, method, properties, body): message = body self.callback_ins.handle_rabbitmq_message(message) channel.basic_ack(delivery_tag = method.delivery_tag) logger.info('Recieve a new Message: %r' % message) def on_closed(self, connection): logger.info('PikaClient: rabbit connection closed') self.connection.close() self.channel.close() self.ioloop.stop()
class PikaClient(object): def __init__(self, username='******', exchange_name='ws', password='******', host='localhost', port=5672, virtual_host='/'): self.exchange_name = exchange_name # Options self.username = username self.password = password self.host = host self.port = port self.virtual_host = virtual_host # Default values self.connected = False self.connecting = False self.connection = None self.channel = None def connect(self): if self.connecting: pika.log.info('PikaClient: Already connecting to RabbitMQ') return pika.log.info('PikaClient: Connecting to RabbitMQ on localhost:5672') self.connecting = True credentials = pika.PlainCredentials(self.username, self.password) param = pika.ConnectionParameters(host=self.host, port=self.port, virtual_host=self.virtual_host, credentials=credentials) self.connection = TornadoConnection(param, on_open_callback=self.on_connected) self.connection.add_on_close_callback(self.on_closed) def on_connected(self, connection): pika.log.info('PikaClient: Connected to RabbitMQ on localhost:5672') self.connected = True self.connection = connection self.connection.channel(self.on_channel_open) def on_channel_open(self, channel): pika.log.info('PikaClient: Channel Open, Declaring Exchange') self.channel = channel self.channel.exchange_declare(exchange=self.exchange_name, type="direct", callback=self.on_exchange_declared) def on_exchange_declared(self, frame): pika.log.info('PikaClient: Exchange Declared, Ready for declaring Queues') def on_basic_cancel(self, frame): pika.log.info('PikaClient: Basic Cancel Ok') # If we don't have any more consumer processes running close self.connection.close() def on_closed(self, connection): # We've closed our pika connection so stop the demo tornado.ioloop.IOLoop.instance().stop()
class PikaClient(object): def __init__(self, io_loop): self.io_loop = io_loop self.connected = False self.connecting = False self.connection = None self.channel = None self.event_listeners = set([]) def connect(self): if self.connecting: return self.connecting = True self.connection = TornadoConnection(on_open_callback=self.on_connected) self.connection.add_on_close_callback(self.on_closed) def on_connected(self, connection): self.connected = True self.connection = connection self.connection.channel(self.on_channel_open) def on_channel_open(self, channel): self.channel = channel channel.queue_declare(queue="plase", durable=True, callback=self.on_queue_declared) def on_queue_declared(self, frame): self.channel.basic_consume(self.on_message, queue='plase') def on_closed(self, connection): self.io_loop.stop() def on_message(self, channel, method, header, body): self.notify_listeners(body) def notify_listeners(self, event_obj): event_json = json.dumps(event_obj) for listener in self.event_listeners: listener.write_message(event_json) def add_event_listener(self, listener): self.event_listeners.add(listener) def remove_event_listener(self, listener): try: self.event_listeners.remove(listener) except KeyError: pass
class PikaClient(object): def __init__(self, io_loop): self.io_loop = io_loop self.connected = False self.connecting = False self.connection = None self.channel = None self.event_listeners = set([]) def connect(self): if self.connecting: return self.connecting = True self.connection = TornadoConnection(on_open_callback=self.on_connected) self.connection.add_on_close_callback(self.on_closed) def on_connected(self, connection): self.connected = True self.connection = connection self.connection.channel(self.on_channel_open) def on_channel_open(self, channel): self.channel = channel channel.queue_declare(queue="plase", durable=True, callback=self.on_queue_declared) def on_queue_declared(self, frame): self.channel.basic_consume(self.on_message, queue="plase") def on_closed(self, connection): self.io_loop.stop() def on_message(self, channel, method, header, body): self.notify_listeners(body) def notify_listeners(self, event_obj): event_json = json.dumps(event_obj) for listener in self.event_listeners: listener.write_message(event_json) def add_event_listener(self, listener): self.event_listeners.add(listener) def remove_event_listener(self, listener): try: self.event_listeners.remove(listener) except KeyError: pass
class PikaConsumer(object): def __init__(self): if tornado is None: raise Exception('You must add tornado to your requirements!') if pika is None: raise Exception('You must add pika to your requirements!') self.ioloop = tornado.ioloop.IOLoop.instance() self.connection = None self.channel = None self._delivery_tag = 0 self.parameters = None self.queue_name = None self.on_task_end = None def connect(self): self.connection = TornadoConnection(self.parameters, on_open_callback=self.on_connected, stop_ioloop_on_close=False) self.connection.add_on_close_callback(self.on_closed) def on_connected(self, connection): logger.info('PikaConsumer: connected to RabbitMQ') self.connection.channel(on_open_callback=self.on_channel_declared) def on_channel_declared(self, channel): self.channel = channel self.channel.queue_declare(self.on_queue_declared, queue=self.queue_name) def on_queue_declared(self, method_frame): self.channel.basic_qos(prefetch_count=1) self.channel.basic_consume(self.on_consume, queue=self.queue_name, no_ack=False) def on_closed(self, connection): logger.info('PikaConsumer: rabbit connection closed') self.connection.close() self.channel.close() self.ioloop.stop() def on_consume(self, channel, method, properties, body): try: if self.on_task_end: self.on_task_end(body) except Exception as e: logger.info('PikaConsumer: Error %r' % e) channel.basic_ack(delivery_tag=method.delivery_tag)
class PikaClient: def __init__(self, io_loop): print(f'{self.__str__()}: __init__()') self.io_loop = io_loop self.connection = None self.connected = False self.channel = None # 打开链接 def connect(self): if self.connected: return cred = PlainCredentials(USERNAME, PASSWORD) params = ConnectionParameters(host=HOST, port=PORT, virtual_host=VHOST, credentials=cred) self.connection = TornadoConnection(params, on_open_callback=self.on_connected) self.connection.add_on_close_callback(callback=self.on_closed) # 连接成功 callback def on_connected(self, connection): self.connected = True self.connection = connection print(f'{self.__str__()}: connected() succeed') self.connection.channel(on_open_callback=self.on_channel_open) # 关闭连接 callback def on_closed(self, connection): print(f'{self.__str__()}: on_closed()') connection.close() self.connection = None self.connected = False self.io_loop.stop() # 打开通道 callback def on_channel_open(self, channel): self.channel = channel self.channel.exchange_declare(exchange=EXCHANGE, exchange_type="direct", durable=True) # queue如果通过其他方式已经创建的话可能出错, 如果出错先删除再重新创建 try: self.channel.queue_declare(queue=QUEUE, durable=True) except Exception as e: print(f'{self.__str__()} channel.queue_declare Error: ', e) channel.queue_delete(queue=QUEUE) channel.queue_declare(queue=QUEUE, durable=True) self.channel.queue_bind(queue=QUEUE, exchange=EXCHANGE, routing_key=ROUTING_KEY) # 如果是消费者, 绑定消息处理 callback if isinstance(self, Consumer): self.channel.basic_consume(queue=QUEUE, on_message_callback=self.handle_message, auto_ack=True)
class PikaClient(object): """A modified class as described in pika's demo_tornado.py. It handles the connection for the Tornado instance. Messaging/RPC callbacks are handled by the Tornado RequestHandler above.""" def __init__(self): self.connecting = False self.connection = None self.channel = None #self.L = log_class.Logger() def connect(self): if self.connecting: log.info('Already connecting to RabbitMQ.') return #self.L.logger.info("Connecting to RabbitMQ") self.connecting = True creds = pika.PlainCredentials('guest', 'guest') params = pika.ConnectionParameters(host='localhost', port=5672, virtual_host='/', credentials=creds) self.connection = TornadoConnection(params, on_open_callback=self.on_connect) self.connection.add_on_close_callback(self.on_closed) def on_connect(self, connection): self.connection = connection connection.channel(self.on_channel_open) def on_channel_open(self, channel): #self.L.logger.info('Channel Open') self.channel = channel # I'm having trouble using named exchanges. ## channel.exchange_declare(exchange='rpc_ex', type='direct', ## auto_delete=True, durable=False, ## callback=self.on_exchange_declare) def on_exchange_declare(self, frame): log.info("Exchange declared.") def on_basic_cancel(self, frame): log.info('Basic Cancel Ok.') # If we don't have any more consumer processes running close self.connection.close() def on_closed(self, connection): # We've closed our pika connection so stop the demo tornado.ioloop.IOLoop.instance().stop()
class PikaClient(object): def __init__(self, host, queue): self.queue = queue self.host = host self.connected = False self.connecting = False self.connection = None self.channel = None self.callbacks = {} def connect(self): if self.connecting: return self.connecting = True self.connection = TornadoConnection( pika.ConnectionParameters(host=self.host), on_open_callback=self.on_connected ) self.connection.add_on_close_callback(self.on_closed) def on_connected(self, connection): self.connected = True self.connection = connection self.connection.channel(self.on_channel_open) def on_channel_open(self, chanel): self.channel = chanel self.channel.queue_declare( queue=self.queue, durable=True, ) def on_basic_cancel(self, frame): self.connection.close() def on_closed(self, connection): tornado.ioloop.IOLoop.instance().stop() def send(self, body): self.channel.basic_publish( exchange='', routing_key=self.queue, body=json.dumps(body), )
class PikaClient(object): """A modified class as described in pika's demo_tornado.py. It handles the connection for the Tornado instance. Messaging/RPC callbacks are handled by the Tornado RequestHandler above.""" def __init__(self): self.connecting = False self.connection = None self.channel = None # self.L = log_class.Logger() def connect(self): if self.connecting: log.info("Already connecting to RabbitMQ.") return # self.L.logger.info("Connecting to RabbitMQ") self.connecting = True creds = pika.PlainCredentials("guest", "guest") params = pika.ConnectionParameters(host="localhost", port=5672, virtual_host="/", credentials=creds) self.connection = TornadoConnection(params, on_open_callback=self.on_connect) self.connection.add_on_close_callback(self.on_closed) def on_connect(self, connection): self.connection = connection connection.channel(self.on_channel_open) def on_channel_open(self, channel): # self.L.logger.info('Channel Open') self.channel = channel # I'm having trouble using named exchanges. ## channel.exchange_declare(exchange='rpc_ex', type='direct', ## auto_delete=True, durable=False, ## callback=self.on_exchange_declare) def on_exchange_declare(self, frame): log.info("Exchange declared.") def on_basic_cancel(self, frame): log.info("Basic Cancel Ok.") # If we don't have any more consumer processes running close self.connection.close() def on_closed(self, connection): # We've closed our pika connection so stop the demo tornado.ioloop.IOLoop.instance().stop()
class PikaClient(object): """A modified class as described in pika's demo_tornado.py. It handles the connection for the Tornado instance. Messaging/RPC callbacks are handled by the Tornado RequestHandler above.""" def __init__(self): self.connecting = False self.connection = None self.channel = None def connect(self): # print 22222222 if self.connecting: logger.info('Already connecting to RabbitMQ.') return logger.info("Connecting to RabbitMQ") self.connecting = True creds = pika.PlainCredentials('zyl', 'pwd_zyl') params = pika.ConnectionParameters(host='112.74.75.38', port=5672, credentials=creds) self.connection = TornadoConnection(params, on_open_callback=self.on_connect) self.connection.add_on_close_callback(self.on_closed) # print 4444444444 def on_connect(self, connection): self.connection = connection connection.channel(self.on_channel_open) def on_channel_open(self, channel): logger.info('Channel Open') self.channel = channel def on_exchange_declare(self, frame): logger.info("Exchange declared.") def on_basic_cancel(self, frame): logger.info('Basic Cancel Ok.') # If we don't have any more consumer processes running close self.connection.close() def on_closed(self, connection): # We've closed our pika connection so stop the demo print 'close'
class Pika(object): """ rabbitMQ manager. """ def __init__(self): """ rabbitMQ manager init. """ self.connection = None self.connecting = False self.channel = None def connect(self): """ rabbitMQ manager connect. """ if self.connecting: logging.info('Already connecting to RabbitMQ') return logging.info('Connectiong to RabbitMQ') self.connecting = True credentials = pika.PlainCredentials('guest', 'guest') params = pika.ConnectionParameters( host='localhost', port=5672, virtual_host='/smsgo', credentials=credentials) self.connection = TornadoConnection( params, on_open_callback=self.on_connect) self.connection.add_on_close_callback(on_closed) def on_connect(self, connection): """ rabbitMQ manager on connect callback. """ self.connection = connection connection.channel(self.on_channel_open) def on_channel_open(self, channel): """ rabbitMQ manager on channel open callback. """ logging.info('Channel Open') self.channel = channel #def on_exchange_declare(self, frame): #logging.info('Exchange Declare') def on_basic_cancel(self, frame): """ rabbitMQ manager on basic cancel callback. """ logging.info('Basic Cancel Ok') self.connection.close()
class PikaClient(object): def __init__(self, host, queue): self.queue = queue self.host = host self.connected = False self.connecting = False self.connection = None self.channel = None self.callbacks = {} def connect(self): if self.connecting: return self.connecting = True self.connection = TornadoConnection( pika.ConnectionParameters(host=self.host), on_open_callback=self.on_connected) self.connection.add_on_close_callback(self.on_closed) def on_connected(self, connection): self.connected = True self.connection = connection self.connection.channel(self.on_channel_open) def on_channel_open(self, chanel): self.channel = chanel self.channel.queue_declare( queue=self.queue, durable=True, ) def on_basic_cancel(self, frame): self.connection.close() def on_closed(self, connection): tornado.ioloop.IOLoop.instance().stop() def send(self, body): self.channel.basic_publish( exchange='', routing_key=self.queue, body=json.dumps(body), )
class PikaProducer(object): def __init__(self): if tornado is None: raise Exception('You must add tornado to your requirements!') if pika is None: raise Exception('You must add pika to your requirements!') self.ioloop = tornado.ioloop.IOLoop.instance() self.connection = None self.channel = None self._delivery_tag = 0 self.parameters = None self.queue_name = None def connect(self): self.connection = TornadoConnection(self.parameters, on_open_callback=self.on_connected, stop_ioloop_on_close=False) self.connection.add_on_close_callback(self.on_closed) def on_connected(self, connection): logger.info('PikaProducer: connected to RabbitMQ') self.connection.channel(on_open_callback=self.on_channel_declared) def on_channel_declared(self, channel): self.channel = channel self.channel.queue_declare(self.on_queue_declared, queue=self.queue_name) def on_queue_declared(self, method_frame): pass def publish(self, body): self.channel.basic_publish(exchange='', routing_key=self.queue_name, body=body) def on_closed(self, connection): logger.info('PikaProducer: rabbit connection closed') self.connection.close() self.channel.close()
class RabbitClient(object): """Managing incoming messages from Rabbit Service""" def __init__(self, app=None): self.app = app self._connect() def _connect(self): conn = pika.ConnectionParameters( host=CREDS['host'], port=int(CREDS['port']), virtual_host='/', credentials=pika.PlainCredentials( CREDS['user'], CREDS['pasw'])) self.tc = TornadoConnection( conn, on_open_callback=self.on_connected, on_open_error_callback=self.on_disconnect ) self.tc.add_on_close_callback(self.on_disconnect) def on_disconnect(self, *args): logger.warning("Connection lost, reconnect in 5 seconds...") IOLoop.instance().add_timeout( time.time() + 5, self._connect) def on_connected(self, con): con.channel(self.on_channel_open) def on_channel_open(self, channel): channel.basic_consume(consumer_callback=self.on_message, queue=CREDS['queue'], no_ack=True) def on_message(self, channel, method, header, body): self.app.manager.send(body)
class PikaClient(object): def __init__(self, ): # Default values self.connected = False self.connecting = False self.connection = None self.channel = None self.main_future = Future() # A place for us to keep messages sent to us by Rabbitmq self.messages = list() def _make_response(code, message): # TODO: pass def _get_callback_with_future(self, callback, future): def _callback(*args, **kwargs): res = callback(*args, **kwargs) future.set_result(res) return _callback # ----------------------------------------------------- # Connection functions # ----------------------------------------------------- # TODO: make this async def connect(self): if self.connecting: logging.info('Already connecting to RabbitMQ') return logging.info('Connecting to RabbitMQ on localhost:5672') self.connecting = True credentials = pika.PlainCredentials('guest', 'guest') param = pika.ConnectionParameters(host='localhost', port=5672, virtual_host="/", credentials=credentials) self.connection = TornadoConnection(param, on_open_callback=self.on_connected) self.connection.add_on_close_callback(self.on_closed) def on_connected(self, connection): logging.info('Connected to RabbitMQ on localhost:5672') self.connected = True self.connection = connection self._init_channel() def on_closed(self, connection): # We've closed our pika connection so stop the demo tornado.ioloop.IOLoop.instance().stop() # ----------------------------------------------------- # Channel functions # ----------------------------------------------------- @property def channel(self): if self._channel is None or not self._channel.is_open: self._init_channel() return self._channel def _init_channel(self): self.connection.channel(self.on_channel_open) def on_channel_close(self, channel, reply_code, reply_text): logging.info('Channel closed, reply_code=%s, reply_text=%s', reply_code, reply_text) self.main_future.set_result(reply_code) def on_channel_open(self, channel): logging.info('Channel Open') self._channel = channel self._channel.add_on_close_callback(self.on_channel_close) # ----------------------------------------------------- # Queue functions # ----------------------------------------------------- @async_func def queue_declare(self, future, queue): logging.info('Declaring Queue') cb = self._get_callback_with_future(self._on_queue_declare_ok, future) self.channel.queue_declare(queue=queue, durable=True, callback=cb) def _on_queue_declare_ok(self, frame): logging.info('Queue Declared') return frame @async_func def queue_bind(self, future, exchange, queue, routing_key): logging.info('Binding Queue %s to Exchange %s', queue, exchange) cb = self._get_callback_with_future(self.on_queue_bind_ok, future) self.channel.queue_bind(exchange=exchange, queue=queue, routing_key=routing_key, callback=cb) def on_queue_bind_ok(self, frame): logging.info('Queue Bound') return frame def get_messages(): self.channel.basic_consume(consumer_callback=self.on_pika_message, queue=self.queue_name, no_ack=True) def on_pika_message(self, channel, method, header, body): logging.info('Message receive, delivery tag #%i' % \ method.delivery_tag) # Append it to our messages list self.messages.append(body) def on_basic_cancel(self, frame): logging.info('Basic Cancel Ok') # If we don't have any more consumer processes running close self.connection.close() def publish_message(self, exchange, queue): # Build a message to publish to RabbitMQ body = '%.8f: Request from %s [%s]' % \ (tornado_request._start_time, tornado_request.remote_ip, tornado_request.headers.get("User-Agent")) # Send the message properties = pika.BasicProperties(content_type="text/plain", delivery_mode=1) self.channel.basic_publish(exchange='tornado', routing_key='tornado.*', body=body, properties=properties)
class PikaConsumer(object): EXCHANGE_TYPE = "topic" def __init__(self, io_loop="", amqp_url="", exchange="", queue_name="", routing_key=""): self._io_loop = io_loop self._connection = None self._closing = False self._channel = None self._url = amqp_url self._consumer_tag = None self.QUEUE = queue_name or "default_queue" self.ROUTING_KEY = routing_key or "default.get_routing_key" self.EXCHANGE = exchange or "default_exchange" LOG.info("消费者绑定: 交换机{},类型{},队列名字{},路由前缀{}".format( self.EXCHANGE, self.EXCHANGE_TYPE, self.QUEUE, self.ROUTING_KEY)) def connect(self): """连接到rabbitmq, 赋值给self._connection, 并调用self._on_connection_open方法""" self._connection = TornadoConnection( parameters=pika.URLParameters(self._url), on_open_callback=self._on_connection_open) def close_connection(self): self._connection.close() def _on_connection_open(self, unused_connection): """当rabbitmq被打开时, 设置基本信息""" self._add_on_connection_close_callback() self._open_channel() def _add_on_connection_close_callback(self): """当rabbitmq意外关闭与发布者的连接的时候, 调用关闭连接""" self._connection.add_on_close_callback(self._on_connection_closed) def _on_connection_closed(self, connection, reason): """当rabbitmq意外关闭的时候尝试重新连接""" self._channel = None if self._closing: self._io_loop.stop() else: LOG.warning("connection close, reopening in 5 seconds: %s", reason) self._connection.ioloop.call_later(5, self._reconnect) def _reconnect(self): """重新连接""" if not self._closing: self._connection = self.connect() def _open_channel(self): """发布channel, 打开一个新的rabbitmq渠道...成功的时候将调用self._on_channel_open""" self._connection.channel(on_open_callback=self._on_channel_open) def _on_channel_open(self, channel): """当通道打开的时候, 我们申明使用的交换信息""" self._channel = channel self._add_on_channel_close_callback() self._set_exchange(self.EXCHANGE) def _add_on_channel_close_callback(self): """设置通道关闭回调""" self._channel.add_on_close_callback(self._on_channel_closed) def _on_channel_closed(self, channel, reason): self._connection.close() def _set_exchange(self, exchange_name): LOG.info('Declaring exchange %s', exchange_name) self._channel.exchange_declare(callback=self._on_exchange_declareok, exchange=exchange_name, exchange_type=self.EXCHANGE_TYPE) def _on_exchange_declareok(self, unused_frame): self._setup_queue(self.QUEUE) def _setup_queue(self, queue_name): self._channel.queue_declare(callback=self._on_queue_declareok, queue=queue_name) def _on_queue_declareok(self, method_frame): self._channel.queue_bind(callback=self._on_bindok, queue=self.QUEUE, exchange=self.EXCHANGE, routing_key=self.ROUTING_KEY) def _on_bindok(self, unused_frame): self._start_consuming() def _start_consuming(self): self._add_on_cancel_callback() self._consumer_tag = self._channel.basic_consume( on_message_callback=self._on_message, queue=self.QUEUE) def _add_on_cancel_callback(self): self._channel.add_on_cancel_callback(self._on_consumer_cancelled) def _on_consumer_cancelled(self, method_frame): if self._channel: self._channel.close() def _on_message(self, channel, basic_deliver, properties, body): """ 作为消费者接受消息,并做处理 """ # LOG.info("rabbitmq get body: %s", str(body)) code, return_data = self.handler_body(body) return_exchange = properties.message_id return_routing_key = properties.reply_to # return_correlation_id = properties.correlation_id if (return_exchange and return_routing_key and return_data and code): # LOG.info( # "send msg to rabbitmq: exchange: %s, routing_key: %s, body: %s", # return_exchange, return_routing_key, return_data) # props = pika.BasicProperties(correlation_id=return_correlation_id) channel.basic_publish(exchange=return_exchange, body=json.dumps(return_data), routing_key=return_routing_key) elif code: LOG.warning("request properties info: %s", [return_exchange, return_routing_key]) if code: self._acknowledge_message(basic_deliver.delivery_tag) def _acknowledge_message(self, delivery_tag): self._channel.basic_ack(delivery_tag) def handler_body(self, body): return True, "ok"
class PikaClient(object): def __init__(self, config, exchanges): self.connecting = False self.connection = None self.channel = None self.config = config self.exchanges = exchanges def connect(self): if self.connecting: return self.connecting = True host = self.config.get('host') port = 5671 if self.config.get('amqps') else 5672 scheme = 'amqps' if self.config.get('amqps') else 'amqp' virtual_host = urllib.parse.quote(self.config.get('vhost'), safe='') user = self.config.get('user') pw = self.config.get('pw') heartbeat = self.config.get('heartbeat') if self.config.get( 'heartbeat') else 0 params = pika.URLParameters( f"{scheme}://{user}:{pw}@{host}:{port}/{virtual_host}?heartbeat={heartbeat}" ) self.connection = TornadoConnection(params) self.connection.add_on_open_callback(self.on_connect) self.connection.add_on_close_callback(self.on_closed) return def on_connect(self, connection): self.connection = connection self.channel = self.connection.channel( on_open_callback=self.on_channel_open) return def on_channel_open(self, channel): for backend, config in self.exchanges.items(): ex_name = config.get('exchange') channel.exchange_declare(ex_name, exchange_type='topic', durable=True) logging.info(f'rabbitmq exchange: {ex_name} declared') return def on_basic_cancel(self, frame): self.connection.close() def on_closed(self, connection): tornado.ioloop.IOLoop.instance().stop() def publish_message(self, exchange=None, routing_key=None, method=None, uri=None, version=None, data=None, persistent=True): """ Publilsh a message to an exchange. Parameters ---------- exchange: str, exchange name routing_key: str, routing key for topic exchange method: str, HTTP method uri: str, HTTP request URI version: str, e.g. v1 data: dict persistent: bool, default True tell rabbitmq to persist messages to disk, or not """ data = {'method': method, 'uri': uri, 'version': version, 'data': data} message = json.dumps(data) delivery_mode = 2 if persistent else 1 self.channel.basic_publish(exchange=exchange, routing_key=routing_key, body=message, properties=pika.BasicProperties( content_type='application/json', delivery_mode=delivery_mode)) return
class PikaClient(object): def __init__(self, io_loop, ank_accessor, host_address): #pika.log.info('PikaClient: __init__') self.io_loop = io_loop self.connected = False self.connecting = False self.connection = None self.channel = None self.event_listeners = set([]) self.queue_name = 'webserver-%i' % os.getpid() self.ank_accessor = ank_accessor self.host_address = host_address def connect(self): if self.connecting: #pika.log.info('PikaClient: Already connecting to RabbitMQ') return #pika.log.info('PikaClient: Connecting to RabbitMQ') self.connecting = True #cred = pika.PlainCredentials('guest', 'guest') param = pika.ConnectionParameters( host= self.host_address, #port=5672, #virtual_host='/', #credentials=cred ) try: self.connection = TornadoConnection(param, on_open_callback=self.on_connected) self.connection.add_on_close_callback(self.on_closed) except pika.exceptions.AMQPConnectionError: print("Unable to connect to RabbitMQ") def on_connected(self, connection): #pika.log.info('PikaClient: connected to RabbitMQ') self.connected = True self.connection = connection self.connection.channel(self.on_channel_open) def on_channel_open(self, channel): #pika.log.info('PikaClient: Channel open, Declaring exchange') self.channel = channel self.channel.exchange_declare(exchange='www', type="direct", callback=self.on_exchange_declared) return def on_exchange_declared(self, frame): #pika.log.info('PikaClient: Exchange Declared, Declaring Queue') self.channel.queue_declare(queue=self.queue_name, auto_delete=True, durable=False, exclusive=False, callback=self.on_queue_declared) return def on_queue_declared(self, frame): #pika.log.info('PikaClient: Queue Declared, Binding Queue') self.channel.queue_bind(exchange='www', queue=self.queue_name, routing_key='client', callback=self.on_queue_bound) def on_queue_bound(self, frame): #pika.log.info('PikaClient: Queue Bound, Issuing Basic Consume') self.channel.basic_consume(consumer_callback=self.on_message, queue=self.queue_name, no_ack=True) def on_closed(self, connection): #pika.log.info('PikaClient: rabbit connection closed') self.io_loop.stop() def on_message(self, channel, method, header, body): #pika.log.info('PikaClient: message received: %s' % body) import zlib try: body = zlib.decompress(body) except zlib.error: pass # likely not compressed body body_parsed = json.loads(body) if body_parsed.has_key("anm"): print "Received updated network topology" try: self.ank_accessor.anm = body_parsed['anm'] #TODO: could process diff and only update client if data has changed -> more efficient client side self.update_listeners("overlays") # TODO: find better way to replace object not just local reference, as need to replace for RequestHandler too except Exception, e: print "Exception is", e elif body_parsed.has_key("ip_allocations"): alloc = json.loads(body_parsed['ip_allocations']) self.ank_accessor.ip_allocation = alloc self.update_listeners("ip_allocations")
class PikaClient(object): def __init__(self, io_loop): logger.info('Initializing') self.io_loop = io_loop self.connected = False self.connecting = False self.connection = None self.channel = None self.event_listeners = set([]) def connect(self): if self.connecting: logger.info('Already connecting to RabbitMQ') return logger.info('Connecting to RabbitMQ') self.connecting = True cred = pika.PlainCredentials( settings.BAGOU.get('AMQP_BROKER_USER'), settings.BAGOU.get('AMQP_BROKER_PASS')) param = pika.ConnectionParameters( host=settings.BAGOU.get('AMQP_BROKER_ADDR'), port=settings.BAGOU.get('AMQP_BROKER_PORT'), virtual_host=settings.BAGOU.get('AMQP_BROKER_PATH'), credentials=cred) self.connection = TornadoConnection(param, on_open_callback=self.on_connected) self.connection.add_on_close_callback(self.on_closed) def on_connected(self, connection): logger.info('Connected to RabbitMQ') self.connected = True self.connection = connection self.connection.channel(self.on_channel_open) def on_channel_open(self, channel): logger.info('Channel open, Declaring exchange') self.channel = channel self.channel.queue_declare( queue=settings.BAGOU.get('QUEUE_NAME'), durable=True, exclusive=False, auto_delete=True, callback=self.on_queue_declared) def on_queue_declared(self, frame): self.channel.basic_consume(self.on_message, queue=settings.BAGOU.get('QUEUE_NAME')) def on_closed(self, connection): logger.info('RabbitMQ connection closed') self.io_loop.stop() def on_message(self, channel, method, header, body): logger.debug('Message received: %s' % body) self.notify_listeners(body) def notify_listeners(self, event_obj): event_json = json.loads(event_obj) channels = event_json.get('channel') for channel in channels: for listener in self.event_listeners: if channel: if channel in listener.channels: listener.write_message(event_obj) logger.info('Notified %s (channels: %s)' % (repr(listener), listener.channels)) else: listener.write_message(event_obj) logger.info('Notified %s' % repr(listener)) def add_event_listener(self, listener): self.event_listeners.add(listener) logger.info('Listener %s added' % repr(listener)) def remove_event_listener(self, listener): try: self.event_listeners.remove(listener) logger.info('Listener %s removed' % repr(listener)) except KeyError: pass
class SockjsServer(object): def __init__(self, io_loop): self.logger = logging.getLogger(__name__) self.logger.info('SockjsServer: __init__') self.io_loop = io_loop self.connected = False self.connecting = False self.connection = None self.channel = None self.redis = redis_client self.event_listeners_count = 0 self.event_listeners = set() self.connection_dict = dict() self.subscription_dict = defaultdict(set) self.last_reconnect = now() self.uptime_start = now() self.config = SockJSServerSettings() def connect(self): if self.connecting: self.logger.info( 'django-sockjs-server(SockjsServer): Already connecting to RabbitMQ' ) return self.logger.info( 'django-sockjs-server(SockjsServer): Connecting to RabbitMQ') self.connecting = True cred = pika.PlainCredentials(self.config.rabbitmq_user, self.config.rabbitmq_password) param = pika.ConnectionParameters( host=self.config.rabbitmq_host, port=self.config.rabbitmq_port, virtual_host=self.config.rabbitmq_vhost, credentials=cred) try: self.connection = TornadoConnection( param, on_open_callback=self.on_connected) self.connection.add_on_close_callback(self.on_closed) except AMQPConnectionError: self.logger.info( 'django-sockjs-server(SockjsServer): error connect, wait 5 sec' ) time.sleep(5) self.reconnect() self.last_reconnect = now() def on_connected(self, connection): self.logger.info( 'django-sockjs-server(SockjsServer): connected to RabbitMQ') self.connected = True self.connection = connection self.connection.channel(self.on_channel_open) def on_channel_open(self, channel): self.logger.info( 'django-sockjs-server(SockjsServer): Channel open, Declaring exchange' ) self.channel = channel self.channel.exchange_declare( exchange=self.config.rabbitmq_exchange_name, exchange_type=self.config.rabbitmq_exchange_type) self.channel.queue_declare( queue=self.config.rabbitmq_queue1_name, # declare MQ1 exclusive=False, auto_delete=True, callback=self.on_queue1_declared) self.channel.queue_declare( queue=self.config.rabbitmq_queue2_name, # declare MQ2 exclusive=False, auto_delete=True, callback=self.on_queue2_declared) # queue binding def on_queue1_declared(self, frame): # self.logger.info(frame) self.logger.info('django-sockjs-server(SockjsServer): queue1 bind') # self.queue = frame.method.queue self.channel.queue_bind(callback=None, exchange=self.config.rabbitmq_exchange_name, queue=frame.method.queue) self.channel.basic_consume(self.handle_delivery, queue=frame.method.queue, no_ack=True) def on_queue2_declared(self, frame): self.logger.info('django-sockjs-server(SockjsServer): queue2 bind') self.queue = frame.method.queue # declare the host--MQ2 # self.logger.info(self.queue) self.channel.queue_bind(callback=None, exchange=self.config.rabbitmq_exchange_name, queue=frame.method.queue) self.channel.basic_consume(self.handle_delivery, queue=frame.method.queue, no_ack=True) def handle_delivery(self, channel, method, header, body): """Called when we receive a message from RabbitMQ""" # call zhenfeng's function(body: message) here without caring the return # to simulate json_obj = json.loads(body.decode()) # self.logger.info('json_obj >>> :', json_obj) if json_obj['host'] == 'MQ1': self.logger.info("host = MQ1!!!") from django_sockjs_server.lib.client import SockJsServerClient s = SockJsServerClient() json_obj['host'] = 'MQ2' s.publish_message(json_obj, 'MQ2') # put into MQ2 self.logger.info("Has put into MQ2") else: self.logger.info("host = MQ2!!!") self.notify_listeners(body) def handle_delivery2(self, channel, method, header, body: bytes): """ Called when we receive a message from RabbitMQ body like this: (b'{"data": {"user_name": "Sergey Kravchuk", "user_id": 1}, "host": "MQ2", "uid": "262ef88da87537086e0d93f9fc2c40ba", "room": "user2"}',) """ # json_obj = json.loads(body.decode()) # self.logger.info('handle_delivery2 body >>> :', json_obj, json_obj['host']) # self.logger.info('handle_delivery2 body >>> :', type(channel) , type(method), type(header)) self.notify_listeners(body) def on_closed(self, connection, error_code, error_message): self.logger.info( 'django-sockjs-server(SockjsServer): rabbit connection closed, wait 5 seconds' ) connection.add_timeout(5, self.reconnect) def reconnect(self): self.connecting = False self.logger.info('django-sockjs-server(SockjsServer): reconnect') self.connect() def notify_listeners(self, event_json): event_obj = json.loads(event_json) self.logger.debug( 'django-sockjs-server(SockjsServer): send message %s ' % event_obj) try: client = self.connection_dict[event_obj['uid']] except KeyError: self.redis.lrem( event_obj['room'], 0, json.dumps({ 'id': event_obj['uid'], 'host': event_obj['host'] })) else: new_event_json = json.dumps({'data': event_obj['data']}) client.broadcast([client], new_event_json) def add_event_listener(self, listener): self.event_listeners_count += 1 self.event_listeners.add(listener) self.logger.debug( 'django-sockjs-server(SockjsServer): listener %s added' % repr(listener)) def remove_event_listener(self, listener): try: self.event_listeners_count -= 1 self.event_listeners.remove(listener) self.logger.debug( 'django-sockjs-server(SockjsServer): listener %s removed' % repr(listener)) except KeyError: pass def add_subscriber_room(self, room, conn): try: conn_id = conn.id self.connection_dict.setdefault(conn_id, conn) client = self.connection_dict[conn_id] self.subscription_dict[conn_id].add(room) self.logger.debug( 'django-sockjs-server(SockjsServer): listener %s add to room %s' % (repr(client), room)) except KeyError as exc: pass def remove_subscriber(self, conn_id): try: client = self.connection_dict[conn_id] del self.subscription_dict[conn_id] del self.connection_dict[conn_id] self.logger.debug( 'django-sockjs-server(SockjsServer): listener %s del connection %s' % (repr(client), conn_id)) except KeyError as exc: pass def get_event_listeners_count(self): return self.event_listeners_count def get_subscribe_connection_count(self): return len(self.connection_dict.keys()) def get_subscribe_connections(self): return self.connection_dict.keys() def get_last_reconnect(self): return self.last_reconnect def get_uptime(self): return (now() - self.uptime_start).seconds
class PikaClient(object): def __init__(self, config, app=None): self.classifier = Classifier() # Connection params self.host = config['host'] or 'localhost' self.port = config['port'] or '5672' self.vhost = config['vhost'] or '/' self.user = config['user'] or 'guest' self.passwd = config['passwd'] or 'guest' self.exchange = config['exchange'] or 'twitter2' self.queue_name = config['queue_name'] or 'twitter_topic_feed' self.routing_key = config['routing_key'] or 'twitter_topic_feed' # Default values self.connected = False self.connecting = False self.connection = None self.channel = None self.app = app self.event_listeners = set([]) # A place for us to keep messages sent to us by Rabbitmq self.messages = list() # A place for us to put pending messages while we're waiting to connect self.pending = list() self.connect() def connect(self): if self.connecting: print('PikaClient: Already connecting to RabbitMQ') return print('PikaClient: Connecting to RabbitMQ on %s:%i' \ % (self.host, self.port)) self.connecting = True credentials = pika.PlainCredentials(self.user, self.passwd) param = pika.ConnectionParameters(host=self.host, port=self.port, virtual_host=self.vhost, credentials=credentials) logging.debug('Events: Connecting to AMQP Broker: %s:%i' % (self.host, self.port)) # from pika.adapters import SelectConnection # connection = SelectConnection(parameters, on_connected) self.connection = TornadoConnection(param, on_open_callback=self.on_connected) self.connection.add_on_close_callback(self.on_closed) def on_connected(self, connection): print('PikaClient: Connected to RabbitMQ on %s:%i' \ % (self.host, self.port)) self.connected = True self.connection = connection self.connection.channel(self.on_channel_open) def on_channel_open(self, channel): print('PikaClient: Channel Open, Declaring Exchange %s' \ % self.exchange) self.channel = channel self.channel.exchange_declare(exchange=self.exchange, type="direct", durable=False, callback=self.on_exchange_declared) def on_exchange_declared(self, frame): print('PikaClient: Exchange Declared, Declaring Queue %s' \ % self.queue_name) self.channel.queue_declare(queue=self.queue_name, durable=False, exclusive=False, callback=self.on_queue_declared) def on_queue_declared(self, frame): print('PikaClient: Queue Declared, Binding Queue') self.channel.queue_bind(exchange=self.exchange, queue=self.queue_name, routing_key=self.routing_key, callback=self.on_queue_bound) def on_queue_bound(self, frame): print('PikaClient: Queue Bound, Issuing Basic Consume') self.channel.basic_consume(consumer_callback=self.on_message, queue=self.queue_name, no_ack=True) # Send any messages pending for properties, body in self.pending: self.channel.basic_publish(exchange=self.exchange, routing_key=self.routing_key, body=body, properties=properties) def on_basic_cancel(self, frame): print('PikaClient: Basic Cancel Ok') # If we don't have any more consumer processes running close self.connection.close() def on_closed(self, connection): # We've closed our pika connection so stop the demo tornado.ioloop.IOLoop.instance().stop() def on_message(self, channel, method, header, body): print('PikaCient: Message received: %s delivery tag #%i: %s' \ % (header.content_type, method.delivery_tag, body)) # Append it to our messages list self.messages.append(body) self.app.dispatcher.notifyCallbacks(body) self.notify_listeners(body) def notify_listeners(self, event_obj): # here we assume the message the sourcing app # post to the message queue is in JSON format event_json = self.classify_text(event_obj) for listener in self.event_listeners: listener.write_message(event_json) print('PikaClient: notified %s' % repr(listener)) def add_event_listener(self, listener): self.event_listeners.add(listener) print('PikaClient: listener %s added' % repr(listener)) def remove_event_listener(self, listener): try: self.event_listeners.remove(listener) print('PikaClient: listener %s removed' % repr(listener)) except KeyError: pass def get_messages(self): # Get the messages to return, then empty the list output = self.messages self.messages = list() return output def publish(self, msg): # Build a message to publish to RabbitMQ #body = '%.8f: Request from %s [%s]' % \ # (tornado_request._start_time, # tornado_request.remote_ip, # tornado_request.headers.get("User-Agent")) # Send the message properties = pika.BasicProperties(content_type="text/plain", delivery_mode=1) self.channel.basic_publish(exchange=self.exchange, routing_key=self.routing_key, #body='Message: %s - %s' % (msg, body), body='Message: %s' % msg, properties=properties) def classify_text(self,event_json): #Temporaray: Processing should happen in intermediary handler #Parse JSON, return JSON with text geo and polarity print("CLASSIFIER MESSAGE " + event_json) parsedJson =json.loads(event_json) tweettext = parsedJson['text'] tweetgeo = parsedJson['geo'] #tweetplace = parsedJson['place'] #tweetcoords = parsedJson['coordinates'] tweet_polarity = self.classifier.classify(tweettext) return {'text':tweettext, 'geo': tweetgeo, 'polarity':tweet_polarity}#, 'place':tweetplace,'coordinates':tweetcoords}
class AQMPMultiCastAsyncRxQueueManager(RxMultiDelegateQueueWrapper): def __init__(self, io_loop, exchange): super().__init__() self.logger = logging.getLogger(__name__) self.logger.info('AQMPMultiCastAsyncRxQueueManager: __init__') self.io_loop = io_loop self.exchange = exchange self.connected = False self.connecting = False self.connection = None self.channel = None def start(self, routing_key): if self.connecting: self.logger.info('AQMPMultiCastAsyncRxQueueManager: Already connecting to RabbitMQ') return self.routing_key = routing_key self.logger.info('AQMPMultiCastAsyncRxQueueManager: Connecting to RabbitMQ') self.connecting = True cred = pika.PlainCredentials('guest', 'guest') param = pika.ConnectionParameters( host='localhost', port=5672, virtual_host='/', credentials=cred ) self.connection = TornadoConnection(param, on_open_callback=self.on_connected) self.connection.add_on_close_callback(self.on_closed) self.io_loop.start() # Callback 1 def on_connected(self, connection): self.logger.info('AQMPMultiCastAsyncRxQueueManager: connected to RabbitMQ') self.connected = True self.connection = connection self.connection.channel(self.on_channel_open) # Callback 2 def on_channel_open(self, channel): self.logger.info('AQMPMultiCastAsyncRxQueueManager: Channel open, Declaring exchange') self.channel = channel self.channel.queue_declare(exclusive=True, callback=self.on_queue_declared) # Callback 3 def on_queue_declared(self, frame): """Called when RabbitMQ has told us our Queue has been declared, frame is the response from RabbitMQ""" self.queue_name = frame.method.queue self.channel.queue_bind(self.on_queue_bind, exchange=self.exchange, queue=self.queue_name, routing_key=self.routing_key) # Callback 4 def on_queue_bind(self, frame): self.channel.basic_consume(self.on_message, queue=self.queue_name, no_ack=True) def on_closed(self, connection): self.logger.info('AQMPMultiCastAsyncRxQueueManager: rabbit connection closed') self.io_loop.stop() def on_message(self, channel, method, header, body): self.logger.info('AQMPMultiCastAsyncRxQueueManager: message received: %s' % body) self.notify_delegates(body)
class PikaClient(object): def __init__(self, io_loop): logger.info('Initializing') self.io_loop = io_loop self.connected = False self.connecting = False self.connection = None self.channel = None self.event_listeners = set([]) def connect(self): if self.connecting: logger.info('Already connecting to RabbitMQ') return logger.info('Connecting to RabbitMQ') self.connecting = True cred = pika.PlainCredentials(settings.BAGOU.get('AMQP_BROKER_USER'), settings.BAGOU.get('AMQP_BROKER_PASS')) param = pika.ConnectionParameters( host=settings.BAGOU.get('AMQP_BROKER_ADDR'), port=settings.BAGOU.get('AMQP_BROKER_PORT'), virtual_host=settings.BAGOU.get('AMQP_BROKER_PATH'), credentials=cred) self.connection = TornadoConnection(param, on_open_callback=self.on_connected) self.connection.add_on_close_callback(self.on_closed) def on_connected(self, connection): logger.info('Connected to RabbitMQ') self.connected = True self.connection = connection self.connection.channel(self.on_channel_open) def on_channel_open(self, channel): logger.info('Channel open, Declaring exchange') self.channel = channel self.channel.queue_declare(queue=settings.BAGOU.get('QUEUE_NAME'), durable=True, exclusive=False, auto_delete=True, callback=self.on_queue_declared) def on_queue_declared(self, frame): self.channel.basic_consume(self.on_message, queue=settings.BAGOU.get('QUEUE_NAME')) def on_closed(self, connection): logger.info('RabbitMQ connection closed') self.io_loop.stop() def on_message(self, channel, method, header, body): logger.debug('Message received: %s' % body) self.notify_listeners(body) def notify_listeners(self, event_obj): event_json = json.loads(event_obj) channels = event_json.get('channel') for channel in channels: for listener in self.event_listeners: if channel: if channel in listener.channels: listener.write_message(event_obj) logger.info('Notified %s (channels: %s)' % (repr(listener), listener.channels)) else: listener.write_message(event_obj) logger.info('Notified %s' % repr(listener)) def add_event_listener(self, listener): self.event_listeners.add(listener) logger.info('Listener %s added' % repr(listener)) def remove_event_listener(self, listener): try: self.event_listeners.remove(listener) logger.info('Listener %s removed' % repr(listener)) except KeyError: pass
class PikaClient(object): tornado_callback = None _closing = False _connect_index = 0 _connect_pull = None _one_respond_made = None _waiting_to_reconnect = 3 _expire_reconnect = 15 _last_reconnect_fail = None def __init__(self, logger, queue_answer, queue_read, queue_create, node_list): # Construct a queue name we'll use for this instance only self.queue_answer = queue_answer # Create queue for sending self.queue_read = queue_read self.queue_create = queue_create self.logger = logger self.connected = False self.connecting = False self.connection = None self.channel = None # A place for us to keep messages sent to us by Rabbitmq self.messages = list() # A place for us to put pending messages while we're waiting to connect self.pending = list() self._connect_pull = [] for node in node_list: self._connect_pull.append( pika.ConnectionParameters( host=node[0], port=int(node[1]))) def connect(self): if self.connecting: self.logger.warning('Already connecting to RabbitMQ') return param = self._connect_pull[self._connect_index] self.logger.debug('Connecting to RabbitMQ on ' '{host}:{port}'.format(host=param.host, port=param.port)) self.connecting = True # TODO: add on_connection_error try: self.connection = TornadoConnection( param, on_open_callback=self.on_connected ) self.connection.add_on_close_callback(self.on_closed) except AMQPConnectionError: self.reconnect() def on_connected(self, connection): self.logger.debug('Connected to RabbitMQ on: ' '{connection}'.format(connection=connection)) self.connected = True self.connection = connection self.connection.channel(self.on_channel_open) def on_channel_open(self, channel): self.logger.debug('Channel Open, Declaring Exchange') self.channel = channel self.channel.exchange_declare(exchange='tornado', type='topic', durable=True, callback=self.on_exchange_declared) def on_exchange_declared(self, frame): self.logger.debug('Exchange Declared, Declaring Queue') self.channel.queue_declare(queue=self.queue_answer, durable=True, callback=self.on_queue_declared) self.channel.queue_declare(queue=self.queue_create, durable=True, callback=lambda frame: self.channel.queue_bind( exchange='tornado', queue=self.queue_create, routing_key=self.queue_create, callback=None)) self.channel.queue_declare(queue=self.queue_read, durable=True, callback=lambda frame: self.channel.queue_bind( exchange='tornado', queue=self.queue_read, routing_key=self.queue_read, callback=None)) def on_queue_declared(self, frame): self.logger.debug('Queue Declared, Binding Queue') self.channel.queue_bind(exchange='tornado', queue=self.queue_answer, callback=self.on_queue_bound) def on_queue_bound(self, frame): self.logger.debug('Queue Bound, Issuing Basic Consume') self.channel.basic_consume(consumer_callback=self.on_pika_message, queue=self.queue_answer, no_ack=True) # TODO: still not implemented # for properties, body in self.pending: # self.logger.debug('Pending Message:' # ' %s | %s' % (properties, body)) # self.channel.basic_publish(exchange='tornado', # # TODO: save routing_key or # # it already in properties # routing_key='reading', # body=body, # properties=properties) def on_pika_message(self, channel, method, header, body): self.logger.debug('Message receive: ' 'body: {body}'.format(method=method, header=header, body=body)) if self.tornado_callback and not self._one_respond_made: self.messages.append(body) self.tornado_callback(self.get_messages()) self._one_respond_made = True def on_basic_cancel(self, frame): self.logger.debug('Basic Cancel Ok') self.connection.close() def on_closed(self, *args): self.logger.warning('On closed. Try to reconnect...') self.reconnect() def reconnect(self): self.logger.warning('Fail to connect') if self._last_reconnect_fail: current_fail_time = datetime.datetime.now() fail_timedelta = current_fail_time - self._last_reconnect_fail self.logger.debug('Check reconnect expires, ' 'timedelta: %s seconds' % fail_timedelta.seconds) if fail_timedelta.seconds >= self._expire_reconnect: self._closing = False self._connect_index = 0 self._last_reconnect_fail = datetime.datetime.now() self.logger.debug('Reconnect time is expired, ' 'reset reconnect parameters') else: self.logger.warning('Set first fail reconnect time') self._last_reconnect_fail = datetime.datetime.now() self.connecting = False self.connected = False self.logger.warning('%s sec waiting...' % self._waiting_to_reconnect) time.sleep(self._waiting_to_reconnect) if not self._closing: self._connect_index += 1 self.logger.warning('Try to reconnect to %s, try number %s' % (self._connect_pull[self._connect_index], self._connect_index)) if self._connect_index == len(self._connect_pull) - 1: self._closing = True self.connect() else: self.logger.warning('Closing. Stop trying') self.stop() def stop(self): self.logger.warning('STOP') self._closing = True tornado.ioloop.IOLoop.instance().stop() def sample_message(self, msg, routing_key, tornado_callback): self.logger.debug('Sample Message to %s' % routing_key) self.tornado_callback = tornado_callback properties = pika.BasicProperties(delivery_mode=1) self._one_respond_made = False self.channel.basic_publish(exchange='tornado', routing_key=routing_key, body=msg, properties=properties) def get_messages(self): output = self.messages self.messages = list() return output
class PikaPublisher(object): def __init__(self, amqp_parameters, exchange, exchange_type, queue, routing_key): self._exchange = exchange self._exchange_type = exchange_type self._queue = queue self._routing_key = routing_key self._connection = None self._channel = None self._deliveries = [] self._acked = 0 self._nacked = 0 self._message_number = 0 self._stopping = False self._params = amqp_parameters self._closing = False logging.basicConfig(level=logging.INFO, format=LOG_FORMAT) def connect(self): self._connection = TornadoConnection(self._params, self.on_connection_open, stop_ioloop_on_close=False) LOGGER.info('Adding connection close callback') self._connection.add_on_close_callback(self.on_connection_closed) return self._connection def close_connection(self): LOGGER.info('Closing connection') self._closing = True self._connection.close() def on_connection_closed(self, connection, reply_code, reply_text): self._channel = None if self._closing: self._connection.ioloop.stop() else: LOGGER.warning('Connection closed, reopening in 5 seconds: (%s) %s', reply_code, reply_text) self._connection.add_timeout(5, self.reconnect) def on_connection_open(self, unused_connection): LOGGER.info('Connection opened') self.open_channel() def reconnect(self): # This is the old connection IOLoop instance, stop its ioloop self._connection.ioloop.stop() # Create a new connection self._connection = self.connect() # There is now a new connection, needs a new ioloop to run self._connection.ioloop.start() def add_on_channel_close_callback(self): LOGGER.info('Adding channel close callback') self._channel.add_on_close_callback(self.on_channel_closed) def on_channel_closed(self, channel, reply_code, reply_text): LOGGER.warning('Channel was closed: (%s) %s', reply_code, reply_text) if not self._closing: self._connection.close() def on_channel_open(self, channel): LOGGER.info('Channel opened') self._channel = channel self.add_on_channel_close_callback() self.setup_exchange(self._exchange) def setup_exchange(self, exchange_name): LOGGER.info('Declaring exchange %s', exchange_name) self._channel.exchange_declare(self.on_exchange_declareok, exchange_name, self._exchange_type, durable=True) def on_exchange_declareok(self, unused_frame): LOGGER.info('Exchange declared') self.setup_queue(self._queue) def setup_queue(self, queue_name): LOGGER.info('Declaring queue %s', queue_name) self._channel.queue_declare(self.on_queue_declareok, queue_name, durable = True) def on_queue_declareok(self, method_frame): LOGGER.info('Binding %s to %s with %s', self._exchange, self._queue, self._routing_key) self._channel.queue_bind(self.on_bindok, self._queue, self._exchange, self._routing_key) def on_delivery_confirmation(self, method_frame): confirmation_type = method_frame.method.NAME.split('.')[1].lower() LOGGER.info('Received %s for delivery tag: %i', confirmation_type, method_frame.method.delivery_tag) if confirmation_type == 'ack': self._acked += 1 elif confirmation_type == 'nack': self._nacked += 1 self._deliveries.remove(method_frame.method.delivery_tag) LOGGER.info('Published %i messages, %i have yet to be confirmed, ' '%i were acked and %i were nacked', self._message_number, len(self._deliveries), self._acked, self._nacked) def enable_delivery_confirmations(self): LOGGER.info('Issuing Confirm.Select RPC command') self._channel.confirm_delivery(self.on_delivery_confirmation) def publish_message(self, message): if self._stopping: return properties = pika.BasicProperties(app_id='example-publisher', content_type='application/json', delivery_mode = 2, headers=message) self._channel.basic_publish(self._exchange, self._routing_key, json.dumps(message, ensure_ascii=False), properties) self._message_number += 1 self._deliveries.append(self._message_number) LOGGER.info('Published message # %i', self._message_number) def on_bindok(self, unused_frame): LOGGER.info('Queue bound') def close_channel(self): LOGGER.info('Closing the channel') if self._channel: self._channel.close() def open_channel(self): LOGGER.info('Creating a new channel') self._connection.channel(on_open_callback=self.on_channel_open) def stop(self): LOGGER.info('Stopping') self._stopping = True self.close_channel() self.close_connection() self._connection.ioloop.start() LOGGER.info('Stopped')
class SockjsServer(object): def __init__(self, io_loop): self.logger = logging.getLogger(__name__) self.logger.info('SockjsServer: __init__') self.io_loop = io_loop self.connected = False self.connecting = False self.connection = None self.channel = None self.redis = redis_client self.event_listeners_count = 0 self.event_listeners = set() self.connection_dict = dict() self.subscription_dict = defaultdict(set) self.last_reconnect = now() self.uptime_start = now() self.config = SockJSServerSettings() def connect(self): if self.connecting: self.logger.info('django-sockjs-server(SockjsServer): Already connecting to RabbitMQ') return self.logger.info('django-sockjs-server(SockjsServer): Connecting to RabbitMQ') self.connecting = True cred = pika.PlainCredentials(self.config.rabbitmq_user, self.config.rabbitmq_password) param = pika.ConnectionParameters( host=self.config.rabbitmq_host, port=self.config.rabbitmq_port, virtual_host=self.config.rabbitmq_vhost, credentials=cred ) try: self.connection = TornadoConnection(param, on_open_callback=self.on_connected) self.connection.add_on_close_callback(self.on_closed) except AMQPConnectionError: self.logger.info('django-sockjs-server(SockjsServer): error connect, wait 5 sec') time.sleep(5) self.reconnect() self.last_reconnect = now() def on_connected(self, connection): self.logger.info('django-sockjs-server(SockjsServer): connected to RabbitMQ') self.connected = True self.connection = connection self.connection.channel(self.on_channel_open) def on_channel_open(self, channel): self.logger.info('django-sockjs-server(SockjsServer): Channel open, Declaring exchange') self.channel = channel self.channel.exchange_declare(exchange=self.config.rabbitmq_exchange_name, exchange_type=self.config.rabbitmq_exchange_type) self.channel.queue_declare( queue=self.config.rabbitmq_queue_name, exclusive=False, auto_delete=True, callback=self.on_queue_declared ) def on_queue_declared(self, frame): self.logger.info('django-sockjs-server(SockjsServer): queue bind') self.queue = frame.method.queue self.channel.queue_bind(callback=None, exchange=self.config.rabbitmq_exchange_name, queue=frame.method.queue) self.channel.basic_consume(self.handle_delivery, queue=frame.method.queue, no_ack=True) def handle_delivery(self, channel, method, header, body): """Called when we receive a message from RabbitMQ""" self.notify_listeners(body) def on_closed(self, connection, error_code, error_message): self.logger.info('django-sockjs-server(SockjsServer): rabbit connection closed, wait 5 seconds') connection.add_timeout(5, self.reconnect) def reconnect(self): self.connecting = False self.logger.info('django-sockjs-server(SockjsServer): reconnect') self.connect() def notify_listeners(self, event_json): event_obj = json.loads(event_json) self.logger.debug('django-sockjs-server(SockjsServer): send message %s ' % event_obj) try: client = self.connection_dict[event_obj['uid']] except KeyError: self.redis.lrem(event_obj['room'], 0, json.dumps({'id': event_obj['uid'], 'host': event_obj['host']})) else: new_event_json = json.dumps({'data': event_obj['data']}) client.broadcast([client], new_event_json) def add_event_listener(self, listener): self.event_listeners_count += 1 self.event_listeners.add(listener) self.logger.debug('django-sockjs-server(SockjsServer): listener %s added' % repr(listener)) def remove_event_listener(self, listener): try: self.event_listeners_count -= 1 self.event_listeners.remove(listener) self.logger.debug('django-sockjs-server(SockjsServer): listener %s removed' % repr(listener)) except KeyError: pass def add_subscriber_room(self, room, conn): try: conn_id = conn.id self.connection_dict.setdefault(conn_id, conn) client = self.connection_dict[conn_id] self.subscription_dict[conn_id].add(room) self.logger.debug('django-sockjs-server(SockjsServer): listener %s add to room %s' % (repr(client), room)) except KeyError, exc: pass
class PikaClient(object): def __init__(self, settings): self.settings = settings # Construct a queue name we'll use for this instance only self.queue_name = "tornado-test-%i" % os.getpid() # Default values self.connected = False self.connecting = False self.connection = None self.channel = None # A place for us to keep messages sent to us by Rabbitmq self.messages = list() # A place for us to put pending messages while we're waiting to connect self.pending = list() def connect(self): if self.connecting: pika.log.info("PikaClient: Already connecting to RabbitMQ") return pika.log.info("PikaClient: Connecting to RabbitMQ on localhost:5672") self.connecting = True credentials = pika.PlainCredentials(self.settings.user, self.settings.password) param = pika.ConnectionParameters( host=self.settings.host, port=self.settings.port, virtual_host=self.settings.virtual_host, credentials=credentials, ) self.connection = TornadoConnection(param, on_open_callback=self.on_connected) self.connection.add_on_close_callback(self.on_closed) def on_connected(self, connection): pika.log.info("PikaClient: Connected to RabbitMQ on localhost:5672") self.connected = True self.connection = connection self.connection.channel(self.on_channel_open) def on_channel_open(self, channel): pika.log.info("PikaClient: Channel Open, Declaring Exchange") self.channel = channel self.channel.exchange_declare( exchange="tornado", type="direct", auto_delete=True, durable=False, callback=self.on_exchange_declared ) def on_exchange_declared(self, frame): pika.log.info("PikaClient: Exchange Declared, Declaring Queue") self.channel.queue_declare( queue=self.queue_name, auto_delete=True, durable=False, exclusive=False, callback=self.on_queue_declared ) def on_queue_declared(self, frame): pika.log.info("PikaClient: Queue Declared, Binding Queue") self.channel.queue_bind( exchange="tornado", queue=self.queue_name, routing_key="tornado.*", callback=self.on_queue_bound ) def on_queue_bound(self, frame): pika.log.info("PikaClient: Queue Bound, Issuing Basic Consume") self.channel.basic_consume(consumer_callback=self.on_pika_message, queue=self.queue_name, no_ack=True) # Send any messages pending for properties, body in self.pending: self.channel.basic_publish(exchange="tornado", routing_key="tornado.*", body=body, properties=properties) def on_pika_message(self, channel, method, header, body): pika.log.info("PikaCient: Message receive, delivery tag #%i" % method.delivery_tag) # Append it to our messages list self.messages.append(body) # ============================================================================== self.callback(body) # 3.use external callback(self.send) on here def cb(self, callback): # 1. use app.pika.cb for setting external callback function self.callback = callback # 2.assign external callback to self.callback # ============================================================================== def on_basic_cancel(self, frame): pika.log.info("PikaClient: Basic Cancel Ok") # If we don't have any more consumer processes running close self.connection.close() def on_closed(self, connection): # TODO:We've closed our pika connection so stop the demo tornado.ioloop.IOLoop.instance().stop() def sample_message(self, msg): # Build a message to publish to RabbitMQ body = msg # Send the message properties = pika.BasicProperties(content_type="text/plain", delivery_mode=1) self.channel.basic_publish(exchange="tornado", routing_key="tornado.*", body=body, properties=properties) def get_messages(self): # Get the messages to return, then empty the list output = self.messages self.messages = list() return output
class AsyncTransport(BaseTransport): def _connect(self): self._connection = TornadoConnection( self._parameters, on_open_callback=self.on_connection_open, stop_ioloop_on_close=False ) def on_connection_open(self, unused_connection): """This method is called by pika once the connection to RabbitMQ has been established. It passes the handle to the connection object in case we need it, but in this case, we'll just mark it unused. :type unused_connection: pika.SelectConnection """ self._connection.add_on_close_callback(self.on_connection_closed) self._open_channel() def on_connection_closed(self, method_frame): self._connection = None self._connect() def close(self): self._connection.close() def _open_channel(self): """Open a new channel with RabbitMQ by issuing the Channel.Open RPC command. When RabbitMQ responds that the channel is open, the on_channel_open callback will be invoked by pika. """ self._connection.channel(self._on_channel_open) def _on_channel_open(self, channel): self._channel = channel self._channel.basic_qos(prefetch_count=1) self.add_on_channel_close_callback() self.setup_exchange() def add_on_channel_close_callback(self): """This method tells pika to call the on_channel_closed method if RabbitMQ unexpectedly closes the channel. """ self._channel.add_on_close_callback(self.on_channel_closed) def on_channel_closed(self, channel, reply_code, reply_text): """Invoked by pika when RabbitMQ unexpectedly closes the channel. Channels are usually closed if you attempt to do something that violates the protocol, such as re-declare an exchange or queue with different parameters. In this case, we'll close the connection to shutdown the object. :param pika.channel.Channel: The closed channel :param int reply_code: The numeric reason the channel was closed :param str reply_text: The text reason the channel was closed """ LOGGER.warning('Channel %i was closed: (%s) %s', channel, reply_code, reply_text) self._connection.close() def setup_exchange(self): """Setup the exchange on RabbitMQ by invoking the Exchange.Declare RPC command. When it is complete, the on_exchange_declareok method will be invoked by pika. :param str|unicode exchange_name: The name of the exchange to declare """ self._channel.exchange_declare(self.on_exchange_declareok, self.EXCHANGE_NAME, self.EXCHANGE_TYPE) def on_exchange_declareok(self, unused_frame): """Invoked by pika when RabbitMQ has finished the Exchange.Declare RPC command. :param pika.Frame.Method unused_frame: Exchange.DeclareOk response frame """ LOGGER.info('Exchange declared') self.setup_queue(self.QUEUE) def setup_queue(self, queue_name): """Setup the queue on RabbitMQ by invoking the Queue.Declare RPC command. When it is complete, the on_queue_declareok method will be invoked by pika. :param str|unicode queue_name: The name of the queue to declare. """ LOGGER.info('Declaring queue %s', queue_name) self._channel.queue_declare(self.on_queue_declareok, queue_name) def on_queue_declareok(self, method_frame): """Method invoked by pika when the Queue.Declare RPC call made in setup_queue has completed. In this method we will bind the queue and exchange together with the routing key by issuing the Queue.Bind RPC command. When this command is complete, the on_bindok method will be invoked by pika. :param pika.frame.Method method_frame: The Queue.DeclareOk frame """ LOGGER.info('Binding %s to %s with %s', self.EXCHANGE_NAME, self.QUEUE, self.ROUTING_KEY) self._channel.queue_bind(self.on_bindok, self.QUEUE, self.EXCHANGE_NAME, self.ROUTING_KEY) def add_on_cancel_callback(self): """Add a callback that will be invoked if RabbitMQ cancels the consumer for some reason. If RabbitMQ does cancel the consumer, on_consumer_cancelled will be invoked by pika. """ LOGGER.info('Adding consumer cancellation callback') self._channel.add_on_cancel_callback(self.on_consumer_cancelled) def on_consumer_cancelled(self, method_frame): """Invoked by pika when RabbitMQ sends a Basic.Cancel for a consumer receiving messages. :param pika.frame.Method method_frame: The Basic.Cancel frame """ LOGGER.info('Consumer was cancelled remotely, shutting down: %r', method_frame) if self._channel: self._channel.close() def acknowledge_message(self, delivery_tag): """Acknowledge the message delivery from RabbitMQ by sending a Basic.Ack RPC method for the delivery tag. :param int delivery_tag: The delivery tag from the Basic.Deliver frame """ LOGGER.info('Acknowledging message %s', delivery_tag) self._channel.basic_ack(delivery_tag) def on_message(self, unused_channel, basic_deliver, properties, body): """Invoked by pika when a message is delivered from RabbitMQ. The channel is passed for your convenience. The basic_deliver object that is passed in carries the exchange, routing key, delivery tag and a redelivered flag for the message. The properties passed in is an instance of BasicProperties with the message properties and the body is the message that was sent. :param pika.channel.Channel unused_channel: The channel object :param pika.Spec.Basic.Deliver: basic_deliver method :param pika.Spec.BasicProperties: properties :param str|unicode body: The message body """ LOGGER.info('Received message # %s from %s: %s', basic_deliver.delivery_tag, properties.app_id, body) self.work_q.put(json.loads(body)) self.acknowledge_message(basic_deliver.delivery_tag) def on_cancelok(self, unused_frame): """This method is invoked by pika when RabbitMQ acknowledges the cancellation of a consumer. At this point we will close the channel. This will invoke the on_channel_closed method once the channel has been closed, which will in-turn close the connection. :param pika.frame.Method unused_frame: The Basic.CancelOk frame """ LOGGER.info('RabbitMQ acknowledged the cancellation of the consumer') self.close_channel() def stop_consuming(self): """Tell RabbitMQ that you would like to stop consuming by sending the Basic.Cancel RPC command. """ if self._channel: LOGGER.info('Sending a Basic.Cancel RPC command to RabbitMQ') self._channel.basic_cancel(self.on_cancelok, self._consumer_tag) def start_consuming(self): """This method sets up the consumer by first calling add_on_cancel_callback so that the object is notified if RabbitMQ cancels the consumer. It then issues the Basic.Consume RPC command which returns the consumer tag that is used to uniquely identify the consumer with RabbitMQ. We keep the value to use it when we want to cancel consuming. The on_message method is passed in as a callback pika will invoke when a message is fully received. """ LOGGER.info('Issuing consumer related RPC commands') self.add_on_cancel_callback() self._consumer_tag = self._channel.basic_consume(self.on_message, self.QUEUE) def on_bindok(self, unused_frame): """Invoked by pika when the Queue.Bind method has completed. At this point we will start consuming messages by calling start_consuming which will invoke the needed RPC commands to start the process. :param pika.frame.Method unused_frame: The Queue.BindOk response frame """ LOGGER.info('Queue bound') self.start_consuming()
class PikaClient(object): def __init__(self, io_loop): self.logger = logging.getLogger(__name__) self.logger.info('PikaClient: __init__') self.io_loop = io_loop self.connected = False self.connecting = False self.connection = None self.channel = None self.redis = redis_client self.event_listeners_count = 0 self.event_listeners = set() self.subscrib_channel = dict() self.last_reconnect = now() self.uptime_start = now() self.config = SockJSServerSettings() def connect(self): if self.connecting: self.logger.info( 'django-sockjs-server(PikaClient): Already connecting to RabbitMQ' ) return self.logger.info( 'django-sockjs-server(PikaClient): Connecting to RabbitMQ') self.connecting = True cred = pika.PlainCredentials(self.config.rabbitmq_user, self.config.rabbitmq_password) param = pika.ConnectionParameters( host=self.config.rabbitmq_host, port=self.config.rabbitmq_port, virtual_host=self.config.rabbitmq_vhost, credentials=cred) try: self.connection = TornadoConnection( param, on_open_callback=self.on_connected) self.connection.add_on_close_callback(self.on_closed) except AMQPConnectionError: self.logger.info( 'django-sockjs-server(PikaClient): error connect, wait 5 sec') time.sleep(5) self.reconnect() self.last_reconnect = now() def on_connected(self, connection): self.logger.info( 'django-sockjs-server(PikaClient): connected to RabbitMQ') self.connected = True self.connection = connection self.connection.channel(self.on_channel_open) def on_channel_open(self, channel): self.logger.info( 'django-sockjs-server(PikaClient): Channel open, Declaring exchange' ) self.channel = channel self.channel.exchange_declare( exchange=self.config.rabbitmq_exchange_name, exchange_type=self.config.rabbitmq_exchange_type) self.channel.queue_declare(queue=self.config.rabbitmq_queue_name, exclusive=False, auto_delete=True, callback=self.on_queue_declared) def on_queue_declared(self, frame): self.logger.info('django-sockjs-server(PikaClient): queue bind') self.queue = frame.method.queue self.channel.queue_bind(callback=None, exchange=self.config.rabbitmq_exchange_name, queue=frame.method.queue) self.channel.basic_consume(self.handle_delivery, queue=frame.method.queue, no_ack=True) def handle_delivery(self, channel, method, header, body): """Called when we receive a message from RabbitMQ""" self.notify_listeners(body) def on_closed(self, connection, error_code, error_message): self.logger.info( 'django-sockjs-server(PikaClient): rabbit connection closed, wait 5 seconds' ) connection.add_timeout(5, self.reconnect) def reconnect(self): self.connecting = False self.logger.info('django-sockjs-server(PikaClient): reconnect') self.connect() def notify_listeners(self, event_json): event_obj = json.loads(event_json) self.logger.debug( 'django-sockjs-server(PikaClient): send message %s ' % event_obj) try: channel = self.subscrib_channel[event_obj['uid']] except KeyError: self.redis.lrem( event_obj['room'], 0, json.dumps({ 'id': event_obj['uid'], 'host': event_obj['host'] })) else: client = channel['conn'] new_event_json = json.dumps({'data': event_obj['data']}) client.broadcast([client], new_event_json) def add_event_listener(self, listener): self.event_listeners_count += 1 self.event_listeners.add(listener) self.logger.debug( 'django-sockjs-server(PikaClient): listener %s added' % repr(listener)) def remove_event_listener(self, listener): try: self.event_listeners_count -= 1 self.event_listeners.remove(listener) self.logger.debug( 'django-sockjs-server(PikaClient): listener %s removed' % repr(listener)) except KeyError: pass def add_subscriber_channel(self, conn_id, room, client): self.subscrib_channel[conn_id] = {'room': room, 'conn': client} self.logger.debug( 'django-sockjs-server(PikaClient): listener %s add to room %s' % (repr(client), room)) def remove_subscriber_channel(self, conn_id, client): try: room = self.subscrib_channel[conn_id].get('room') del self.subscrib_channel[conn_id] self.logger.debug( 'django-sockjs-server(PikaClient): listener %s del connection %s from room %s' % (repr(client), conn_id, room)) except KeyError: pass def get_event_listeners_count(self): return self.event_listeners_count def get_subscribe_channel_count(self): return len(self.subscrib_channel.keys()) def get_subscribe_channels(self): return self.subscrib_channel.keys() def get_last_reconnect(self): return self.last_reconnect def get_uptime(self): return (now() - self.uptime_start).seconds
class MQConnection(object): """ MQ连接管理类 """ def __init__(self, url, type='producer', callback=None, *arg, **settings): """Create a new instance of the MQConnection class, passing in the AMQP URL used to connect to RabbitMQ. :param str amqp_url: The AMQP url to connect with :param str type: connection type,for excmple,'consumer','producer' :param str callback: if type is 'consumer',callback is not None """ self._connection = None self._channel = None self._closing = False self._consumer_tag = None self._url = url self._type = type self._was_consuming = False self._was_publishing = False self._reconnect_delay = 0 self._callback = callback self.EXCHANGE = settings.get('exchange') self.QUEUE = settings.get('queue') self.ROUTING_KEY = settings.get('routing_key') self.EXCHANGE_TYPE = settings.get('exchange_type') self.AE_EXCHANGE = settings.get('ae_exchange') self.AE_QUEUE = settings.get('ae_queue') self.AE_EXCHANGE_TYPE = settings.get('ae_exchange_type') self.DL_EXCHANGE = settings.get('dl_exchange') self.DL_QUEUE = settings.get('dl_queue') self.DL_EXCHANGE_TYPE = settings.get('dl_exchange_type') self._passive = settings.get('passive', True) self._durable = settings.get('durable', True) self._prefetch_count = settings.get('prefetch_count', 128) @exception_catch def connect(self): """This method connects to RabbitMQ, returning the connection handle. When the connection is established, the on_connection_open method will be invoked by pika. :rtype: pika.SelectConnection """ LOGGER.info('Connecting to %s', self._url) self._connection = TornadoConnection( pika.URLParameters(self._url), on_open_callback=self.on_connection_open, on_open_error_callback=self.on_connection_open_error) return self._connection def on_connection_open_error(self, _unused_connection, err): """This method is called by pika if the connection to RabbitMQ can't be established. :param pika.SelectConnection _unused_connection: The connection :param Exception err: The error """ reconnect_delay = self._get_reconnect_delay() LOGGER.error('Connection open failed, reopening in %d seconds: %s', reconnect_delay, err) self._connection.ioloop.call_later(reconnect_delay, self.reconnect) def close_connection(self): """This method closes the connection to RabbitMQ.""" LOGGER.info('Closing connection') self._connection.close() def add_on_connection_close_callback(self): """This method adds an on close callback that will be invoked by pika when RabbitMQ closes the connection to the publisher unexpectedly. """ LOGGER.info('Adding connection close callback') self._connection.add_on_close_callback(self.on_connection_closed) def on_connection_closed(self, connection, reason): """This method is invoked by pika when the connection to RabbitMQ is closed unexpectedly. Since it is unexpected, we will reconnect to RabbitMQ if it disconnects. :param pika.connection.Connection connection: The closed connection obj :param Exception reason: exception representing reason for loss of connection. """ self._channel = None if self._closing: pass # self._connection.ioloop.stop() else: reconnect_delay = self._get_reconnect_delay() LOGGER.warning('Connection closed, reopening in %d seconds: %s', reconnect_delay, reason) self._connection.ioloop.call_later(reconnect_delay, self.reconnect) def on_connection_open(self, unused_connection): """This method is called by pika once the connection to RabbitMQ has been established. It passes the handle to the connection object in case we need it, but in this case, we'll just mark it unused. :param pika.SelectConnection _unused_connection: The connection """ LOGGER.info('Connection opened') self.add_on_connection_close_callback() self.open_channel() def reconnect(self): """Will be invoked by the IOLoop timer if the connection is closed. See the on_connection_closed method. """ self._was_consuming = False self._was_publishing = False if not self._closing: # Create a new connection self._connection = self.connect() def add_on_channel_close_callback(self): """This method tells pika to call the on_channel_closed method if RabbitMQ unexpectedly closes the channel. """ LOGGER.info('Adding channel close callback') self._channel.add_on_close_callback(self.on_channel_closed) def on_channel_closed(self, channel, reason): """Invoked by pika when RabbitMQ unexpectedly closes the channel. Channels are usually closed if you attempt to do something that violates the protocol, such as re-declare an exchange or queue with different parameters. In this case, we'll close the connection to shutdown the object. :param pika.channel.Channel: The closed channel :param Exception reason: why the channel was closed """ LOGGER.warning('Channel %i was closed: %s', channel, reason) if self._connection.is_open: self._connection.close() def on_channel_open(self, channel): """This method is invoked by pika when the channel has been opened. The channel object is passed in so we can make use of it. Since the channel is now open, we'll declare the exchange to use. :param pika.channel.Channel channel: The channel object """ LOGGER.info('Channel opened') self._channel = channel self.add_on_channel_close_callback() self.setup_exchange(self.EXCHANGE) if self.AE_EXCHANGE: self.setup_ae_exchange(self.AE_EXCHANGE) if self.DL_EXCHANGE: self.setup_dl_exchange(self.DL_EXCHANGE) @exception_catch def setup_exchange(self, exchange_name): """Setup the exchange on RabbitMQ by invoking the Exchange.Declare RPC command. When it is complete, the on_exchange_declareok method will be invoked by pika. :param str|unicode exchange_name: The name of the exchange to declare """ cb = functools.partial(self.on_exchange_declareok, userdata=exchange_name) args = {} if self.AE_EXCHANGE: args['alternate-exchange'] = self.AE_EXCHANGE self._channel.exchange_declare(passive=self._passive, durable=self._durable, exchange=exchange_name, exchange_type=self.EXCHANGE_TYPE, arguments=args, callback=cb) @exception_catch def setup_ae_exchange(self, exchange_name): """Setup the exchange on RabbitMQ by invoking the Exchange.Declare RPC command. When it is complete, the on_exchange_declareok method will be invoked by pika. :param str|unicode exchange_name: The name of the exchange to declare """ ae_cb = functools.partial(self.on_ae_exchange_declareok, userdata=exchange_name) self._channel.exchange_declare(passive=self._passive, durable=False, exchange=exchange_name, exchange_type=self.AE_EXCHANGE_TYPE, arguments={}, callback=ae_cb) @exception_catch def setup_dl_exchange(self, exchange_name): """Setup the exchange on RabbitMQ by invoking the Exchange.Declare RPC command. When it is complete, the on_exchange_declareok method will be invoked by pika. :param str|unicode exchange_name: The name of the exchange to declare """ cb = functools.partial(self.on_dl_exchange_declareok, userdata=exchange_name) self._channel.exchange_declare(passive=self._passive, durable=False, exchange=exchange_name, exchange_type=self.DL_EXCHANGE_TYPE, arguments={}, callback=cb) def on_exchange_declareok(self, _unused_frame, userdata): """Invoked by pika when RabbitMQ has finished the Exchange.Declare RPC command. :param pika.Frame.Method unused_frame: Exchange.DeclareOk response frame :param str|unicode userdata: Extra user data (exchange name) """ LOGGER.info('Exchange declared: %s', userdata) self.setup_queue(self.QUEUE) def on_ae_exchange_declareok(self, _unused_frame, userdata): """Invoked by pika when RabbitMQ has finished the Exchange.Declare RPC command. :param pika.Frame.Method unused_frame: Exchange.DeclareOk response frame :param str|unicode userdata: Extra user data (exchange name) """ LOGGER.info('Exchange declared: %s', userdata) self.setup_ae_queue(self.AE_QUEUE) def on_dl_exchange_declareok(self, _unused_frame, userdata): """Invoked by pika when RabbitMQ has finished the Exchange.Declare RPC command. :param pika.Frame.Method unused_frame: Exchange.DeclareOk response frame :param str|unicode userdata: Extra user data (exchange name) """ LOGGER.info('Exchange declared: %s', userdata) self.setup_dl_queue(self.DL_QUEUE) @exception_catch def setup_queue(self, queue_name): """Setup the queue on RabbitMQ by invoking the Queue.Declare RPC command. When it is complete, the on_queue_declareok method will be invoked by pika. :param str|unicode queue_name: The name of the queue to declare. """ LOGGER.info('Declaring queue %s', queue_name) if self._type == 'consumer' and self.EXCHANGE_TYPE == 'x-modulus-hash': if not self._was_consuming: self.start_consuming() else: args = {} if self.DL_EXCHANGE: args['x-dead-letter-exchange'] = self.DL_EXCHANGE self._channel.queue_declare(durable=self._durable, passive=self._passive, queue=queue_name, arguments=args, callback=self.on_queue_declareok) @exception_catch def setup_ae_queue(self, queue_name): """Setup the queue on RabbitMQ by invoking the Queue.Declare RPC command. When it is complete, the on_queue_declareok method will be invoked by pika. :param str|unicode queue_name: The name of the queue to declare. """ LOGGER.info('Declaring queue %s', queue_name) self._channel.queue_declare(durable=False, passive=self._passive, queue=queue_name, callback=self.on_ae_queue_declareok) @exception_catch def setup_dl_queue(self, queue_name): """Setup the queue on RabbitMQ by invoking the Queue.Declare RPC command. When it is complete, the on_queue_declareok method will be invoked by pika. :param str|unicode queue_name: The name of the queue to declare. """ LOGGER.info('Declaring queue %s', queue_name) self._channel.queue_declare(durable=False, passive=self._passive, queue=queue_name, callback=self.on_dl_queue_declareok) @exception_catch def on_queue_declareok(self, _unused_frame): """Method invoked by pika when the Queue.Declare RPC call made in setup_queue has completed. In this method we will bind the queue and exchange together with the routing key by issuing the Queue.Bind RPC command. When this command is complete, the on_bindok method will be invoked by pika. :param pika.frame.Method method_frame: The Queue.DeclareOk frame """ LOGGER.info('Binding %s to %s with %s', self.EXCHANGE, self.QUEUE, self.ROUTING_KEY) self._channel.queue_bind(self.QUEUE, self.EXCHANGE, routing_key=self.ROUTING_KEY, callback=self.on_bindok) @exception_catch def on_ae_queue_declareok(self, _unused_frame): """Method invoked by pika when the Queue.Declare RPC call made in setup_queue has completed. In this method we will bind the queue and exchange together with the routing key by issuing the Queue.Bind RPC command. When this command is complete, the on_bindok method will be invoked by pika. :param pika.frame.Method method_frame: The Queue.DeclareOk frame """ LOGGER.info('Binding %s to %s with %s', self.AE_EXCHANGE, self.AE_QUEUE, self.ROUTING_KEY) self._channel.queue_bind(self.AE_QUEUE, self.AE_EXCHANGE, routing_key=self.ROUTING_KEY, callback=self.on_bindok) @exception_catch def on_dl_queue_declareok(self, _unused_frame): """Method invoked by pika when the Queue.Declare RPC call made in setup_queue has completed. In this method we will bind the queue and exchange together with the routing key by issuing the Queue.Bind RPC command. When this command is complete, the on_bindok method will be invoked by pika. :param pika.frame.Method method_frame: The Queue.DeclareOk frame """ LOGGER.info('Binding %s to %s with %s', self.DL_EXCHANGE, self.DL_QUEUE, self.ROUTING_KEY) self._channel.queue_bind(self.DL_QUEUE, self.DL_EXCHANGE, routing_key=self.ROUTING_KEY, callback=self.on_bindok) def on_bindok(self, unused_frame): """Invoked by pika when the Queue.Bind method has completed. At this point we will start consuming messages by calling start_consuming which will invoke the needed RPC commands to start the process. :param pika.frame.Method unused_frame: The Queue.BindOk response frame """ LOGGER.info('Queue bound') if self._type == 'consumer': if not self._was_consuming: self.start_consuming() else: if not self._was_publishing: self.start_publishing() @exception_catch def stop_consuming(self): """Tell RabbitMQ that you would like to stop consuming by sending the Basic.Cancel RPC command. """ if self._channel: LOGGER.info('Sending a Basic.Cancel RPC command to RabbitMQ') self._channel.basic_cancel(self.on_cancelok, self._consumer_tag) @exception_catch def start_consuming(self): """This method sets up the consumer by first calling add_on_cancel_callback so that the object is notified if RabbitMQ cancels the consumer. It then issues the Basic.Consume RPC command which returns the consumer tag that is used to uniquely identify the consumer with RabbitMQ. We keep the value to use it when we want to cancel consuming. The on_message method is passed in as a callback pika will invoke when a message is fully received. """ LOGGER.info('start consuming') self._was_consuming = True self.add_on_cancel_callback() self._channel.basic_qos(prefetch_count=self._prefetch_count) self._consumer_tag = self._channel.basic_consume( self.QUEUE, self._callback) def add_on_cancel_callback(self): """Add a callback that will be invoked if RabbitMQ cancels the consumer for some reason. If RabbitMQ does cancel the consumer, on_consumer_cancelled will be invoked by pika. """ LOGGER.info('Adding consumer cancellation callback') self._channel.add_on_cancel_callback(self.on_consumer_cancelled) def on_consumer_cancelled(self, method_frame): """Invoked by pika when RabbitMQ sends a Basic.Cancel for a consumer receiving messages. :param pika.frame.Method method_frame: The Basic.Cancel frame """ LOGGER.info('Consumer was cancelled remotely, shutting down: %r', method_frame) if self._channel: self._channel.close() def start_publishing(self): """This method will enable delivery confirmations and schedule the first message to be sent to RabbitMQ """ LOGGER.info('start publishing') self._was_publishing = True self.enable_delivery_confirmations() # self.schedule_next_message() def enable_delivery_confirmations(self): """Send the Confirm.Select RPC method to RabbitMQ to enable delivery confirmations on the channel. The only way to turn this off is to close the channel and create a new one. When the message is confirmed from RabbitMQ, the on_delivery_confirmation method will be invoked passing in a Basic.Ack or Basic.Nack method from RabbitMQ that will indicate which messages it is confirming or rejecting. """ LOGGER.info('Issuing Confirm.Select RPC command') self._channel.confirm_delivery(self._callback) def close_channel(self): """Call to close the channel with RabbitMQ cleanly by issuing the Channel.Close RPC command. """ LOGGER.info('Closing the channel') self._channel.close() @exception_catch def open_channel(self): """Open a new channel with RabbitMQ by issuing the Channel.Open RPC command. When RabbitMQ responds that the channel is open, the on_channel_open callback will be invoked by pika. """ LOGGER.info('Creating a new channel') self._connection.channel(on_open_callback=self.on_channel_open) def get_channel(self): """return _channel. """ return self._channel def get_connection(self): """return _connection. """ return self._connection def stop(self): """Cleanly shutdown the connection to RabbitMQ by stopping the consumer with RabbitMQ. When RabbitMQ confirms the cancellation, on_cancelok will be invoked by pika, which will then closing the channel and connection. The IOLoop is started again because this method is invoked when CTRL-C is pressed raising a KeyboardInterrupt exception. This exception stops the IOLoop which needs to be running for pika to communicate with RabbitMQ. All of the commands issued prior to starting the IOLoop will be buffered but not processed. """ self._closing = True LOGGER.info('Stopped') def _get_reconnect_delay(self): if self._was_consuming: self._reconnect_delay = 0 else: self._reconnect_delay += 1 if self._reconnect_delay > 30: self._reconnect_delay = 30 return self._reconnect_delay
class LoginPublish(object): def __init__(self): self.channel = None self.connection = None self.messages = [] def initialcon(self): _logger.debug("pikapublish begin initialcon") credentials = pika.PlainCredentials('mxpub', 'mhearts2015') parameters = pika.ConnectionParameters('localhost', 6500, credentials=credentials) self.connection = TornadoConnection(parameters, on_open_callback = self.on_connected) def on_connected(self, connection): _logger.debug("pikapublish begin on_connected") self.connection = connection self.connection.channel(self.on_channel_open) self.connection.add_on_close_callback(self.on_connection_closed) def on_connection_closed(self, connection, reply_code, reply_text): _logger.info("connection closed") self.connection.add_timeout(5, self.initialcon) def on_channel_open(self, channel): _logger.debug("pikapublish begin on_channel_open") self.channel = channel self.publishmsg() def releasecon(self): if self.connection: self.connection.close() def pushmsg(self, loginkey, body): _logger.debug("begin pushmsg %r %s" % (body, loginkey)) if not loginkey or not body: return self.messages.append({"key":loginkey, "body":body}) self.publishmsg() def publishmsg(self): _logger.debug("begin to publishmsg") if not self.channel: _logger.debug("begin to open channel") self.initialcon() return for item in self.messages: key = item.get("key", "") body = item.get("body", "") _logger.info("begin to publish %s body = %r" % (key, body)) if not isinstance(key, str) and not isinstance(key, bytes): _logger.error("invalid key") continue noti_body = json.dumps(body) try: self.channel.basic_publish(exchange = 'pclogin', routing_key = key, body = noti_body) except Exception as e: _logger.error("pikapublish catch exception {0}".format(e)) self.messages[:] = []
class PikaClient(object): def __init__(self, io_loop): self.logger = logging.getLogger(__name__) self.logger.info('PikaClient: __init__') self.io_loop = io_loop self.connected = False self.connecting = False self.connection = None self.channel = None self.event_listeners_count = 0 self.event_listeners = set() self.subscrib_channel = defaultdict(set) self.last_reconnect = now() self.uptime_start = now() self.config = SockJSSereverSettings() def connect(self): if self.connecting: self.logger.info('django-sockjs-server(PikaClient): Already connecting to RabbitMQ') return self.logger.info('django-sockjs-server(PikaClient): Connecting to RabbitMQ') self.connecting = True cred = pika.PlainCredentials(self.config.rabbitmq_user, self.config.rabbitmq_password) param = pika.ConnectionParameters( host=self.config.rabbitmq_host, port=self.config.rabbitmq_port, virtual_host=self.config.rabbitmq_vhost, credentials=cred ) try: self.connection = TornadoConnection(param, on_open_callback=self.on_connected) self.connection.add_on_close_callback(self.on_closed) except AMQPConnectionError: self.logger.info('django-sockjs-server(PikaClient): error connect, wait 5 sec') time.sleep(5) self.reconnect() self.last_reconnect = now() def on_connected(self, connection): self.logger.info('django-sockjs-server(PikaClient): connected to RabbitMQ') self.connected = True self.connection = connection self.connection.channel(self.on_channel_open) def on_channel_open(self, channel): self.logger.info('django-sockjs-server(PikaClient): Channel open, Declaring exchange') self.channel = channel self.channel.exchange_declare(exchange=self.config.rabbitmq_exhange_name, exchange_type=self.config.rabbitmq_exchange_type) self.channel.queue_declare(exclusive=False, auto_delete=True, callback=self.on_queue_declared) def on_queue_declared(self, frame): self.logger.info('django-sockjs-server(PikaClient): queue bind') self.channel.queue_bind(callback=None, exchange=self.config.rabbitmq_exhange_name, queue=frame.method.queue) self.channel.basic_consume(self.handle_delivery, queue=frame.method.queue, no_ack=True) def handle_delivery(self, channel, method, header, body): """Called when we receive a message from RabbitMQ""" self.notify_listeners(body) def on_closed(self, connection, error_code, error_message): self.logger.info('django-sockjs-server(PikaClient): rabbit connection closed, wait 5 seconds') connection.add_timeout(5, self.reconnect) def reconnect(self): self.connecting = False self.logger.info('django-sockjs-server(PikaClient): reconnect') self.connect() def notify_listeners(self, event_json): event_obj = json.loads(event_json) self.logger.debug('django-sockjs-server(PikaClient): get new data = %s ' % event_obj) try: if len(self.subscrib_channel[event_obj['channel']]) > 0: for client in self.subscrib_channel[event_obj['channel']]: self.logger.debug('django-sockjs-server(PikaClient): send message channel = %s ' % event_obj['channel']) client.broadcast(self.subscrib_channel[event_obj['channel']], event_json) break except KeyError: pass def add_event_listener(self, listener): self.event_listeners.add(listener) self.event_listeners_count += 1 self.logger.debug('django-sockjs-server(PikaClient): listener %s added' % repr(listener)) def remove_event_listener(self, listener): try: self.event_listeners.remove(listener) self.event_listeners_count -= 1 self.logger.debug('django-sockjs-server(PikaClient): listener %s removed' % repr(listener)) except KeyError: pass def add_subscriber_channel(self, chanel, client): self.subscrib_channel[chanel].add(client) self.logger.debug('django-sockjs-server(PikaClient): listener %s add to channel %s' % (repr(client), chanel)) def remove_subscriber_channel(self, chanel, client): try: self.subscrib_channel[chanel].remove(client) self.logger.debug('django-sockjs-server(PikaClient): listener %s remove from channel %s' % (repr(client), chanel)) except KeyError: pass def get_event_listeners_count(self): return self.event_listeners_count def get_subscribe_channel_count(self): return len(self.subscrib_channel.keys()) def get_subscribe_channels(self): return self.subscrib_channel.keys() def get_last_reconnect(self): return self.last_reconnect def get_uptime(self): return (now() - self.uptime_start).seconds
class PikaClient(object): def __init__(self, io_loop): pika.log.info("PikaClient: __init__") self.io_loop = io_loop self.connected = False self.connecting = False self.connection = None self.channel = None self.event_listeners = set([]) self.queue_name = "tornado-test-%i" % os.getpid() def connect(self): if self.connecting: pika.log.info("PikaClient: Already connecting to RabbitMQ") return pika.log.info("PikaClient: Connecting to RabbitMQ") self.connecting = True # cred = pika.PlainCredentials('guest', 'guest') param = pika.ConnectionParameters( host="115.146.94.68", # port=5672, # virtual_host='/', # credentials=cred ) self.connection = TornadoConnection(param, on_open_callback=self.on_connected) self.connection.add_on_close_callback(self.on_closed) def on_connected(self, connection): pika.log.info("PikaClient: connected to RabbitMQ") self.connected = True self.connection = connection self.connection.channel(self.on_channel_open) def on_channel_open(self, channel): pika.log.info("PikaClient: Channel open, Declaring exchange") self.channel = channel self.channel.exchange_declare(exchange="www", type="direct", callback=self.on_exchange_declared) return def on_exchange_declared(self, frame): pika.log.info("PikaClient: Exchange Declared, Declaring Queue") self.channel.queue_declare( queue=self.queue_name, auto_delete=True, durable=False, exclusive=False, callback=self.on_queue_declared ) return def on_queue_declared(self, frame): pika.log.info("PikaClient: Queue Declared, Binding Queue") self.channel.queue_bind( exchange="www", queue=self.queue_name, routing_key="client", callback=self.on_queue_bound ) def on_queue_bound(self, frame): pika.log.info("PikaClient: Queue Bound, Issuing Basic Consume") self.channel.basic_consume(consumer_callback=self.on_message, queue=self.queue_name, no_ack=True) def on_closed(self, connection): pika.log.info("PikaClient: rabbit connection closed") self.io_loop.stop() def on_message(self, channel, method, header, body): pika.log.info("PikaClient: message received: %s" % body) self.notify_listeners(body) def send_message(self, body): self.channel.basic_publish(exchange="www", routing_key="server", body=body) def notify_listeners(self, body): for listener in self.event_listeners: listener.write_message(body) pika.log.info("PikaClient: notified %s" % repr(listener)) def add_event_listener(self, listener): self.event_listeners.add(listener) pika.log.info("PikaClient: listener %s added" % repr(listener)) def remove_event_listener(self, listener): try: self.event_listeners.remove(listener) pika.log.info("PikaClient: listener %s removed" % repr(listener)) except KeyError: pass
class PikaClient(object): def __init__(self, config: dict, exchanges: dict) -> None: self.connecting = False self.connection = None self.channel = None self.config = config self.exchanges = exchanges def connect(self): if self.connecting: logging.info('connecting - so not re-connecting') return self.connecting = True host = self.config.get('host') port = 5671 if self.config.get('amqps') else 5672 scheme = 'amqps' if self.config.get('amqps') else 'amqp' virtual_host = urllib.parse.quote(self.config.get('vhost'), safe='') user = self.config.get('user') pw = self.config.get('pw') heartbeat = self.config.get('heartbeat') if self.config.get('heartbeat') else 0 params = pika.URLParameters( f"{scheme}://{user}:{pw}@{host}:{port}/{virtual_host}?heartbeat={heartbeat}" ) self.connection = TornadoConnection(params) self.connection.add_on_open_callback(self.on_connect) self.connection.add_on_close_callback(self.on_closed) self.connection.add_on_open_error_callback(self.on_open_error_callback) self.connecting = False return def on_open_error_callback(self, connection: TornadoConnection, exception: Exception) -> None: logging.error('could not connect') def on_connect(self, connection: TornadoConnection) -> None: self.connection = connection self.channel = self.connection.channel( on_open_callback=self.on_channel_open ) return def on_channel_open(self, channel: pika.channel.Channel) -> None: for backend, config in self.exchanges.items(): ex_name = config.get('exchange') channel.exchange_declare( ex_name, exchange_type='topic', durable=True, ) logging.info(f'rabbitmq exchange: {ex_name} declared') return def on_basic_cancel(self, frame: pika.frame.Frame) -> None: self.connection.close() def on_closed(self, connection: TornadoConnection, exception: Exception) -> None: logging.info('rabbitmq connection closed') logging.info(exception) def publish_message( self, *, exchange: str, routing_key: str, method: str, uri: str, version: str, data: dict, persistent: bool = True, timestamp: int = int(time.time()), ) -> None: """ Publilsh a message to an exchange. Parameters ---------- exchange: str, exchange name routing_key: str, routing key for topic exchange method: str, HTTP method uri: str, HTTP request URI version: str, e.g. v1 data: dict persistent: bool, default True tell rabbitmq to persist messages to disk, or not """ data = { 'method': method, 'uri': uri, 'version': version, 'data': data, } message = json.dumps(data) delivery_mode = 2 if persistent else 1 self.channel.basic_publish( exchange=exchange, routing_key=routing_key, body=message, properties=pika.BasicProperties( content_type='application/json', delivery_mode=delivery_mode, timestamp=timestamp, message_id=str(uuid.uuid4()), ) ) return
class PikaClient(object): def __init__(self): self.ioloop = tornado.ioloop.IOLoop.instance() self.connection = None self.channel = None self._delivery_tag = 0 self.parameters = pika.ConnectionParameters(rabbitmq_server) def connect(self): self.connection = TornadoConnection( self.parameters, on_open_callback=self.on_connected, stop_ioloop_on_close=False, on_open_error_callback=self.on_open_error) self.connection.add_on_close_callback(self.on_closed) def on_open_error(self, unused_connection, err): sys.exit(1) def on_connected(self, connection): logging.info('PikaClient: connected to RabbitMQ') self.connection.channel(self.on_exchange_declare) def on_exchange_declare(self, channel): logging.info('PikaClient: Channel %s open, Declaring exchange' % channel) self.channel = channel self.channel.exchange_declare(self.on_queue_declare, exchange='notification', exchange_type='direct') def on_queue_declare(self, method_frame): logging.info('PikaClient: Channel open, Declaring queue') self.channel.queue_declare(self.on_queue_bind, queue='notification') #, durable=True) def on_queue_bind(self, method_frame): logging.info('Queue bound') self.channel.queue_bind(self.on_consume_bind, queue="notification", exchange="notification", routing_key="notification") def on_consume_bind(self, frame): logging.info("Consume bind") self.channel.basic_qos(prefetch_count=1) self.channel.basic_consume(self.on_response, queue='notification', no_ack=False) def on_response(self, channel, method, properties, body): logging.info('on_response') message = pickle.loads(body) logging.info(message) ChatSocketHandler.on_caculate_success(message['id'], message['index']) channel.basic_ack(delivery_tag=method.delivery_tag) def on_closed(self, connection): logging.info('PikaClient: rabbit connection closed') self.connection.close() self.channel.close() self.ioloop.stop()
class RabbitMQConnection(object): __io_loop = None __connected = None __connecting = None __connection = None __channel = None __callback_queue = None def __init__( self, io_loop, ): logger.debug('PikaClient: __init__') self.__io_loop = io_loop self.__connected = False self.__connecting = False self.__connection = None self.__channel = None self.connect = self.__connect self.call = self.__call def __connect(self): if self.__connecting: logger.debug('PikaClient: Already connecting to RabbitMQ') return logger.debug('PikaClient: Connecting to RabbitMQ') self.__connecting = True cred = pika.PlainCredentials(Config.RabbitMQ.username, Config.RabbitMQ.password) param = pika.ConnectionParameters( host=Config.RabbitMQ.host, port=Config.RabbitMQ.port, virtual_host=Config.RabbitMQ.virtual_host, credentials=cred) self.__connection = TornadoConnection( param, on_open_callback=self.__on_connected, stop_ioloop_on_close=False) self.__connection.add_on_close_callback(self.__on_close) def __on_connected(self, connection): logger.debug('PikaClient: connected to RabbitMQ') self.__connected = True self.__connection = connection self.__connection.channel(self.__on_channel_open) def __call(self, message, correlation_id): self.__channel.basic_publish(exchange='', routing_key='rpc_queue', properties=pika.BasicProperties( reply_to=self.__callback_queue, correlation_id=correlation_id, ), body=str(message)) def __on_response(self, ch, method, props, body): print("I got", props.correlation_id) ClientWebSocket.on_response(message=body, correlation_id=props.correlation_id) def __on_close(self, connection): logger.debug('PikaClient: rabbit connection closed') self.__io_loop.stop() def __on_queue_open(self, a): self.__callback_queue = a.method.queue self.__channel.basic_consume(self.__on_response, no_ack=True, queue=self.__callback_queue) def __on_channel_open(self, channel): # print("channel: ", channel) logger.debug('PikaClient: Channel %s open, Declaring exchange' % channel) self.__channel = channel self.__channel.queue_declare(callback=self.__on_queue_open, exclusive=True)
class PikaClient(object): def __init__(self): # Construct a queue name we'll use for this instance only self.queue_name = 'tornado-test-%i' % os.getpid() # Default values self.connected = False self.connecting = False self.connection = None self.channel = None # A place for us to keep messages sent to us by Rabbitmq self.messages = list() # A place for us to put pending messages while we're waiting to connect self.pending = list() def connect(self): if self.connecting: pika.log.info('PikaClient: Already connecting to RabbitMQ') return pika.log.info('PikaClient: Connecting to RabbitMQ on localhost:5672') self.connecting = True credentials = pika.PlainCredentials('guest', 'guest') param = pika.ConnectionParameters(host='localhost', port=5672, virtual_host="/", credentials=credentials) self.connection = TornadoConnection(param, on_open_callback=self.on_connected) self.connection.add_on_close_callback(self.on_closed) def on_connected(self, connection): pika.log.info('PikaClient: Connected to RabbitMQ on localhost:5672') self.connected = True self.connection = connection self.connection.channel(self.on_channel_open) def on_channel_open(self, channel): pika.log.info('PikaClient: Channel Open, Declaring Exchange') self.channel = channel self.channel.exchange_declare(exchange='tornado', type="direct", auto_delete=True, durable=False, callback=self.on_exchange_declared) def on_exchange_declared(self, frame): pika.log.info('PikaClient: Exchange Declared, Declaring Queue') self.channel.queue_declare(queue=self.queue_name, auto_delete=True, durable=False, exclusive=False, callback=self.on_queue_declared) def on_queue_declared(self, frame): pika.log.info('PikaClient: Queue Declared, Binding Queue') self.channel.queue_bind(exchange='tornado', queue=self.queue_name, routing_key='tornado.*', callback=self.on_queue_bound) def on_queue_bound(self, frame): pika.log.info('PikaClient: Queue Bound, Issuing Basic Consume') self.channel.basic_consume(consumer_callback=self.on_pika_message, queue=self.queue_name, no_ack=True) # Send any messages pending for properties, body in self.pending: self.channel.basic_publish(exchange='tornado', routing_key='tornado.*', body=body, properties=properties) def on_pika_message(self, channel, method, header, body): pika.log.info('PikaCient: Message receive, delivery tag #%i' % \ method.delivery_tag) # Append it to our messages list self.messages.append(body) def on_basic_cancel(self, frame): pika.log.info('PikaClient: Basic Cancel Ok') # If we don't have any more consumer processes running close self.connection.close() def on_closed(self, connection): # We've closed our pika connection so stop the demo tornado.ioloop.IOLoop.instance().stop() def sample_message(self, tornado_request): # Build a message to publish to RabbitMQ body = '%.8f: Request from %s [%s]' % \ (tornado_request._start_time, tornado_request.remote_ip, tornado_request.headers.get("User-Agent")) # Send the message properties = pika.BasicProperties(content_type="text/plain", delivery_mode=1) self.channel.basic_publish(exchange='tornado', routing_key='tornado.*', body=body, properties=properties) def get_messages(self): # Get the messages to return, then empty the list output = self.messages self.messages = list() return output
class PikaClient(object): def __init__(self, config, app=None): # Connection params self.host = config['host'] or 'localhost' self.port = config['port'] or '5672' self.vhost = config['vhost'] or '/' self.user = config['user'] or 'guest' self.passwd = config['passwd'] or 'guest' self.exchange = config['exchange'] or 'myx' self.queue_name = config['queue_name'] or 'tornado-test-%i' \ % os.getpid() self.routing_key = config['routing_key'] or 'tornado.*' # Default values self.connected = False self.connecting = False self.connection = None self.channel = None self.app = app # Set our pika.log options pika.log.setup(color=True) # A place for us to keep messages sent to us by Rabbitmq self.messages = list() # A place for us to put pending messages while we're waiting to connect self.pending = list() self.connect() def connect(self): if self.connecting: pika.log.info('PikaClient: Already connecting to RabbitMQ') return pika.log.info('PikaClient: Connecting to RabbitMQ on %s:%i' \ % (self.host, self.port)) self.connecting = True credentials = pika.PlainCredentials(self.user, self.passwd) param = pika.ConnectionParameters(host=self.host, port=self.port, virtual_host=self.vhost, credentials=credentials) srs = SimpleReconnectionStrategy() logging.debug('Events: Connecting to AMQP Broker: %s:%i' % (self.host, self.port)) # from pika.adapters import SelectConnection # connection = SelectConnection(parameters, on_connected) self.connection = TornadoConnection(param, reconnection_strategy=srs, on_open_callback=self.on_connected) self.connection.add_on_close_callback(self.on_closed) def on_connected(self, connection): pika.log.info('PikaClient: Connected to RabbitMQ on %s:%i' \ % (self.host, self.port)) self.connected = True self.connection = connection self.connection.channel(self.on_channel_open) def on_channel_open(self, channel): pika.log.info('PikaClient: Channel Open, Declaring Exchange %s' \ % self.exchange) self.channel = channel self.channel.exchange_declare(exchange=self.exchange, type="direct", auto_delete=True, durable=False, callback=self.on_exchange_declared) def on_exchange_declared(self, frame): pika.log.info('PikaClient: Exchange Declared, Declaring Queue %s' \ % self.queue_name) self.channel.queue_declare(queue=self.queue_name, auto_delete=True, durable=False, exclusive=False, callback=self.on_queue_declared) def on_queue_declared(self, frame): pika.log.info('PikaClient: Queue Declared, Binding Queue') self.channel.queue_bind(exchange=self.exchange, queue=self.queue_name, routing_key=self.routing_key, callback=self.on_queue_bound) def on_queue_bound(self, frame): pika.log.info('PikaClient: Queue Bound, Issuing Basic Consume') self.channel.basic_consume(consumer_callback=self.on_message, queue=self.queue_name, no_ack=True) # Send any messages pending for properties, body in self.pending: self.channel.basic_publish(exchange=self.exchange, routing_key=self.routing_key, body=body, properties=properties) def on_basic_cancel(self, frame): pika.log.info('PikaClient: Basic Cancel Ok') # If we don't have any more consumer processes running close self.connection.close() def on_closed(self, connection): # We've closed our pika connection so stop the demo tornado.ioloop.IOLoop.instance().stop() def on_message(self, channel, method, header, body): pika.log.info('PikaCient: Message received: %s delivery tag #%i: %s' \ % (header.content_type, method.delivery_tag, body)) # Append it to our messages list self.messages.append(body) self.app.dispatcher.notifyCallbacks(body) def get_messages(self): # Get the messages to return, then empty the list output = self.messages self.messages = list() return output def publish(self, msg): # Build a message to publish to RabbitMQ #body = '%.8f: Request from %s [%s]' % \ # (tornado_request._start_time, # tornado_request.remote_ip, # tornado_request.headers.get("User-Agent")) # Send the message properties = pika.BasicProperties(content_type="text/plain", delivery_mode=1) self.channel.basic_publish(exchange=self.exchange, routing_key=self.routing_key, #body='Message: %s - %s' % (msg, body), body='Message: %s' % msg, properties=properties)
class PikaClient(object): def __init__(self, io_loop): pika.log.info('PikaClient: __init__') self.io_loop = io_loop self.connected = False self.connecting = False self.connection = None self.channel = None self.event_listeners = set([]) def connect(self): if self.connecting: pika.log.info('PikaClient: Already connecting to RabbitMQ') return pika.log.info('PikaClient: Connecting to RabbitMQ') self.connecting = True cred = pika.PlainCredentials('guest', 'guest') param = pika.ConnectionParameters( host='localhost', port=5672, virtual_host='/', credentials=cred ) self.connection = TornadoConnection(param, on_open_callback=self.on_connected) self.connection.add_on_close_callback(self.on_closed) def on_connected(self, connection): pika.log.info('PikaClient: connected to RabbitMQ') self.connected = True self.connection = connection self.connection.channel(self.on_channel_open) def on_channel_open(self, channel): pika.log.info('PikaClient: Channel open, Declaring exchange') self.channel = channel # declare exchanges, which in turn, declare # queues, and bind exchange to queues def on_closed(self, connection): pika.log.info('PikaClient: rabbit connection closed') self.io_loop.stop() def on_message(self, channel, method, header, body): pika.log.info('PikaClient: message received: %s' % body) self.notify_listeners(event_factory(body)) def notify_listeners(self, event_obj): # here we assume the message the sourcing app # post to the message queue is in JSON format event_json = json.dumps(event_obj) for listener in self.event_listeners: listener.write_message(event_json) pika.log.info('PikaClient: notified %s' % repr(listener)) def add_event_listener(self, listener): self.event_listeners.add(listener) pika.log.info('PikaClient: listener %s added' % repr(listener)) def remove_event_listener(self, listener): try: self.event_listeners.remove(listener) pika.log.info('PikaClient: listener %s removed' % repr(listener)) except KeyError: pass
class PikaClient(object): def __init__(self, io_loop): pika.log.info('PikaClient: __init__') self.io_loop = io_loop self.connected = False self.connecting = False self.connection = None self.channel = None self.event_listeners = set([]) def connect(self): if self.connecting: pika.log.info('PikaClient: Already connecting to RabbitMQ') return pika.log.info('PikaClient: Connecting to RabbitMQ') self.connecting = True #cred = pika.PlainCredentials('guest', 'guest') param = pika.ConnectionParameters( host='115.146.93.175', #port=5672, #virtual_host='/', #credentials=cred ) self.connection = TornadoConnection(param, on_open_callback=self.on_connected) self.connection.add_on_close_callback(self.on_closed) def on_connected(self, connection): pika.log.info('PikaClient: connected to RabbitMQ') self.connected = True self.connection = connection self.connection.channel(self.on_channel_open) def on_channel_open(self, channel): pika.log.info('PikaClient: Channel open, Declaring exchange') self.channel = channel self.channel.queue_declare(queue='hello') self.channel.basic_consume(self.on_message, queue='hello', no_ack=True) # declare exchanges, which in turn, declare # queues, and bind exchange to queues def on_closed(self, connection): pika.log.info('PikaClient: rabbit connection closed') self.io_loop.stop() def on_message(self, channel, method, header, body): pika.log.info('PikaClient: message received: %s' % body) self.notify_listeners(body) def send_message(self, body): self.channel.basic_publish(exchange='', routing_key='hello', body=body) def notify_listeners(self, body): for listener in self.event_listeners: listener.write_message(body) pika.log.info('PikaClient: notified %s' % repr(listener)) def add_event_listener(self, listener): print "added listener" self.event_listeners.add(listener) pika.log.info('PikaClient: listener %s added' % repr(listener)) def remove_event_listener(self, listener): try: self.event_listeners.remove(listener) pika.log.info('PikaClient: listener %s removed' % repr(listener)) except KeyError: pass
class QueueManager(object): def __init__(self): self.connection = None self.channel = None self._connected = False self.event_listeners = {} self.connections = {} self.cache = None self.run() def run(self): t1 = Thread(target=sd_redis.watch_redis) t1.start() while not getattr(sd_redis, 'redis_master'): time.sleep(0.01) self.cache = ZRedis(host=sd_redis.redis_master, password=redis_db_pw, db=os.getenv('REDIS_DB')) self.connect() def connect(self, host=None): """ Connection with Rabbit. """ if self._connected: zlogger.info('PikaClient: Already connecting to RabbitMQ') return if not host: _, host = list(json.loads(sd_rabbit.rabbit_nodes).items())[0] zlogger.info('PikaClient: Connecting to RabbitMQ in Queue Manager') param = ConnectionParameters(host=host, port=5672, virtual_host=VIRTUAL_HOST, credentials=sd_rabbit.rabbit_credential) self.connection = TornadoConnection(param, on_open_callback=self.on_connected) self.connection.add_on_close_callback(self.on_closed) self._connected = True zlogger.info("Connection is successful: host:{}".format(host)) def on_connected(self, connection): """ AMQP connection callback. Creates input channel. Args: connection: AMQP connection """ zlogger.info('PikaClient: connected to RabbitMQ') self.connection.channel(self.on_channel_open) def on_channel_open(self, channel): """ Input channel creation callback, exchange declaring. """ zlogger.info('PikaClient: Channel open, Declaring exchange') self.channel = channel self.channel.exchange_declare(exchange='messages', type='topic', durable=True) def on_message(self, channel, method, header, body): """ When message is received, prepare notifier list. """ user_id = method.consumer_tag notify_list = self.event_listeners[user_id] self.notify_listeners(body, notify_list) def listen_messages(self, queue_name, user_id): """ Listen rabbit messages. """ self.channel.basic_consume(consumer_callback=self.on_message, queue=queue_name, consumer_tag=user_id, no_ack=True) def notify_listeners(self, message, notify_list): """ Write message to notifier list. """ for listener in notify_list: listener.write_message(message) def add_event_listener(self, listener, user_info): """ Add listener to user set. If queue creation is new, """ user_id = user_info.get("user") if not self.event_listeners.get(user_id): queue_name = self.get_queue_name(user_id) self.channel.queue_declare(queue=queue_name, auto_delete=True, callback=None) self.event_listeners.setdefault(user_id, []).append(listener) self.input_queue_bind(queue_name, user_info) self.listen_messages(queue_name, user_id) self.cache.sadd('QueueList:{}'.format(user_id), queue_name) else: self.event_listeners[user_id].append(listener) zlogger.info( "New websocket connection is added for user which has user_id:{}". format(user_id)) def remove_event_listener(self, listener, user_id): """ Remove listener from listener list. """ try: if self.event_listeners.get(user_id): self.event_listeners[user_id].remove(listener) if not self.event_listeners[user_id]: self.channel.queue_delete( queue=self.get_queue_name(user_id)) del self.event_listeners[user_id] except Exception as exc: zlogger.error( "An error occurred on remove_event_listener method inside QueueManager. " "User Id: {}, Exception: {}".format(user_id, exc)) def input_queue_bind(self, queue, user_info): """ Input queue declaration callback. Input Queue/Exchange binding done here Args: queue: input queue user_info: user information dict include project, service and user ids """ bind_list = self.get_bind_list(user_info) for route_key in bind_list: self.channel.queue_bind(callback=None, exchange='messages', queue=queue, routing_key=route_key) def get_bind_list(self, user_info): """ Args: user_info: user information dict include project, service and user ids """ user_id = user_info.get("user") project_id = user_info.get("project") service = user_info.get("service") bind_list = [user_id] # 'CACHE_SUBSCRIBER_CHANNELS' key is Channels list of a user subs_channels_key = CACHE_SUBSCRIBER_CHANNELS.format( project_id=project_id, service=service, subscriber_id=user_id) channels = self.cache.smembers(subs_channels_key) channels = [channel_id.decode() for channel_id in channels] bind_list.extend(channels) return bind_list def on_closed(self, connection, _, __): """ Moves listeners from close node's queue manager to queue's new master node. """ self._connected = False @staticmethod def get_queue_name(user_id): """ Gets queue name according to user id. """ return "{}_{}".format(user_id, os.getpid())
class CaculateWorker(object): REDIS_HASHMAP_KEY = os.environ.get("REDIS_HASHMAP_KEY") def __init__(self): rabbitmq_server = os.environ.get("RABBITMQ_SERVER") redis_server = os.environ.get("REDIS_SERVER") self.redis_client = redis.Redis(host=redis_server, port=6379, db=0) self.ioloop = tornado.ioloop.IOLoop.instance() self.connection = None self.channel = None self._delivery_tag = 0 self.parameters = pika.ConnectionParameters(rabbitmq_server) @staticmethod def fibonacci(index): if index <= 1: return 1 return CaculateWorker.fibonacci(index - 1) + CaculateWorker.fibonacci(index - 2) def cache_cacalate(self, index, value): self.redis_client.hset(CaculateWorker.REDIS_HASHMAP_KEY, index, value) def connect(self): try: self.connection = TornadoConnection( self.parameters, on_open_callback=self.on_connected, stop_ioloop_on_close=False, on_open_error_callback=self.on_open_error) self.connection.add_on_close_callback(self.on_closed) except: logging.info("connect faield") def on_open_error(self, unused_connection, err): sys.exit(1) def on_connected(self, connection): logging.info('PikaClient: connected to RabbitMQ') self.connection.channel(self.on_exchange_declare) def on_exchange_declare(self, channel): logging.info('PikaClient: Channel %s open, Declaring exchange' % channel) self.channel = channel self.channel.exchange_declare(self.on_queue_declare, exchange='calc_fibonacci', exchange_type='direct') self.channel.exchange_declare(self.on_queue_declare, exchange='notification', exchange_type='direct') def on_queue_declare(self, method_frame): logging.info('PikaClient: Channel open, Declaring queue') self.channel.queue_declare(self.on_queue_bind, queue='calc_fibonacci') #, durable=True) self.channel.queue_declare(self.on_queue_bind, queue='notification') #, durable=True) def on_queue_bind(self, method_frame): logging.info('Queue bound') self.channel.queue_bind(self.on_consume_bind, queue="calc_fibonacci", exchange="calc_fibonacci", routing_key="calc_fibonacci") self.channel.queue_bind(self.on_consume_bind, queue="notification", exchange="notification", routing_key="notification") def on_consume_bind(self, frame): logging.info("Consume bind") self.channel.basic_qos(prefetch_count=1) self.channel.basic_consume(self.on_response, queue='calc_fibonacci', no_ack=False) def on_response(self, channel, method, properties, body): logging.info('on_response') message = pickle.loads(body) logging.info(message) uuid, index = message['uuid'], message['index'] value = CaculateWorker.fibonacci(index) self.cache_cacalate(index, value) # self.channel.exchange_declare(exchange="notification", exchange_type="direct") # self.channel.queue_declare(queue='notification') # self.channel.queue_bind(exchange='notification', queue=notification_queue, routing_key='notification') result = { 'id': uuid, 'index': index, 'value': value, } logging.info(result) channel.basic_publish(exchange='notification', routing_key='notification', body=pickle.dumps(result)) logging.info("publish done") channel.basic_ack(delivery_tag=method.delivery_tag) def on_closed(self, connection): logging.info('PikaClient: rabbit connection closed') self.connection.close() self.channel.close() self.ioloop.stop()
class PikaClient(object): def __init__(self): self.connected = False self.connecting = False self.connection = None self.channel = None self.messages = list() self.pending = list() def connect(self): if self.connecting: logging.error("PikaClient already connected") return logging.info("connecting to RabbitMQ") self.connecting = True param = pika.ConnectionParameters(host='10.212.66.144', port=5672) self.connection = TornadoConnection(param, on_open_callback=self.on_connected) self.connection.add_on_close_callback(self.on_closed) def on_connected(self, connection): pika.log.info( 'PikaClient: Connected to RabbitMQ on 10.212.66.144:5672') self.connected = True self.connection = connection self.connection.channel(self.on_channel_open) def on_channel_open(self, channel): pika.log.info('PikaClient: Channel Open, Declaring Exchange') self.channel = channel self.channel.exchange_declare(exchange='market', type="fanout", callback=self.on_exchange_declared) def on_exchange_declared(self, frame): logging.info("Exchange declared. Declaring Queue") self.channel.queue_bind(exchange='market', queue='market', callback=self.on_queue_bound) def on_queue_bound(self, frame): logging.info('Queue Bound, Issuing Basic Consume') self.channel.basic_consume(consumer_callback=self.on_pika_message, queue='market', no_ack=True) def on_pika_message(self, channel, method, header, body): logging.info('Message receive, %s' % body) # Append it to our messages list # self.messages.append(body) symbol, uuid = body.split("|") trend = db.get("""SELECT * FROM trends.trends WHERE uuid = %s""", uuid) for listener in LISTENERS: date = datetime.fromtimestamp(int(trend['created'])) date = date - timedelta(hours=5) date = date.strftime("%Y-%m-%d %I:%M:%S") html = uuid + "|<p id='%s'><img src='/static/images/remove-button.gif' width='10' height='10' style='margin-left:1px; margin-right:3px' class='remove_trend'/><span class='label labelcolor1'>" + date + " " + symbol + " </span><span class='trend'>%s trend discovered</span><a href='/full/?uuid=%s' target='_blank'><img src='/static/images/enlarge-button.gif' width='10' height='10' style='margin-left:1px'></a></p>" % ( uuid, trend['type'], uuid) listener.write_message(html) def on_closed(self, connection): tornado.ioloop.IOLoop.instance().stop() def send_message(self, body): logging.info("Sending message") self.channel.basic_publish(exchange='market', routing_key='', body=body) def get_messages(self): # Get the messages to return, then empty the list output = self.messages self.messages = list() return output
class RabbitClient(object): EXCHANGE = 'message' EXCHANGE_TYPE = 'topic' PUBLISH_INTERVAL = .001 QUEUE = 'text' ROUTING_KEY = 'example.text' def __init__(self, app, ioloop): """Setup the example publisher object, passing in the URL we will use to connect to RabbitMQ. :param str amqp_url: The URL for connecting to RabbitMQ """ self._connection = None self._channel = None self._deliveries = [] self._acked = 0 self._nacked = 0 self._message_number = 0 self._stopping = False self.app = app self.ioloop = ioloop ## Connection Logic ########################################################## def connect(self): LOGGER.info('Connecting to RabbitMQ') # exc_type, exc_value, exc_traceback = sys.exc_info() # traceback.print_tb(exc_traceback, limit=None, file=sys.stdout) # self.connecting = True self._connection = TornadoConnection(rc.connParam, on_open_callback=self.on_connection_open) def close_connection(self): """This method closes the connection to RabbitMQ.""" LOGGER.info('Closing connection') self._connection.close() def add_on_connection_close_callback(self): LOGGER.info('Adding connection close callback') self._connection.add_on_close_callback(self.on_connection_closed) def on_connection_closed(self, method_frame): # if we loose the connection, try to reconnect LOGGER.warning('Server closed connection, reopening: (%s) %s', method_frame.method.reply_code, method_frame.method.reply_text) self._channel = None self._connection = self.connect() def on_connection_open(self, unused_connection): LOGGER.info('Connection opened') self.add_on_connection_close_callback() self.open_channel() def add_on_channel_close_callback(self): LOGGER.info('Adding channel close callback') self._channel.add_on_close_callback(self.on_channel_closed) def on_channel_closed(self, method_frame): # if rabbit closes the channel, quit server LOGGER.warning('Channel was closed: (%s) %s', method_frame.method.reply_code, method_frame.method.reply_text) self._connection.close() def on_channel_open(self, channel): LOGGER.info('Channel opened') self._channel = channel self.add_on_channel_close_callback() self.setupExngsQueues() def sendNodeCtrlMsg(self, nodeHostname, ctrlCode): self._channel.basic_publish( rc.msgExngAttr["exchange"], "node.{0}.cmd".format(nodeHostname), str(ctrlCode) ) ## Message Route Init ######################################################## def setupExngsQueues(self): LOGGER.info('') self.exngQueCount = 0 self.exngQueNum = len(rc.svrExngs) + len(rc.svrQueues) # open all exchanges and queues we need asynchronously for exng in rc.svrExngs: self.setupExchange(exng) for queue in rc.svrQueues: self.setupQueue(queue) # callback fn counts when everything is declared def setupExchange(self, exng): LOGGER.info('Declaring exchange %s', exng["exchange"]) self._channel.exchange_declare(self.onExngQueDeclare, **exng) def setupQueue(self, queue): LOGGER.info('Declaring queue %s', queue["queue"]) self._channel.queue_declare(self.onExngQueDeclare, **queue) def onExngQueDeclare(self, mystery=None): # got unknown return # check to see how many rabbit entities are declared self.exngQueCount = self.exngQueCount + 1 LOGGER.info('Declared %d exchanges/queues.', self.exngQueCount) if(self.exngQueCount == self.exngQueNum): # all our exchanges and queues are accounted for; setup bindings self.setupBindings() def setupBindings(self): self.bindingsCount = 0 self.bindingsNum = len(rc.svrBindings) for binding in rc.svrBindings: LOGGER.info('Binding exchange %s to queue %s', binding["exchange"], binding["queue"]) self._channel.queue_bind(self.onBind, binding["queue"], binding["exchange"], binding["routing_key"]) def onBind(self, mystery=None): # check to see if we made all our bindings self.bindingsCount = self.bindingsCount + 1 LOGGER.info('Made %d bindings.', self.bindingsCount) if(self.bindingsCount == self.bindingsNum): # all of our bindings are accounted for; build message logic self.setupMsgRouting() def setupMsgRouting(self): # map the message queues we defined to our handlers LOGGER.info('Mapping handlers...') self._channel.basic_consume(self.valueHandler, rc.valueQueAttr["queue"], no_ack=True) self._channel.basic_consume(self.initNodeHandler, rc.handshakeQueAttr["queue"]) self._channel.basic_consume(self.serverCmdHandler, rc.cmdQueAttr["queue"]) self._channel.basic_consume(self.nErrHandler, rc.nErrQueAttr["queue"]) LOGGER.info('Server ready!') ## Handlers ################################################################## # [I 130116 01:17:40 wifiWattSrv:203] # Bad message (TypeError('not all arguments converted during string formatting',)): # {'threadName': 'MainThread', 'name': '__main__', 'thread': 139807606114048, 'created': 1358317060.982559, 'process': 10225, 'processName': 'MainProcess', # 'args': (<pika.channel.Channel object at 0x1b77590>, # <Basic.Deliver(['consumer_tag=ctag1.0', 'redelivered=True', 'routing_key=', 'delivery_tag=1', 'exchange=ww.valueUpdate'])>, # <BasicProperties(['delivery_mode=1', "headers={'hostname': 'applepi'}"])>, '2.9493'), 'module': 'wifiWattSrv', 'filename': 'wifiWattSrv.py', 'levelno': 20, 'exc_text': None, 'pathname': 'wifiWattSrv.py', 'lineno': 203, 'msg': # 'Got new channel: %s, basicDeliver: %s, prop: %s, body: ', 'exc_info': None, 'funcName': 'valueHandler', 'relativeCreated': 252.96688079833984, 'levelname': 'INFO', 'msecs': 982.5589656829834} def valueHandler(self, channel, basicDeliver, prop, body): """ :param pika.channel.Channel unused_channel: The channel object :param pika.Spec.Basic.Deliver: basic_deliver method :param pika.Spec.BasicProperties: properties :param str|unicode body: The message body """ # attempt to parse message and check fields try: msgData = json.loads(body) except: LOGGER.error("Couldn't parse message. Malformed.") return -1 if(("hostname" not in msgData) or ("current" not in msgData) or ("relayState" not in msgData)): LOGGER.error("Couldn't parse message. Mising fields.") return -1 hostname = msgData["hostname"] value = float(msgData["current"]) relayState = bool(int(msgData["relayState"])) # LOGGER.info("Raw: %s Parsed: %d", msgData["relayState"], int(relayState)) # if we've inited this node, call it's append method if hostname in self.app.nodes: nodeObj = self.app.nodes[hostname] newDP = wifiWattNode.wwDataPoint(value, time.time()) nodeObj.appendData(newDP, relayState) else: LOGGER.error('No node with hostname <%s>!', hostname) # attempt to read message header info # if "hostname" in prop.headers: # hostname = prop.headers["hostname"] # else: # LOGGER.error('Couldn\'t read hostname from message!') # return -1 # # parse the message # value = float(body) # LOGGER.info('Got new value "%f" from hostname <%s>.', value, hostname) def initNodeHandler(self, channel, basicDeliver, prop, body): """ Initialize the data stores for a node on the server. Works as a callback for a new message to the server handshake queue. """ # parse the hostname from handshake message hostname = body nodes = self.app.nodes # check if we already have structures for this node if(hostname in nodes): # we have it already LOGGER.info('Got handshake for node <%s>; already exists.', hostname) else: # make a new node instance LOGGER.info('Initing new node for hostname: %s', hostname) newNode = wifiWattNode.wifiWattNode(hostname) nodes[hostname] = newNode # send back the prefered message rate to start value stream self._channel.basic_publish( rc.msgExngAttr["exchange"], "node.{0}.handshake".format(hostname), str(prefMsgRate) ) # notify all our webclients that we have a new node for webClient in self.app.webClients: webClient.newNodeCb(newNode) def serverCmdHandler(self, channel, basicDeliver, prop, body): LOGGER.info('Got new command: %s', msg) def nErrHandler(self, channel, basicDeliver, prop, body): LOGGER.info('Got node error: %s', msg) def close_channel(self): """Invoke this command to close the channel with RabbitMQ by sending the Channel.Close RPC command. """ LOGGER.info('Closing the channel') self._channel.close() def open_channel(self): """This method will open a new channel with RabbitMQ by issuing the Channel.Open RPC command. When RabbitMQ confirms the channel is open by sending the Channel.OpenOK RPC reply, the on_channel_open method will be invoked. """ LOGGER.info('Creating a new channel') self._connection.channel(on_open_callback=self.on_channel_open)
class ExamplePublisher(object): """This is an example publisher that will handle unexpected interactions with RabbitMQ such as channel and connection closures. If RabbitMQ closes the connection, it will reopen it. You should look at the output, as there are limited reasons why the connection may be closed, which usually are tied to permission related issues or socket timeouts. It uses delivery confirmations and illustrates one way to keep track of messages that have been sent and if they've been confirmed by RabbitMQ. """ EXCHANGE = 'message' EXCHANGE_TYPE = 'topic' PUBLISH_INTERVAL = .001 QUEUE = 'text' ROUTING_KEY = 'example.text' def __init__(self, app, ioloop): """Setup the example publisher object, passing in the URL we will use to connect to RabbitMQ. :param str amqp_url: The URL for connecting to RabbitMQ """ self._connection = None self._channel = None self._deliveries = [] self._acked = 0 self._nacked = 0 self._message_number = 0 self._stopping = False self.app = app self.ioloop = ioloop def connect(self): LOGGER.info('Connecting to RabbitMQ') # exc_type, exc_value, exc_traceback = sys.exc_info() # traceback.print_tb(exc_traceback, limit=None, file=sys.stdout) # self.connecting = True self._connection = TornadoConnection(rc.connParam, on_open_callback=self.on_connection_open) def close_connection(self): """This method closes the connection to RabbitMQ.""" LOGGER.info('Closing connection') self._connection.close() def add_on_connection_close_callback(self): """This method adds an on close callback that will be invoked by pika when RabbitMQ closes the connection to the publisher unexpectedly. """ LOGGER.info('Adding connection close callback') self._connection.add_on_close_callback(self.on_connection_closed) def on_connection_closed(self, method_frame): """This method is invoked by pika when the connection to RabbitMQ is closed unexpectedly. Since it is unexpected, we will reconnect to RabbitMQ if it disconnects. :param pika.frame.Method method_frame: The method frame from RabbitMQ """ LOGGER.warning('Server closed connection, reopening: (%s) %s', method_frame.method.reply_code, method_frame.method.reply_text) self._channel = None self._connection = self.connect() def on_connection_open(self, unused_connection): """This method is called by pika once the connection to RabbitMQ has been established. It passes the handle to the connection object in case we need it, but in this case, we'll just mark it unused. :type unused_connection: pika.SelectConnection """ LOGGER.info('Connection opened') self.add_on_connection_close_callback() self.open_channel() def add_on_channel_close_callback(self): """This method tells pika to call the on_channel_closed method if RabbitMQ unexpectedly closes the channel. """ LOGGER.info('Adding channel close callback') self._channel.add_on_close_callback(self.on_channel_closed) def on_channel_closed(self, method_frame): """Invoked by pika when RabbitMQ unexpectedly closes the channel. Channels are usually closed if you attempt to do something that violates the protocol, such as redeclare an exchange or queue with different paramters. In this case, we'll close the connection to shutdown the object. :param pika.frame.Method method_frame: The Channel.Close method frame """ LOGGER.warning('Channel was closed: (%s) %s', method_frame.method.reply_code, method_frame.method.reply_text) self._connection.close() def on_channel_open(self, channel): """This method is invoked by pika when the channel has been opened. The channel object is passed in so we can make use of it. Since the channel is now open, we'll declare the exchange to use. :param pika.channel.Channel channel: The channel object """ LOGGER.info('Channel opened') self._channel = channel self.add_on_channel_close_callback() self.setup_exchange() def setup_exchange(self): """Setup the exchange on RabbitMQ by invoking the Exchange.Declare RPC command. When it is complete, the on_exchange_declareok method will be invoked by pika. :param str|unicode exchange_name: The name of the exchange to declare """ LOGGER.info('Declaring exchange %s', rc.valueExngAttr["exchange"]) self._channel.exchange_declare(self.on_exchange_declareok, **rc.valueExngAttr) def on_exchange_declareok(self, unused_frame): """Invoked by pika when RabbitMQ has finished the Exchange.Declare RPC command. :param pika.Frame.Method unused_frame: Exchange.DeclareOk response frame """ LOGGER.info('Exchange declared') self.setup_queue(self.QUEUE) def setup_queue(self, queue_name): """Setup the queue on RabbitMQ by invoking the Queue.Declare RPC command. When it is complete, the on_queue_declareok method will be invoked by pika. :param str|unicode queue_name: The name of the queue to declare. """ LOGGER.info('Declaring queue %s', queue_name) self._channel.queue_declare(self.on_queue_declareok, queue_name) def on_queue_declareok(self, method_frame): """Method invoked by pika when the Queue.Declare RPC call made in setup_queue has completed. In this method we will bind the queue and exchange together with the routing key by issuing the Queue.Bind RPC command. When this command is complete, the on_bindok method will be invoked by pika. :param pika.frame.Method method_frame: The Queue.DeclareOk frame """ LOGGER.info('Binding %s to %s with %s', rc.valueExngAttr["exchange"], self.QUEUE, self.ROUTING_KEY) self._channel.queue_bind(self.on_bindok, self.QUEUE, rc.valueExngAttr["exchange"], self.ROUTING_KEY) def on_delivery_confirmation(self, method_frame): """Invoked by pika when RabbitMQ responds to a Basic.Publish RPC command, passing in either a Basic.Ack or Basic.Nack frame with the delivery tag of the message that was published. The delivery tag is an integer counter indicating the message number that was sent on the channel via Basic.Publish. Here we're just doing house keeping to keep track of stats and remove message numbers that we expect a delivery confirmation of from the list used to keep track of messages that are pending confirmation. :param pika.frame.Method method_frame: Basic.Ack or Basic.Nack frame """ confirmation_type = method_frame.method.NAME.split('.')[1].lower() LOGGER.info('Received %s for delivery tag: %i', confirmation_type, method_frame.method.delivery_tag) if confirmation_type == 'ack': self._acked += 1 elif confirmation_type == 'nack': self._nacked += 1 self._deliveries.remove(method_frame.method.delivery_tag) LOGGER.info('Published %i messages, %i have yet to be confirmed, ' '%i were acked and %i were nacked', self._message_number, len(self._deliveries), self._acked, self._nacked) def enable_delivery_confirmations(self): """Send the Confirm.Select RPC method to RabbitMQ to enable delivery confirmations on the channel. The only way to turn this off is to close the channel and create a new one. When the message is confirmed from RabbitMQ, the on_delivery_confirmation method will be invoked passing in a Basic.Ack or Basic.Nack method from RabbitMQ that will indicate which messages it is confirming or rejecting. """ LOGGER.info('Issuing Confirm.Select RPC command') self._channel.confirm_delivery(self.on_delivery_confirmation) def publish_message(self): """If the class is not stopping, publish a message to RabbitMQ, appending a list of deliveries with the message number that was sent. This list will be used to check for delivery confirmations in the on_delivery_confirmations method. Once the message has been sent, schedule another message to be sent. The main reason I put scheduling in was just so you can get a good idea of how the process is flowing by slowing down and speeding up the delivery intervals by changing the PUBLISH_INTERVAL constant in the class. """ if self._stopping: return message = 'The current epoch value is %i' % time.time() properties = pika.BasicProperties(app_id='example-publisher', content_type='text/plain') self._channel.basic_publish(rc.valueExngAttr["exchange"], self.ROUTING_KEY, message, properties) self._message_number += 1 self._deliveries.append(self._message_number) LOGGER.info('Published message # %i', self._message_number) self.schedule_next_message() def schedule_next_message(self): """If we are not closing our connection to RabbitMQ, schedule another message to be delivered in PUBLISH_INTERVAL seconds. """ if self._stopping: return LOGGER.info('Scheduling next message for %0.1f ms', self.PUBLISH_INTERVAL) self._connection.add_timeout(self.PUBLISH_INTERVAL, self.publish_message) def start_publishing(self): """This method will enable delivery confirmations and schedule the first message to be sent to RabbitMQ """ LOGGER.info('Issuing consumer related RPC commands') self.enable_delivery_confirmations() self.schedule_next_message() def on_bindok(self, unused_frame): """This method is invoked by pika when it receives the Queue.BindOk response from RabbitMQ. Since we know we're now setup and bound, it's time to start publishing.""" LOGGER.info('Queue bound') self.start_publishing() def close_channel(self): """Invoke this command to close the channel with RabbitMQ by sending the Channel.Close RPC command. """ LOGGER.info('Closing the channel') self._channel.close() def open_channel(self): """This method will open a new channel with RabbitMQ by issuing the Channel.Open RPC command. When RabbitMQ confirms the channel is open by sending the Channel.OpenOK RPC reply, the on_channel_open method will be invoked. """ LOGGER.info('Creating a new channel') self._connection.channel(on_open_callback=self.on_channel_open)
class PikaClient(object): def __init__(self): # Construct a queue name we'll use for this instance only self.queue_name = "hello" # Default values self.connected = False self.connecting = False self.connection = None self.channel = None # A place for us to keep messages sent to us by Rabbitmq self.messages = list() # A place for us to put pending messages while we're waiting to connect self.pending = list() def connect(self): if self.connecting: pika.log.info('PikaClient: Already connecting to RabbitMQ') return pika.log.info('PikaClient: Connecting to RabbitMQ on localhost:5672') self.connecting = True param = pika.ConnectionParameters(host='115.146.93.175') self.connection = TornadoConnection(param, on_open_callback=self.on_connected) self.connection.add_on_close_callback(self.on_closed) def on_connected(self, connection): pika.log.info('PikaClient: Connected to RabbitMQ on localhost:5672') self.connected = True self.connection = connection self.connection.channel(self.on_channel_open) def on_channel_open(self, channel): pika.log.info('PikaClient: Channel Open, Declaring Exchange') self.channel = channel self.channel.exchange_declare(exchange='', #type="direct", #auto_delete=True, #durable=False, callback=self.on_exchange_declared) def on_exchange_declared(self, frame): pika.log.info('PikaClient: Exchange Declared, Declaring Queue') self.channel.queue_declare(queue=self.queue_name, #auto_delete=True, #durable=False, #exclusive=False, callback=self.on_queue_declared) def on_queue_declared(self, frame): pika.log.info('PikaClient: Queue Declared, Binding Queue') self.channel.queue_bind(exchange='', queue=self.queue_name, routing_key='hello', callback=self.on_queue_bound) def on_queue_bound(self, frame): pika.log.info('PikaClient: Queue Bound, Issuing Basic Consume') self.channel.basic_consume(consumer_callback=self.on_pika_message, queue=self.queue_name, no_ack=True) # Send any messages pending for properties, body in self.pending: self.channel.basic_publish(exchange='', routing_key='hello', body=body, properties=properties) def on_pika_message(self, channel, method, header, body): pika.log.info('PikaCient: Message receive, delivery tag #%i' % \ method.delivery_tag) # Append it to our messages list self.messages.append(body) def on_basic_cancel(self, frame): pika.log.info('PikaClient: Basic Cancel Ok') # If we don't have any more consumer processes running close self.connection.close() def on_closed(self, connection): # We've closed our pika connection so stop the demo tornado.ioloop.IOLoop.instance().stop() def sample_message(self, tornado_request): # Build a message to publish to RabbitMQ body = '%.8f: Request from %s [%s]' % \ (tornado_request._start_time, tornado_request.remote_ip, tornado_request.headers.get("User-Agent")) # Send the message properties = pika.BasicProperties(content_type="text/plain", delivery_mode=1) self.channel.basic_publish(exchange='', routing_key='hello', body=body, properties=properties) def get_messages(self): # Get the messages to return, then empty the list output = self.messages self.messages = list() return output
class PikaClient(object): ''' Helper class to manage RabbitMQ/Pika interface ''' def __init__(self): # Construct queue names request and response self.queue_name_req = 'request' self.queue_name_resp = 'response' # A place to keep requests and reponses from Rabbitmq self.req_listeners = set([]) self.resp_listeners = set([]) def connect(self): pika.log.info('PikaClient: Connecting to RabbitMQ on localhost:5672') credentials = pika.PlainCredentials('guest', 'guest') param = pika.ConnectionParameters(host='localhost', port=5672, virtual_host="/", credentials=credentials) self.connection = TornadoConnection(param, on_open_callback=self.on_connected) self.connection.add_on_close_callback(self.on_closed) def on_connected(self, connection): pika.log.info('PikaClient: Connected to RabbitMQ on localhost:5672') self.connection = connection self.connection.channel(self.on_channel_open) def on_channel_open(self, channel): pika.log.info('PikaClient: Channel Open, Declaring Exchange') self.channel = channel self.channel.exchange_declare(exchange='tornado', type="topic", auto_delete=True, durable=False, callback=self.on_exchange_declared) def on_exchange_declared(self, frame): pika.log.info('PikaClient: Exchange Declared, Declaring Queues') self.channel.queue_declare(queue=self.queue_name_req, auto_delete=True, durable=False, exclusive=False) self.channel.queue_declare(queue=self.queue_name_resp, auto_delete=True, durable=False, exclusive=False, callback=self.on_queue_declared) def on_queue_declared(self, frame): pika.log.info('PikaClient: Queues Declared, Binding Queues') self.channel.queue_bind(exchange='tornado', queue=self.queue_name_req, routing_key='*.longpoll.request') self.channel.queue_bind(exchange='tornado', queue=self.queue_name_resp, routing_key='*.longpoll.response', callback=self.on_queue_bound) def on_queue_bound(self, frame): pika.log.info('PikaClient: Queue Bound, Issuing Basic Consume') self.channel.basic_consume(consumer_callback=self.on_pika_req_message, queue=self.queue_name_req, no_ack=True) self.channel.basic_consume(consumer_callback=self.on_pika_resp_message, queue=self.queue_name_resp, no_ack=True) def on_pika_req_message(self, channel, method, header, body): log = 'PikaCient: Request Message received: %s' pika.log.info(log % body) self.notify_listeners(body, 'request') def on_pika_resp_message(self, channel, method, header, body): log = 'PikaCient: Response Message received: %s' pika.log.info(log % body) self.notify_listeners(body, 'response') def on_basic_cancel(self, frame): pika.log.info('PikaClient: Basic Cancel Ok') # If we don't have any more consumer processes running close self.connection.close() def on_closed(self, connection): # We've closed our pika connection so stop the demo tornado.ioloop.IOLoop.instance().stop() def server_test(self, tornado_request, action): # Prepare POST information for RabbitMQ if action == 'request': body = json.dumps(tornado_request.arguments) elif action == 'response': body = tornado_request.body # Send the message properties = pika.BasicProperties( content_type="application/json", delivery_mode=1 ) self.channel.basic_publish(exchange='tornado', routing_key='*.longpoll.' + action, properties=properties, body=body) def notify_listeners(self, message, action): # Deliver the message from RabbitMQ and finish the connection if action == 'request': listeners = self.req_listeners.copy() elif action == 'response': listeners = self.resp_listeners.copy() for listener in listeners: listener.finish(json.dumps(message)) pika.log.info('PikaClient: notified %s' % repr(listener)) self.remove_listener(listener, action) def add_listener(self, listener, action): if action == 'request': self.req_listeners.add(listener) elif action == 'response': self.resp_listeners.add(listener) pika.log.info('PikaClient: listener %s added' % repr(listener)) def remove_listener(self, listener, action): try: if action == 'request': self.req_listeners.remove(listener) elif action == 'response': self.resp_listeners.remove(listener) pika.log.info('PikaClient: listener %s removed' % repr(listener)) except KeyError: pass