class DT(threading.Thread): def __init__(self, connection, queue_name): super(DT, self).__init__() self.queue_name = queue_name self.connection = connection self.exchange = Exchange(name="event1", type="direct") self._queue = entity.Queue(queue_name, exchange=self.exchange, routing_key=queue_name) self.queue = SimpleQueue(self.connection, self._queue) def run(self): while True: try: self.main_loop() except: logging.exception("Unknown Exception: ") def main_loop(self): messages = [] print("inside", self.queue_name) messages.append(self.queue.get(block=True)) print("after ") messages[-1].ack() # Check if there is anymore messages try: while True: messages.append(self.queue.get(block=True, timeout=1)) messages[-1].ack() except Empty: pass for message in messages: print(message.payload)
def __init__(self, connection, queue_name): super(DT, self).__init__() self.queue_name = queue_name self.connection = connection self.exchange = Exchange(name="event1", type="direct") self._queue = entity.Queue(queue_name, exchange=self.exchange, routing_key=queue_name) self.queue = SimpleQueue(self.connection, self._queue)
def __init__(self, dc, exchange, exchange_type, queue, binding_key, **kwargs): if self.class_name is None: self.class_name = "KombuQueueReceiver" KombuMessageQueue.__init__(self, dc, exchange, exchange_type, queue, binding_key, **kwargs) self.queue_obj = SimpleQueue( self.channel, self.Queue_obj, ) self.uuid = gen_uuid() if not self.queue: raise RuntimeError("queue is required")
def init_rabbit_mq(self): try: self.logger.info("Initializing RabbitMQ message consumer...") schedule_exchange = Exchange("airtime-media-monitor", "direct", durable=True, auto_delete=True) schedule_queue = Queue("media-monitor", exchange=schedule_exchange, key="filesystem") self.connection = BrokerConnection(self.cfg["rabbitmq_host"], self.cfg["rabbitmq_user"], self.cfg["rabbitmq_password"], self.cfg["rabbitmq_vhost"]) channel = self.connection.channel() self.simple_queue = SimpleQueue(channel, schedule_queue) self.logger.info("Initialized RabbitMQ consumer.") except Exception as e: self.logger.info("Failed to initialize RabbitMQ consumer") self.logger.error(e) return False return True
def SimpleQueue(self, name, no_ack=None, queue_opts=None, exchange_opts=None, channel=None, **kwargs): """Create new :class:`~kombu.simple.SimpleQueue`, using a channel from this connection. If ``name`` is a string, a queue and exchange will be automatically created using that name as the name of the queue and exchange, also it will be used as the default routing key. :param name: Name of the queue/or a :class:`~kombu.entity.Queue`. :keyword no_ack: Disable acknowledgements. Default is false. :keyword queue_opts: Additional keyword arguments passed to the constructor of the automatically created :class:`~kombu.entity.Queue`. :keyword exchange_opts: Additional keyword arguments passed to the constructor of the automatically created :class:`~kombu.entity.Exchange`. :keyword channel: Channel to use. If not specified a new channel from the current connection will be used. Remember to call :meth:`~kombu.simple.SimpleQueue.close` when done with the object. """ from kombu.simple import SimpleQueue channel_autoclose = False if channel is None: channel = self.channel() channel_autoclose = True return SimpleQueue(channel, name, no_ack, queue_opts, exchange_opts, channel_autoclose=channel_autoclose, **kwargs)
def init_rabbit_mq(self): self.logger.info("Initializing RabbitMQ stuff") try: self.logger.info("rabbitmq_host: " + self.config["rabbitmq_host"]) self.logger.info("rabbitmq_user: "******"rabbitmq_user"]) self.logger.info("rabbitmq_password: "******"rabbitmq_password"]) self.logger.info("rabbitmq_vhost: " + self.config["rabbitmq_vhost"]) """""" schedule_exchange = \ Exchange("airtime-pypo", "direct", durable=True, auto_delete=True) schedule_queue = \ Queue("pypo-fetch", exchange=schedule_exchange, key="foo") connection = BrokerConnection(self.config["rabbitmq_host"], \ self.config["rabbitmq_user"], \ self.config["rabbitmq_password"], \ self.config["rabbitmq_vhost"]) channel = connection.channel() self.simple_queue = SimpleQueue(channel, schedule_queue) """ connection = Connection('amqp://*****:*****@172.16.82.1:5672//pypox') self.simple_queue = connection.SimpleQueue('pypo-fetch') #message = simple_queue.get(block=True, timeout=1) """ except Exception, e: self.logger.error(e) return False
def init_rabbit_mq(self): self.logger.info("Initializing RabbitMQ stuff") try: schedule_exchange = Exchange("airtime-pypo", "direct", durable=True, auto_delete=True) schedule_queue = Queue("pypo-fetch", exchange=schedule_exchange, key="foo") connection = BrokerConnection(config["rabbitmq_host"], config["rabbitmq_user"], config["rabbitmq_password"], config["rabbitmq_vhost"]) channel = connection.channel() self.simple_queue = SimpleQueue(channel, schedule_queue) except Exception, e: self.logger.error(e) return False
from __future__ import absolute_import, unicode_literals import datetime from kombu import Connection, Exchange, Queue from kombu.simple import SimpleQueue from amqp_buffer import AmqpBuffer with Connection('amqp://*****:*****@192.168.157.128:5672/') as conn: #simple_queue = AmqpBuffer(conn, 'simple_queue') for i in range(2): queue_name = "que"+str(i) exchange = Exchange(name="event1", type="direct") queue = Queue(queue_name, exchange=exchange, routing_key=queue_name) simple_queue = SimpleQueue(conn, queue) message = 'helloworld, sent at {0}'.format(datetime.datetime.today()) simple_queue.put(message) print(queue_name,'Sent: {0}'.format(message)) simple_queue.close()
def run(self): print 'pypo Pusher' if self.action == 'update_schedule': print 'update_schedule!!' credentials = pika.PlainCredentials(MQ_USER, MQ_PASS) connection = pika.BlockingConnection(pika.ConnectionParameters(MQ_HOST, 5672, '/airtime', credentials)) channel = connection.channel() channel.queue_declare(queue='pypo-fetch', durable=True) message = { 'schedule': { 'media': {} }, 'event_type': 'update_schedule' } import json message = json.dumps(message) message = 'hallo' channel.basic_publish(exchange='airtime-pypo', routing_key='pypo-fetch', body=message) channel.close() connection.close() if self.action == 'update_schedule_kombu': print 'update_schedule!!' exchange = Exchange("airtime-pypo", "direct", durable=True, auto_delete=True) queue = Queue("pypo-fetch", exchange=exchange, key="foo", durable=True) connection = BrokerConnection(MQ_HOST, MQ_USER, MQ_PASS, MQ_VHOST) channel = connection.channel() simple_queue = SimpleQueue(channel, queue) message = { 'schedule': { 'media': {} }, 'event_type': 'update_schedule' } print simple_queue.qsize() print 'do:' producer = Producer(channel, exchange=exchange, routing_key=None, serializer="json") producer.publish(message, routing_key='pypo-fetch') print simple_queue.qsize() channel.close()
class KombuQueueReceiver(KombuMessageQueue): def __init__(self, dc, exchange, exchange_type, queue, binding_key, **kwargs): if self.class_name is None: self.class_name = "KombuQueueReceiver" KombuMessageQueue.__init__(self, dc, exchange, exchange_type, queue, binding_key, **kwargs) self.queue_obj = SimpleQueue( self.channel, self.Queue_obj, ) self.uuid = gen_uuid() if not self.queue: raise RuntimeError("queue is required") def start_consuming(self): pass def dump_cache(self): return self.queue_obj.clear() def stop_consuming(self): self.queue_obj.close() @recover_connect def get_cached_number(self): return self.queue_obj.qsize() @recover_connect def get_nowait(self): try: log_message = self.queue_obj.get_nowait() except Empty: return None log_message.ack() # p = log_message.payload # deserialized data. # if isinstance(p, buffer): # p = unicode(p) return log_message.body @recover_connect def get(self, block=True, timeout=None): # logger.debug("Enter Message get callback method..") try: log_message = self.queue_obj.get(block=True, timeout=timeout) # logger.debug("get return : %s, dir: %s, type: %s", log_message, dir(log_message), type(log_message)) debug_message(log_message) except Empty: logger.debug("Empty error when get @todo infos..") raise Empty("{cn}[{id:#x}] is raise Empty when fetch.") # p = log_message.payload # deserialized data. log_message.ack() # remove message from queue # if isinstance(p, buffer): # p = unicode(p) return log_message.body def __del__(self): if not self.closed: logger.debug( "=============== {class_name}[{id:#x}] Enter closing... ". format(class_name=self.class_name, id=id(self))) self.close() def close(self): logger.debug( "{cn}[{id:#x}] is calling KombuQueueReceiver close method".format( cn=self.class_name, id=id(self))) try: logger.debug( "{cn}[{id:#x}] is about to closing.. Queue size: {qsize}". format(cn=self.class_name, id=id(self), qsize=self.get_cached_number())) # clean Rabbitmq Queue self.dump_cache() logger.debug( "{cn}[{id:#x}] is cleared. Queue size: {qsize}".format( cn=self.class_name, id=id(self), qsize=self.get_cached_number())) # stop all active consumers self.stop_consuming() # how to measure stop all consumers @todo # delete Rabbitmq Queue r = self.Queue_obj.delete() logger.debug( "{cn}[{id:#x}] delete Queue[{qid:#x}] return: {r}".format( cn=self.class_name, id=id(self), qid=id(self.Queue_obj), r=r)) except ChannelError as e: if '404' not in e.message: logger.error("Error when {cn}[{id:#x}] close: {msg}".format( cn=self.class_name, id=id(self), msg=traceback.format_exc())) KombuMessageQueue.close(self)