def Connect(self): self.connection = Connection( hostname=self.host, port=self.port, userid=self.usr, password=self.psw, virtual_host=self.virtual_host, transport='kombu.transport.pyamqp:Transport') self.channel = self.connection.channel() self.producer = Producer(self.channel) self.task_count = 0 if self.needresult: self.back_queue = Queue(name=self.result_queuename, auto_delete=True, exclusive=True, durable=False) self.consumer = Consumer(self.channel, self.back_queue, no_ack=True) self.consumer.qos(prefetch_count=1) self.consumer.register_callback(self.on_response) self.callback_queue = self.consumer.queues[0].name self.consumer.consume() else: self.callback_queue = None
def _connect(self, channel): self._queue = Queue(channel=channel, **self._queue_kwargs) self._queue.declare() self._consumer = Consumer(channel, [self._queue], callbacks=[self._callback]) self._consumer.consume()
def test_publish__consume(self): connection = BrokerConnection(transport=Transport) channel = connection.channel() producer = Producer(channel, self.exchange, routing_key="test_Redis") consumer = Consumer(channel, self.queue) producer.publish({"hello2": "world2"}) _received = [] def callback(message_data, message): _received.append(message_data) message.ack() consumer.register_callback(callback) consumer.consume() self.assertTrue(channel._poller._can_start()) try: connection.drain_events(timeout=1) self.assertTrue(_received) self.assertFalse(channel._poller._can_start()) self.assertRaises(socket.timeout, connection.drain_events, timeout=0.01) finally: channel.close()
def test_consume__cancel(self): channel = self.connection.channel() binding = Binding("qname", self.exchange, "rkey") consumer = Consumer(channel, binding, auto_declare=True) consumer.cancel() self.assertIn("basic_cancel", channel) self.assertFalse(consumer._active_tags)
def test_decode_error(self): channel = self.connection.channel() b1 = Queue('qname1', self.exchange, 'rkey') consumer = Consumer(channel, [b1]) consumer.channel.throw_decode_error = True with self.assertRaises(ValueError): consumer._receive_callback({'foo': 'bar'})
def consumer(self, wakeup=True): """Create event consumer.""" consumer = Consumer(self.connection, queues=[self.queue], no_ack=True) consumer.register_callback(self._receive) with consumer: if wakeup: self.wakeup_workers(channel=consumer.channel) yield consumer
def test_decode_error(self): channel = self.connection.channel() b1 = Queue("qname1", self.exchange, "rkey") consumer = Consumer(channel, [b1]) consumer.channel.throw_decode_error = True with self.assertRaises(ValueError): consumer._receive_callback({"foo": "bar"})
def test_consume__cancel(self): channel = self.connection.channel() queue = Queue('qname', self.exchange, 'rkey') consumer = Consumer(channel, queue, auto_declare=True) consumer.consume() consumer.cancel() self.assertIn('basic_cancel', channel) self.assertFalse(consumer._active_tags)
def get_consumers(self, _, channel): consumer = Consumer(channel, queues=[self.provider.queue], accept=self.accept, no_ack=False, callbacks=[self.provider.handle_message]) consumer.qos(prefetch_count=self.prefetch_count) return [consumer]
def consume_msg(channel): private_exchange = Exchange("private", "direct", durable=True) document_queue = Queue("private", exchange=private_exchange, routing_key="private") consumer = Consumer(channel, document_queue, callbacks=[process_document]) #consumer.register_callback(process_document) consumer.consume() while True: channel.connection.drain_events()
def test_purge(self): channel = self.connection.channel() b1 = Queue('qname1', self.exchange, 'rkey') b2 = Queue('qname2', self.exchange, 'rkey') b3 = Queue('qname3', self.exchange, 'rkey') b4 = Queue('qname4', self.exchange, 'rkey') consumer = Consumer(channel, [b1, b2, b3, b4], auto_declare=True) consumer.purge() self.assertEqual(channel.called.count('queue_purge'), 4)
def init(host,port,virtual_host,usr,psw,queue_name): global connection,channel,producer,task_queue,consumer connection = Connection(hostname=host,port=port,userid=usr,password=psw,virtual_host=virtual_host) channel = connection.channel() producer=Producer(channel) task_queue = Queue(queue_name,durable=True) consumer = Consumer(channel,task_queue,no_ack=False) consumer.qos(prefetch_count=1) consumer.register_callback(RequestCallBack)
def test_purge(self): channel = self.connection.channel() b1 = Queue("qname1", self.exchange, "rkey") b2 = Queue("qname2", self.exchange, "rkey") b3 = Queue("qname3", self.exchange, "rkey") b4 = Queue("qname4", self.exchange, "rkey") consumer = Consumer(channel, [b1, b2, b3, b4], auto_declare=True) consumer.purge() self.assertEqual(channel.called.count("queue_purge"), 4)
def test_produce_consume(self): channel = self.c.channel() producer = Producer(channel, self.e) consumer1 = Consumer(channel, self.q) consumer2 = Consumer(channel, self.q2) self.q2(channel).declare() for i in range(10): producer.publish({"foo": i}, routing_key="test_transport_memory") for i in range(10): producer.publish({"foo": i}, routing_key="test_transport_memory2") _received1 = [] _received2 = [] def callback1(message_data, message): _received1.append(message) message.ack() def callback2(message_data, message): _received2.append(message) message.ack() consumer1.register_callback(callback1) consumer2.register_callback(callback2) consumer1.consume() consumer2.consume() while 1: if len(_received1) + len(_received2) == 20: break self.c.drain_events() self.assertEqual(len(_received1) + len(_received2), 20) # compression producer.publish({"compressed": True}, routing_key="test_transport_memory", compression="zlib") m = self.q(channel).get() self.assertDictEqual(m.payload, {"compressed": True}) # queue.delete for i in range(10): producer.publish({"foo": i}, routing_key="test_transport_memory") self.assertTrue(self.q(channel).get()) self.q(channel).delete() self.q(channel).declare() self.assertIsNone(self.q(channel).get()) # queue.purge for i in range(10): producer.publish({"foo": i}, routing_key="test_transport_memory2") self.assertTrue(self.q2(channel).get()) self.q2(channel).purge() self.assertIsNone(self.q2(channel).get())
def test_revive(self): channel = self.connection.channel() b1 = Queue("qname1", self.exchange, "rkey") consumer = Consumer(channel, [b1]) channel2 = self.connection.channel() consumer.revive(channel2) self.assertIs(consumer.channel, channel2) self.assertIs(consumer.queues[0].channel, channel2) self.assertIs(consumer.queues[0].exchange.channel, channel2)
class MessConsumer(object): def __init__(self, mess, connection, name, exchange_name): self._mess = mess self._conn = connection self._name = name self._exchange = Exchange(name=exchange_name, type='topic', durable=False, auto_delete=True) #TODO parameterize self._channel = None self._ops = {} self.connect() def connect(self): self._channel = self._conn.channel() self._queue = Queue(channel=self._channel, name=self._name, exchange=self._exchange, routing_key=self._name) self._queue.declare() self._consumer = Consumer(self._channel, [self._queue], callbacks=[self._callback]) self._consumer.consume() def consume(self): while True: self._conn.drain_events() def _callback(self, body, message): reply_to = message.headers.get('reply-to') #TODO error handling for message format op = body['op'] args = body['args'] kwargs = body['kwargs'] #TODO error handling for unknown op op_fun = self._ops[op] ret, err = None, None try: ret = op_fun(*args, **kwargs) except Exception: err = sys.exc_info() finally: if reply_to: if err: tb = traceback.format_exception(*err) err = (err[0].__name__, str(err[1]), tb) reply = dict(result=ret, error=err) self._mess.reply(reply_to, reply) message.ack() def add_op(self, name, fun): self._ops[name] = fun
def test___enter____exit__(self): channel = self.connection.channel() queue = Queue("qname", self.exchange, "rkey") consumer = Consumer(channel, queue, auto_declare=True) context = consumer.__enter__() self.assertIs(context, consumer) self.assertTrue(consumer._active_tags) consumer.__exit__() self.assertIn("basic_cancel", channel) self.assertFalse(consumer._active_tags)
def consumer(self, queue_name): video_queue = Queue(queue_name, exchange=self.media_exchange) consumer = Consumer(self.channel, video_queue) consumer.register_callback(self.process_media) consumer.consume() # Process messages on all channels while True: self.connection.drain_events() pass
class AConsumer: def __init__(self, queue_name, key): self.queue = Queue(queue_name, exchange=cons_exch, routing_key=key) self.consumer = Consumer(cons_chan, [self.queue]) self.consumer.consume() def mq_callback(message_data, message): print("%s: %r: %r" % (key, message.delivery_info, message_data,)) #message.ack() self.consumer.register_callback(mq_callback)
def test___enter____exit__(self): channel = self.connection.channel() queue = Queue('qname', self.exchange, 'rkey') consumer = Consumer(channel, queue, auto_declare=True) context = consumer.__enter__() self.assertIs(context, consumer) self.assertTrue(consumer._active_tags) res = consumer.__exit__(None, None, None) self.assertFalse(res) self.assertIn('basic_cancel', channel) self.assertFalse(consumer._active_tags)
def test_basic_reject__requeue(self): channel = self.connection.channel() b1 = Queue("qname1", self.exchange, "rkey") consumer = Consumer(channel, [b1]) def callback(message_data, message): message.requeue() consumer.register_callback(callback) consumer._receive_callback({"foo": "bar"}) self.assertIn("basic_reject:requeue", channel)
def get_consumers(self): """ Create two consumers for each message processor: one for the outgoing message queue, and one for the incoming message queue. """ consumers = {} # import dynamically (use the import path in the settings file) all # message processor then create one instance for each of them mps = (import_class(mp) for mp in settings.MESSAGE_PROCESSORS) self.message_processors = [mp() for mp in mps] # Just a log loop to say that we do mps = (mp.rsplit('.', 1)[1] for mp in settings.MESSAGE_PROCESSORS) self.logger.info('Loading message processors: %s' % ', '.join(mps)) # Create the consumer for incoming messages and attach the callback # of each message processor queue = self.queues['incoming_messages'] c = consumers['incoming_messages'] = Consumer(self.channel, queue) for mp in self.message_processors: c.register_callback(mp.handle_incoming_message) c.consume() # Create the consumer for incoming messages and attach the callback # of each message processor # then attach a router callback that is going to relay the message # to the proper transport queue queue = self.queues['outgoing_messages'] c = consumers['outgoing_messages'] = Consumer(self.channel, queue) for mp in self.message_processors: c.register_callback(mp.handle_outgoing_message) c.register_callback(self.relay_message_to_transport) c.consume() # Create the consumer for the log messages and attach a callback # from the SMS router: all messages sent to this queue are going # to be logged in the router log consumers['logs'] = Consumer(self.channel, self.queues['logs']) consumers['logs'].register_callback(self.handle_log) consumers['logs'].consume() # attach a fall back functions to handle message that kombu can't deliver queue = self.queues['undelivered_kombu_message'] c = consumers['undeliverd_kombu_messages'] = Consumer( self.channel, queue) c.register_callback(self.handle_undelivered_kombu_message) c.consume()
def test_basic_ack_twice(self): channel = self.connection.channel() b1 = Queue("qname1", self.exchange, "rkey") consumer = Consumer(channel, [b1]) def callback(message_data, message): message.ack() message.ack() consumer.register_callback(callback) with self.assertRaises(MessageStateError): consumer._receive_callback({"foo": "bar"})
def test_basic_ack_twice(self): channel = self.connection.channel() b1 = Queue("qname1", self.exchange, "rkey") consumer = Consumer(channel, [b1]) def callback(message_data, message): message.ack() message.ack() consumer.register_callback(callback) self.assertRaises(MessageStateError, consumer._receive_callback, {"foo": "bar"})
def init_rabbit_mq(self): self.logger.info("Initializing RabbitMQ stuff") try: schedule_exchange = Exchange("airtime-media-monitor", "direct", durable=True, auto_delete=True) schedule_queue = Queue("media-monitor", exchange=schedule_exchange, key="filesystem") self.connection = BrokerConnection(self.config.cfg["rabbitmq_host"], self.config.cfg["rabbitmq_user"], self.config.cfg["rabbitmq_password"], self.config.cfg["rabbitmq_vhost"]) channel = self.connection.channel() consumer = Consumer(channel, schedule_queue) consumer.register_callback(self.handle_message) consumer.consume() except Exception, e: self.logger.error(e) return False
def test_basic_reject_twice(self): channel = self.connection.channel() b1 = Queue('qname1', self.exchange, 'rkey') consumer = Consumer(channel, [b1]) def callback(message_data, message): message.reject() message.reject() consumer.register_callback(callback) with self.assertRaises(MessageStateError): consumer._receive_callback({'foo': 'bar'}) self.assertIn('basic_reject', channel)
def test_basic_reject__requeue_twice(self): channel = self.connection.channel() b1 = Binding("qname1", self.exchange, "rkey") consumer = Consumer(channel, [b1]) def callback(message_data, message): message.requeue() message.requeue() consumer.register_callback(callback) self.assertRaises(MessageStateError, consumer._receive_callback, {"foo": "bar"}) self.assertIn("basic_reject:requeue", channel)
def main(): cfg = { 'hostname': 'localhost', 'userid': 'guest', 'password': '******', 'virtual_host': '/', 'port': 5672 } transport = 'pika' #transport = 'librabbitmq' connection = BrokerConnection(transport=transport, **cfg) connection.connect() cfg = { 'name': 'simple-test-1', 'auto_delete': True, 'durable': False, 'delivery_mode': 'transient' } channel = connection.channel() exchange = Exchange(channel=channel, **cfg) #exchange = exchange_def(channel) routing_key = 'simple-test-1-route' queue = Queue(exchange=exchange, routing_key=routing_key, **cfg) channel = connection.channel() producer = Producer(channel=channel, exchange=exchange, routing_key=routing_key) channel = connection.channel() consumer = Consumer(channel=channel, queues=[queue], callbacks=[receive]) consumer.consume() def serve_forever(): while True: #print 'drain' #gevent.sleep(0.0001) connection.drain_events(timeout=1) def publish_forever(): while True: producer.publish(loremIpsum) gevent.sleep(0.0001) #g1, g2 = gevent.spawn(publish_forever), gevent.spawn(serve_forever) g2 = gevent.spawn(serve_forever) g1 = gevent.spawn(publish_forever) gevent.joinall([g1, g2])
def init_rabbit_mq(self): logger = logging.getLogger('fetch') logger.info("Initializing RabbitMQ stuff") try: schedule_exchange = Exchange("airtime-schedule", "direct", durable=True, auto_delete=True) schedule_queue = Queue("pypo-fetch", exchange=schedule_exchange, key="foo") self.connection = BrokerConnection(config["rabbitmq_host"], config["rabbitmq_user"], config["rabbitmq_password"], "/") channel = self.connection.channel() consumer = Consumer(channel, schedule_queue) consumer.register_callback(handle_message) consumer.consume() except Exception, e: logger.error(e) return False
def consume_msg(channel): private_exchange = Exchange("private", "direct", durable=True) document_queue = Queue("private", exchange=private_exchange, routing_key="private") consumer = Consumer(channel, document_queue) consumer.register_callback(process_document) consumer.consume() try: while True: channel.connection.drain_events(timeout=5) except socket.timeout: return HttpResponse("ok") else: return HttpResponse("error")
def consumer(self): """Create event consumer. .. warning:: This creates a new channel that needs to be closed by calling `consumer.channel.close()`. """ consumer = Consumer(self.connection.channel(), queues=[self.queue], no_ack=True) consumer.register_callback(self._receive) return consumer
def consumer(self, queue_name, queue_key): video_queue = Queue(queue_name, exchange=self.media_exchange, routing_key=queue_key, auto_delete=True) consumer = Consumer(self.channel, video_queue) #To delete the queue #bound_science_news = video_queue(self.channel) #bound_science_news.delete() consumer.register_callback(self.process_media) consumer.consume() # Process messages on all channels while True: self.connection.drain_events() pass
def test_multiple_bindings(self): channel = self.connection.channel() b1 = Binding("qname1", self.exchange, "rkey") b2 = Binding("qname2", self.exchange, "rkey") b3 = Binding("qname3", self.exchange, "rkey") b4 = Binding("qname4", self.exchange, "rkey") consumer = Consumer(channel, [b1, b2, b3, b4]) self.assertEqual(channel.called.count("exchange_declare"), 4) self.assertEqual(channel.called.count("queue_declare"), 4) self.assertEqual(channel.called.count("queue_bind"), 4) self.assertEqual(channel.called.count("basic_consume"), 4) self.assertEqual(len(consumer._active_tags), 4) consumer.cancel() self.assertEqual(channel.called.count("basic_cancel"), 4) self.assertFalse(len(consumer._active_tags))
def init(host, port, virtual_host, usr, psw, queue_name, exchange_name=None, routing_key=None): global connection, channel, producer, task_queue, consumer connection = Connection(hostname=host, port=port, userid=usr, password=psw, virtual_host=virtual_host) channel = connection.channel() channel.auto_decode = False producer = Producer(channel) if exchange_name: exchange = Exchange(exchange_name, "topic", channel, durable=True, delivery_mode=2) task_queue = Queue(queue_name, routing_key=routing_key, durable=True, exchange=exchange) else: task_queue = Queue(queue_name, durable=True) consumer = Consumer(channel, task_queue, no_ack=False) consumer.qos(prefetch_count=1) consumer.register_callback(RequestCallBack)
def test_on_decode_error_callback(self): channel = self.connection.channel() b1 = Queue("qname1", self.exchange, "rkey") thrown = [] def on_decode_error(msg, exc): thrown.append((msg.body, exc)) consumer = Consumer(channel, [b1], on_decode_error=on_decode_error) consumer.channel.throw_decode_error = True consumer._receive_callback({"foo": "bar"}) self.assertTrue(thrown) m, exc = thrown[0] self.assertEqual(anyjson.loads(m), {"foo": "bar"}) self.assertIsInstance(exc, ValueError)
def get_consumers(self, _, default_channel): self._default_channel = default_channel consumers = [] exchange = Exchange(self._exchange, type=self._exchange_type, durable=True, auto_delete=False) for _routing_key, handles in self.handler.handle_map.items(): if not isinstance(handles, list): handles = [handles] for handle in handles: channel = default_channel.connection.channel() self._channels[id(channel)] = channel _queue = '{}.{}'.format(handle.__module__, handle.__name__) consumers.append( Consumer(channel=channel, queues=[ Queue(_queue, exchange, routing_key=_routing_key, durable=True, auto_delete=False) ], callbacks=[partial(self.on_message, handle)], on_decode_error=self.on_decode_error)) return consumers
def test_on_message_is_sending_to_reply_queue(self, conn): ret_result = 'foooo' class Foo(A): class state: def bar(self, my_bar): return ret_result a = Foo(conn) ticket = uuid() delivery_tag = uuid() body, message = get_encoded_test_message('bar', {'my_bar': 'bar_arg'}, A.__class__.__name__, reply_to=ticket, delivery_tag=delivery_tag) # Set up a reply queue to read from # reply_q and reply_exchange should be set the sender a.reply_exchange = a.reply_exchange.bind(a.connection.default_channel) maybe_declare(a.reply_exchange) reply_q = a.get_reply_queue(ticket) reply_q(a.connection.default_channel).declare() a._on_message(body, message) a_con = Consumer(conn.channel(), reply_q) self.assertNextMsgDataEqual(a_con, {'ok': ret_result})
def test_set_callbacks(self): channel = self.connection.channel() queue = Queue('qname', self.exchange, 'rkey') callbacks = [lambda x, y: x, lambda x, y: x] consumer = Consumer(channel, queue, auto_declare=True, callbacks=callbacks) self.assertEqual(consumer.callbacks, callbacks)
def _setup_consumer(self): if self.consumer is not None: try: self.consumer.cancel() except socket.error: # pragma: no cover # On some systems (e.g. os x) we need to explicitly cancel the # consumer here. However, e.g. on ubuntu 14.04, the # disconnection has already closed the socket. We try to # cancel, and ignore any socket errors. pass channel = self.connection.channel() # queue.bind returns a bound copy self.queue = self.queue.bind(channel) maybe_declare(self.queue, channel) consumer = Consumer(channel, queues=[self.queue], no_ack=False) consumer.callbacks = [self.on_message] consumer.consume() self.consumer = consumer
def get_consumers(self, Consumer, channel): consume_from = list() for queue in settings.CONSUME_FROM: consume_from.append(self.queues.get(queue)) consumer = [ Consumer(queues=consume_from, on_message=self.on_message, prefetch_count=settings.PREFETCH_COUNT) ] return consumer
def test_publish__consume(self): connection = BrokerConnection(transport=Transport) channel = connection.channel() producer = Producer(channel, self.exchange, routing_key="test_Redis") consumer = Consumer(channel, self.queue) producer.publish({"hello2": "world2"}) _received = [] def callback(message_data, message): _received.append(message_data) message.ack() consumer.register_callback(callback) consumer.consume() self.assertIn(channel, channel.connection.cycle._channels) try: connection.drain_events(timeout=1) self.assertTrue(_received) self.assertRaises(socket.timeout, connection.drain_events, timeout=0.01) finally: channel.close()
def _collect(self, ticket, limit=None, timeout=1, callback=None, channel=None): chan = channel or self.connection.channel() queue = self.get_reply_queue(ticket) consumer = Consumer(channel, [queue], no_ack=True) responses = [] def on_message(body, message): if callback: callback(body) responses.append(body) try: consumer.register_callback(on_message) consumer.consume() for i in limit and range(limit) or count(): try: self.connection.drain_events(timeout=timeout) except socket.timeout: break chan.after_reply_message_received(queue.name) return responses finally: channel or chan.close()
def init_rabbit_mq(self): """ This function will attempt to connect to RabbitMQ Server and if successful return 'True'. Returns 'False' otherwise. """ self.logger.info("Initializing RabbitMQ stuff") try: schedule_exchange = Exchange("airtime-media-monitor", "direct", durable=True, auto_delete=True) schedule_queue = Queue("media-monitor", exchange=schedule_exchange, key="filesystem") self.connection = BrokerConnection( self.config.cfg["rabbitmq_host"], self.config.cfg["rabbitmq_user"], self.config.cfg["rabbitmq_password"], self.config.cfg["rabbitmq_vhost"]) channel = self.connection.channel() consumer = Consumer(channel, schedule_queue) consumer.register_callback(self.handle_message) consumer.consume() except Exception, e: self.logger.error(e) return False
def worker(mq_url): connection = Connection(mq_url) channel = connection.channel() consumer_json = Consumer(channel, task_json_queue, callbacks=[process_json], accept=['json']) consumer_json.consume() consumer_pickle = Consumer(channel, task_pickle_queue, callbacks=[process_pickle], accept=['pickle']) consumer_pickle.consume() while True: connection.drain_events()
def test_consume__cancel(self): channel = self.connection.channel() queue = Queue("qname", self.exchange, "rkey") consumer = Consumer(channel, queue, auto_declare=True) consumer.consume() consumer.cancel() self.assertIn("basic_cancel", channel) self.assertFalse(consumer._active_tags)
def consume(self, connection, queue_name): self._consume_body = None self._consume_message = None with Consumer(channel=connection.default_channel, queues=[Queue(name=queue_name)], callbacks=[self._consume_callback], no_ack=True, auto_declare=False) as consumer: try: connection.drain_events(timeout=3) yield self._consume_body, self._consume_message except socket.timeout: yield None, None
def test_basic_reject__requeue(self): channel = self.connection.channel() b1 = Queue('qname1', self.exchange, 'rkey') consumer = Consumer(channel, [b1]) def callback(message_data, message): message.requeue() consumer.register_callback(callback) consumer._receive_callback({'foo': 'bar'}) self.assertIn('basic_reject:requeue', channel)
def run(self): try: connection = Connection(hostname=self.host, port=self.port, userid=self.usr, password=self.psw, virtual_host=self.virtual_host) channel = connection.channel() self.producer = Producer(channel) task_queue = Queue(self.queue_name, durable=True) consumer = Consumer(channel, task_queue, no_ack=False) consumer.qos(prefetch_count=1) consumer.register_callback(self.RequestCallBack) consumer.consume() while True: connection.drain_events() connection.close() except BaseException, e: print e