def test_publish__consume(self): connection = BrokerConnection(transport=Transport) channel = connection.channel() producer = Producer(channel, self.exchange, routing_key="test_Redis") consumer = Consumer(channel, self.queue) producer.publish({"hello2": "world2"}) _received = [] def callback(message_data, message): _received.append(message_data) message.ack() consumer.register_callback(callback) consumer.consume() self.assertIn(channel, channel.connection.cycle._channels) try: connection.drain_events(timeout=1) self.assertTrue(_received) self.assertRaises(socket.timeout, connection.drain_events, timeout=0.01) finally: channel.close()
def test_publish__consume(self): connection = BrokerConnection(transport=Transport) channel = connection.channel() producer = Producer(channel, self.exchange, routing_key="test_Redis") consumer = Consumer(channel, self.queue) producer.publish({"hello2": "world2"}) _received = [] def callback(message_data, message): _received.append(message_data) message.ack() consumer.register_callback(callback) consumer.consume() self.assertTrue(channel._poller._can_start()) try: connection.drain_events(timeout=1) self.assertTrue(_received) self.assertFalse(channel._poller._can_start()) self.assertRaises(socket.timeout, connection.drain_events, timeout=0.01) finally: channel.close()
class RabbitMQConnection: def connection(self): try: logger.info("Connection to the AMQP broker '" + AMQP_IP + ":" + AMQP_PORT + "' ...") connection = BrokerConnection(hostname=AMQP_IP, port=AMQP_PORT, userid=AMQP_USER, password=AMQP_PASSWORD, virtual_host="/", transport=AMQP_TRANSPORT) #channel = connection.channel() logger.info("Connection to the AMQP broker established.") return connection except Exception as exep: logger.warning("Could not connect to the AMQP broker '" + AMQP_IP + ":" + AMQP_PORT + "'. | " + str(exep)) def producer(self): self.connection = BrokerConnection(hostname=AMQP_IP, port=AMQP_PORT, userid=AMQP_USER, password=AMQP_PASSWORD, virtual_host="/", transport=AMQP_TRANSPORT) self.channel = self.connection.channel() # produce self.media_exchange = Exchange("media", "direct", durable=True) producer = Producer(self.channel, exchange=self.media_exchange, serializer="json") producer.publish({"name": "/tmp/lolcat1.avi", "size": 1301013}) print self.connection.get_transport_cls() def consumer(self): self.connection = BrokerConnection(hostname=AMQP_IP, port=AMQP_PORT, userid=AMQP_USER, password=AMQP_PASSWORD, virtual_host="/") self.channel = self.connection.channel() # consume self.media_exchange = Exchange("media", "direct", durable=True) self.video_queue = Queue("video", exchange=self.media_exchange, key="video") consumer = Consumer(self.channel, self.video_queue) consumer.register_callback(self.process_media) consumer.consume() # Process messages on all channels while True: self.connection.drain_events() def process_media(self, message_data, message): feed_url = message_data["name"] print ("Got feed import message for: %s" % feed_url) # something importing this feed url # import_feed(feed_url) message.ack()
class AMQPWorker(Worker): queues = [ {'routing_key': 'test', 'name': 'test', 'handler': 'handle_test' } ] _connection = None def handle_test(self, body, message): log.debug("Handle message: %s" % body) message.ack() def handle(self): log.debug("Start consuming") exchange = Exchange('amqp.topic', type='direct', durable=True) self._connection = BrokerConnection(*CONNECTION) channel = self._connection.channel() for entry in self.queues: log.debug("prepare to consume %s" % entry['routing_key']) queue = Queue(entry['name'], exchange=exchange, routing_key=entry['routing_key']) consumer = Consumer(channel, queue) consumer.register_callback(getattr(self, entry['handler'])) consumer.consume() log.debug("start consuming...") while True: try: self._connection.drain_events() except socket.timeout: log.debug("nothing to consume...") break self._connection.close() def run(self): while self.alive: try: self.handle() except Exception: self.alive = False raise def handle_quit(self, sig, frame): if self._connection is not None: self._connection.close() self.alive = False def handle_exit(self, sig, frame): if self._connection is not None: self._connection.close() self.alive = False sys.exit(0)
class AMQPWorker(Worker): queues = [{ 'routing_key': 'test', 'name': 'test', 'handler': 'handle_test' }] _connection = None def handle_test(self, body, message): log.debug("Handle message: %s" % body) message.ack() def handle(self): log.debug("Start consuming") exchange = Exchange('amqp.topic', type='direct', durable=True) self._connection = BrokerConnection(*CONNECTION) channel = self._connection.channel() for entry in self.queues: log.debug("prepare to consume %s" % entry['routing_key']) queue = Queue(entry['name'], exchange=exchange, routing_key=entry['routing_key']) consumer = Consumer(channel, queue) consumer.register_callback(getattr(self, entry['handler'])) consumer.consume() log.debug("start consuming...") while True: try: self._connection.drain_events() except socket.timeout: log.debug("nothing to consume...") break self._connection.close() def run(self): while self.alive: try: self.handle() except Exception: self.alive = False raise def handle_quit(self, sig, frame): if self._connection is not None: self._connection.close() self.alive = False def handle_exit(self, sig, frame): if self._connection is not None: self._connection.close() self.alive = False sys.exit(0)
def test_establish_connection(self): conn = BrokerConnection(port=5672, backend_cls=Backend) conn.connect() self.assertTrue(conn.connection.connected) self.assertEqual(conn.host, "localhost:5672") channel = conn.channel() self.assertTrue(channel.open) self.assertEqual(conn.drain_events(), "event") _connection = conn.connection conn.close() self.assertFalse(_connection.connected) self.assertIsInstance(conn.backend, Backend)
class Listener(): """ Simple class to wrap the operations needed for an AMQP listener """ def __init__(self, hostname="127.0.0.1", userid="guest", password="******", virtual_host="/", port=5672): """ Setup a connection to the AMQP server, get a channel Create a topic exchange, attach a bonded queue to it and register a consumer callback. A specific service listener implementation overrides the name and routing_key """ self.connection = BrokerConnection(hostname=hostname, userid=userid, password=password, virtual_host=virtual_host, port=port, insist=False, ssl=False) self.channel = self.connection.channel() self.exchange = Exchange(name=self.name, type="topic", durable=True, channel=self.channel) self.queue = Queue(self.name, exchange=self.exchange, routing_key=self.routing_key) self.queue = self.queue(self.channel) self.queue.declare() self.queue.consume(consumer_tag="", callback=self.callback, no_ack=True) self.connection.connect() return def callback(self, msg): """ This callback is run when a message is recieved """ return def consume(self): """ Event loop """ while True: self.connection.drain_events() return
class test_MemoryTransport(unittest.TestCase): def setUp(self): self.c = BrokerConnection(transport="memory") self.e = Exchange("test_transport_memory") self.q = Queue("test_transport_memory", exchange=self.e, routing_key="test_transport_memory") self.q2 = Queue("test_transport_memory2", exchange=self.e, routing_key="test_transport_memory2") def test_produce_consume_noack(self): channel = self.c.channel() producer = Producer(channel, self.e) consumer = Consumer(channel, self.q, no_ack=True) for i in range(10): producer.publish({"foo": i}, routing_key="test_transport_memory") _received = [] def callback(message_data, message): _received.append(message) consumer.register_callback(callback) consumer.consume() while 1: if len(_received) == 10: break self.c.drain_events() self.assertEqual(len(_received), 10) def test_produce_consume(self): channel = self.c.channel() producer = Producer(channel, self.e) consumer1 = Consumer(channel, self.q) consumer2 = Consumer(channel, self.q2) self.q2(channel).declare() for i in range(10): producer.publish({"foo": i}, routing_key="test_transport_memory") for i in range(10): producer.publish({"foo": i}, routing_key="test_transport_memory2") _received1 = [] _received2 = [] def callback1(message_data, message): _received1.append(message) message.ack() def callback2(message_data, message): _received2.append(message) message.ack() consumer1.register_callback(callback1) consumer2.register_callback(callback2) consumer1.consume() consumer2.consume() while 1: if len(_received1) + len(_received2) == 20: break self.c.drain_events() self.assertEqual(len(_received1) + len(_received2), 20) # compression producer.publish({"compressed": True}, routing_key="test_transport_memory", compression="zlib") m = self.q(channel).get() self.assertDictEqual(m.payload, {"compressed": True}) # queue.delete for i in range(10): producer.publish({"foo": i}, routing_key="test_transport_memory") self.assertTrue(self.q(channel).get()) self.q(channel).delete() self.q(channel).declare() self.assertIsNone(self.q(channel).get()) # queue.purge for i in range(10): producer.publish({"foo": i}, routing_key="test_transport_memory2") self.assertTrue(self.q2(channel).get()) self.q2(channel).purge() self.assertIsNone(self.q2(channel).get()) def test_drain_events(self): self.assertRaises(ValueError, self.c.drain_events, timeout=0.1) c1 = self.c.channel() c2 = self.c.channel() self.assertRaises(socket.timeout, self.c.drain_events, timeout=0.1) del(c1) # so pyflakes doesn't complain. del(c2) def test_drain_events_unregistered_queue(self): c1 = self.c.channel() class Cycle(object): def get(self): return ("foo", "foo"), c1 self.c.transport.cycle = Cycle() self.assertRaises(KeyError, self.c.drain_events)
connection = BrokerConnection( hostname='localhost', userid='gamelion', password='******', virtual_host='/' ) channel = connection.channel() consumer = Consumer(channel, server_queue) consumer.register_callback(receive_message) consumer.consume(no_ack=True) while True: try: connection.drain_events(timeout=CONSUME_TIMEOUT) except socket.timeout: # flush the server buffer if we haven't gotten # any messages in a while if len(server_buffer) > 0: logging.debug('QUEUE TIMED OUT, FLUSHING BUFFER') process_servers() elif display_waiting: processing_time = datetime.now() -\ last_waiting_time -\ timedelta(seconds=CONSUME_TIMEOUT) logging.debug( '... WAITING ... TIME SINCE LAST: %s', processing_time )
self.consumer.register_callback(mq_callback) c1 = AConsumer("test_1","test.1") c2 = AConsumer("testing","test.ing") # consumers can use simple pattern matching when defining a queue c3 = AConsumer("test_all","test.*") # 3. publish something to consume # publishers always send to a specific route, the mq will route to the queues producer = Producer(pub_chan, exchange=pub_exch, serializer=message_serializer) producer.publish({"name": "Shane Caraveo", "username": "******"}, routing_key="test.1") producer.publish({"name": "Micky Mouse", "username": "******"}, routing_key="test.ing") producer.publish({"name": "Anonymous", "username": "******"}, routing_key="test.foobar") def have_messages(): return sum([q.qsize() for q in cons_chan.queues.values()]) # 5. run the event loop while have_messages(): try: cons_conn.drain_events() except KeyboardInterrupt: print print "quitting" break except Exception, e: import traceback print traceback.format_exc() break